Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_event_declarations,
25        validate_hello_rich_ui_descriptor, validate_hello_self_signal_declarations,
26        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
27        ProviderRole,
28    },
29    scope::{
30        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
31        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
32        SCOPE_SYNC_OP,
33    },
34    session::{
35        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
36        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
37        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
38        ROLE_VERSIONS_FIELD,
39    },
40    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
41    Priority, RouteTarget, PROTOCOL_VERSION,
42};
43use tokio::time::{timeout_at, Instant};
44use tracing::{debug, info, warn};
45
46use crate::{
47    caller_identity::CallerIdentity,
48    capability_requirements::{
49        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
50        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
51        RuntimeModule,
52    },
53    child_roster::DaemonShutdownFlag,
54    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
55    forwarding::{
56        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
57        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
58        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
59    },
60    observability::{
61        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
62    },
63    privileged_prompt::{prompt_text, PrivilegedOperation},
64    provenance::{
65        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
66    },
67    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
68    router::{RouteCtx, RouterError},
69    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
70    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
71    stderr_tail::{CaptureState, TailEntry},
72    supervise::{
73        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
74        SupervisorHandle, SwapHelloAdmission,
75    },
76    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
77};
78
79/// Lowest envelope version this subc build will negotiate.
80///
81/// Module HELLO negotiation is exact: peers must use the daemon's locked
82/// protocol version. Older and newer peers receive `version_unsupported` and
83/// are not registered.
84pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
85
86const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
87const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
88const CAP_PING_PONG: &str = "ping_pong_v1";
89const CAP_SESSION_ATTACH: &str = "session_attach_v1";
90const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
91
92const SUBC_CONTROL_OPS: &[&str] = &[
93    ops::ACTIVITY_RECORD,
94    ops::ACTIVITY_QUERY,
95    ops::ACTIVITY_SUBSCRIBE,
96    ops::SERVER_DESCRIBE,
97    ops::CATALOG_LIST,
98    ops::ROUTE_OPEN,
99    ops::ROUTE_POLL,
100    ops::ROUTE_CLOSING,
101    ops::ROUTE_CLOSED,
102    ops::SUPERVISOR_LIST,
103    ops::SUPERVISOR_RESTART,
104    ops::SUPERVISOR_SWAP,
105    ops::SUPERVISOR_RELOAD,
106    ops::SUPERVISOR_RESCAN,
107    ops::SUPERVISOR_RELEASE_RESERVED,
108    ops::SUPERVISOR_SET_ENABLED,
109    ops::SUPERVISOR_HEALTH_PROBE,
110    ops::SUPERVISOR_HEALTH,
111    ops::SUPERVISOR_STDERR_TAIL,
112    ops::SUPERVISOR_TERMINALS,
113    ops::SUPERVISOR_ROUTES,
114    ops::SUPERVISOR_PROVENANCE,
115    ops::SUPERVISOR_SPAWN_SNAPSHOT,
116    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
117];
118
119/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
120/// announced by a capability in `server.describe` instead, and clients check
121/// that capability before sending it. `ck daemon --json` prints `subc_ops`
122/// verbatim and scripts read that output, so a new name there would change it.
123/// A malformed body for one of these is still refused `invalid_control_body`
124/// rather than as an unknown op.
125const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
126
127mod rescan_checked;
128use rescan_checked::{RescanMode, RescanPlan};
129
130const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
131    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
132    "supervisor.live_roots",
133    SCOPE_SYNC_OP,
134    SCOPE_APPLY_OP,
135    SCOPE_DESCRIBE_OP,
136    "operator.confirm",
137];
138
139/// Module-originated ops the daemon answers but does not advertise in
140/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
141/// belongs to is incomplete, so no module is told it works before it does.
142const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
143
144const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
145
146/// How long subc waits for a module to ack a relayed route.bind before returning
147/// `module_timeout`. The ack waits on the module's own configure, which for AFT
148/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
149/// and DB-open work — on a cold page cache or a large repo that legitimately
150/// exceeds a couple of seconds. The default is generous because rejecting a VALID
151/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
152/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
153pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
154
155/// How many CONSECUTIVE full-budget relay timeouts against one target module
156/// open that module's bind-relay breaker.
157///
158/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
159/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
160/// so three consecutive full-budget timeouts take ~36s to observe: every client
161/// whose open contributed to opening the breaker had already given up on its
162/// own. That is what makes opening the breaker unable to turn a call that would
163/// have succeeded into a refusal — it can only make an already-failing module
164/// fail faster.
165///
166/// Two would be reachable inside one default deadline. One would convict a
167/// module on a single cold-cache bind, which is exactly the valid-but-slow case
168/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
169pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
170
171/// How long a module's bind-relay breaker stays open before exactly one
172/// `route.open` is let through as a probe.
173///
174/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
175/// re-pays a full-budget stall almost continuously, and the breaker stops being
176/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
177/// deadline: a client that starts retrying after the module recovers has to get
178/// a probe opportunity inside its own deadline, or the breaker converts a
179/// recovered module into a failed call — the failure it exists to prevent,
180/// pointed the other way.
181///
182/// 20s sits between those with room on both sides, and it caps what a wedged
183/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
184/// rather than one per `route.open` per connection. The stall that motivated
185/// this, with its measurements, is written up in
186/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
187/// waited the whole budget out.
188pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
189
190const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
191const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
192
193fn reload_verdict(
194    configured: &Path,
195    spawned_from: Option<&Path>,
196    image: subc_control::RunningImageAgreement,
197) -> PendingReloadVerdict {
198    let path = match spawned_from {
199        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
200            ReloadPathAgreement::Match
201        }
202        Some(spawned_from) => ReloadPathAgreement::Mismatch {
203            configured: configured.to_path_buf(),
204            spawned_from: spawned_from.to_path_buf(),
205        },
206        None => ReloadPathAgreement::Unavailable {
207            reason: if matches!(
208                image,
209                subc_control::RunningImageAgreement::Unavailable {
210                    reason: subc_control::RunningImageUnavailableReason::NotRunning
211                }
212            ) {
213                ReloadPathUnavailableReason::NotRunning
214            } else {
215                ReloadPathUnavailableReason::SpawnedPathUnavailable
216            },
217        },
218    };
219    PendingReloadVerdict {
220        path,
221        image,
222        spec: None,
223    }
224}
225
226fn launch_spec_agreement(
227    configured: &crate::supervise::ModuleSpec,
228    running: bool,
229    spawned: Option<&crate::supervise::LaunchSpecDigest>,
230) -> subc_control::LaunchSpecAgreement {
231    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
232    if !running {
233        return Agreement::Unavailable {
234            reason: Reason::NotRunning,
235        };
236    }
237    match spawned {
238        None => Agreement::Unavailable {
239            reason: Reason::SpawnedSpecUnavailable,
240        },
241        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
242            Agreement::Match
243        }
244        Some(_) => Agreement::Mismatch,
245    }
246}
247
248fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
249    if configured == spawned_from {
250        return true;
251    }
252    #[cfg(windows)]
253    {
254        // Windows spawn records the resolved verbatim path (\\?\...), while
255        // configuration keeps its original spelling. Resolve both with the OS
256        // instead of stripping prefixes or folding case, which could conflate
257        // distinct names in case-sensitive directories. Different resolved
258        // paths still differ even if their files have identical content.
259        match (
260            std::fs::canonicalize(configured),
261            std::fs::canonicalize(spawned_from),
262        ) {
263            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
264            _ => false,
265        }
266    }
267    #[cfg(not(windows))]
268    {
269        false
270    }
271}
272
273#[derive(Clone)]
274struct DaemonProvenanceFacts {
275    build: DaemonBuildProvenance,
276    pid: Option<u32>,
277    started_at_ms: Option<u64>,
278    start_clock: Option<crate::clock::StartClock>,
279    executable_path: Option<PathBuf>,
280    executable_identity: Option<SpawnedFileIdentity>,
281    process_start_time: Option<u64>,
282    probe: ExecutableIdentityProbe,
283}
284
285impl Default for DaemonProvenanceFacts {
286    fn default() -> Self {
287        Self {
288            build: DaemonBuildProvenance {
289                build_git_sha: None,
290                build_lock_digest: None,
291            },
292            pid: None,
293            started_at_ms: None,
294            start_clock: None,
295            executable_path: None,
296            executable_identity: None,
297            process_start_time: None,
298            probe: ExecutableIdentityProbe::default(),
299        }
300    }
301}
302
303#[derive(Debug, Clone)]
304struct SupervisorRescanContext {
305    supervisor: Supervisor,
306    config_path: PathBuf,
307    configured_port: Option<u16>,
308    storage_config: Option<crate::daemon_config::StorageConfig>,
309    admission_facts_carrier_module_id: Option<String>,
310    admission_facts_targets: Option<Vec<String>>,
311    scope_authority_owners: Vec<String>,
312    privileged_operator_prompt: PrivilegedOperatorPrompt,
313    /// Counts executed rescans that set out to change the supervised modules'
314    /// staged or applied configuration, counted before the first change so a
315    /// rescan that fails partway still counts. With the daemon incarnation it
316    /// names the state a `supervisor.rescan_checked` preview was computed
317    /// against.
318    generation: Arc<std::sync::atomic::AtomicU64>,
319}
320
321/// Refusal labels passed to `observe_route_open_refusal` that mean the target
322/// module is not serving right now, and so open or extend an outage in the
323/// route outage tracker. Every one of them is only reachable after the target
324/// was found in the registry, which is what keeps an arbitrary client-chosen
325/// id from ever creating tracker state.
326///
327/// Deliberately absent: `not_registered` and `removed` (the id may be
328/// anything a client sent, and a removed module is gone on purpose),
329/// `protocol_none` (such a module never serves routes, so nothing is out),
330/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
331/// capability and admission-facts refusals (they refuse the caller, not a
332/// module outage), and `relay_reservation_failed` (its code ranges over
333/// capacity limits as well as a vanished connection). Capacity, breaker,
334/// relay-timeout and module-rejection refusals do not pass through that
335/// function at all; the breaker logs its own transitions.
336///
337/// The two not-serving refusals that bypass that function record themselves
338/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
339/// `required_capability_unprovided` is not tracked: the module itself is up,
340/// and the outage belongs to the missing provider.
341const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
342    "reloading",
343    "supervisor_not_live",
344    "registration_not_active",
345    "no_forwarding_connection",
346    "relay_send_failed",
347];
348
349/// Real channel-0 control handler for subc itself.
350#[derive(Clone)]
351pub struct ControlHandler {
352    registry: Arc<Registry>,
353    activity: Option<crate::activity::ActivityLedger>,
354    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
355    forwarding: Arc<ForwardingTable>,
356    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
357    supervisor: SupervisorHandle,
358    subc_capabilities: Arc<[String]>,
359    /// Daemon-wide route.bind relay budget. Used as the fallback when the
360    /// target module has no per-module override in
361    /// `route_bind_relay_timeouts`.
362    route_bind_relay_timeout: Duration,
363    /// Per-module route.bind relay budget overrides, keyed by module id. When
364    /// `handle_route_open` resolves the deadline for a target module, a
365    /// per-module entry wins over the daemon-wide value above.
366    route_bind_relay_timeouts: BTreeMap<String, Duration>,
367    /// Per-target-module bind-relay breaker state. Shared with the forwarding
368    /// table, which is where a new module connection resets it.
369    route_bind_breakers: RouteBindBreakers,
370    /// Live relay admissions keyed by target module. Shared through the
371    /// forwarding table so cloned or separately built handlers enforce one cap.
372    route_bind_concurrency: RouteBindConcurrency,
373    /// Start and end of each module's not-serving period as seen by
374    /// `route.open`, so an outage gets one line at each edge instead of only
375    /// the per-refusal INFO lines. Taken from the forwarding table, so every
376    /// handler built over one table shares it.
377    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
378    /// Consecutive relay timeouts that open a module's breaker.
379    route_bind_breaker_threshold: u32,
380    /// How long a breaker stays open before one probe is admitted.
381    route_bind_breaker_cooldown: Duration,
382    health_probe_timeout: Duration,
383    /// Central storage policy. When set, each registering module receives its
384    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
385    storage_config: Option<crate::daemon_config::StorageConfig>,
386    /// The machine id established at boot, served on every HELLO_ACK and on
387    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
388    /// changes the file, never this value. `None` serves no id.
389    machine_id: Option<crate::machine_id::MachineId>,
390    admission_facts_carrier_module_id: Option<String>,
391    admission_facts_targets: Option<Vec<String>>,
392    /// Scope records with their sync authorities and tombstones; see
393    /// `crate::scopes`. Shared by clones of this handler, so every connection
394    /// reads and writes one table.
395    scopes: Arc<RwLock<ScopeTable>>,
396    /// Wakes the expiry loop when an accepted scope change may have added a
397    /// deadline, so the loop can wait without ticking while no live scope has
398    /// one. A permit is stored when nothing waits, so no wake is lost.
399    scope_deadline_added: Arc<tokio::sync::Notify>,
400    /// The configured `scope_authority_owners`, kept so a rescan can report a
401    /// changed value as needing a daemon restart; rescan never applies it.
402    scope_authority_owners: Vec<String>,
403    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
404    privileged_operator_prompt: PrivilegedOperatorPrompt,
405    /// The local and peer address of each accepted TCP client connection, read
406    /// from the daemon's own socket, so a privileged request can name its caller.
407    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
408    #[cfg(test)]
409    caller_lookup: Option<TestCallerLookup>,
410    /// The launch nonce each module connection presented at HELLO, which is how
411    /// a `scope.sync` is matched to the owner's current launch.
412    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
413    rescan: Option<SupervisorRescanContext>,
414    connected_clients: ConnectedClients,
415    counters: DaemonCounters,
416    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
417    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
418    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
419    /// their usual severity (see [`log_requirement_events`]). A handler built
420    /// without one has a flag that is never set.
421    daemon_shutdown: DaemonShutdownFlag,
422    daemon_provenance: DaemonProvenanceFacts,
423    #[cfg(test)]
424    control_dispatch_delay: Option<Duration>,
425    #[cfg(test)]
426    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
427    /// Holds `route.open` after the route.bind is queued to the module and
428    /// before it is marked enqueued, so a test can make the module's reply (or
429    /// a client teardown) land exactly inside that window.
430    #[cfg(test)]
431    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
432    /// Holds a HELLO after its registry entry is reserved and before its
433    /// forwarding endpoint is installed, so a test can read the catalog and
434    /// open routes while the module is registered but not yet routable. A
435    /// blocking barrier because the HELLO handler is synchronous; the test
436    /// runs the HELLO on its own thread.
437    #[cfg(test)]
438    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
439}
440
441impl fmt::Debug for ControlHandler {
442    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443        f.debug_struct("ControlHandler")
444            .field("registry", &self.registry)
445            .field("forwarding", &self.forwarding)
446            .field("process_liveness", &self.process_liveness.is_some())
447            .field("supervisor", &self.supervisor)
448            .field("subc_capabilities", &self.subc_capabilities)
449            .finish()
450    }
451}
452
453struct RouteOpenRequest {
454    target: RouteTarget,
455    identity: BindIdentity,
456    consumer_identity: Option<ConsumerIdentity>,
457    consumer_capabilities: Option<Vec<String>>,
458    role_versions: Option<BTreeMap<String, String>>,
459    admission_facts: Option<serde_json::Value>,
460    scope: Option<ScopeSelector>,
461}
462
463#[derive(Clone, Copy)]
464enum RouteOpenPhase {
465    Admission,
466    ProjectRoot,
467    EgressReservation,
468    BindSend,
469    BindAckWait,
470}
471
472/// Store elapsed time for each route-open phase, including attempts that end
473/// in an early refusal or backend error. Admission also includes rechecks and
474/// commit bookkeeping performed between the other timed phases.
475struct RouteOpenTimings {
476    module_id: String,
477    principal: String,
478    admission: Duration,
479    project_root: Duration,
480    egress_reservation: Duration,
481    bind_send: Duration,
482    bind_ack_wait: Duration,
483    phase: Option<RouteOpenPhase>,
484    phase_started_at: Instant,
485}
486
487impl RouteOpenTimings {
488    fn new() -> Self {
489        Self {
490            module_id: String::new(),
491            // Refusals before attestation must not claim a verified principal.
492            principal: "unresolved".to_string(),
493            admission: Duration::ZERO,
494            project_root: Duration::ZERO,
495            egress_reservation: Duration::ZERO,
496            bind_send: Duration::ZERO,
497            bind_ack_wait: Duration::ZERO,
498            phase: Some(RouteOpenPhase::Admission),
499            phase_started_at: Instant::now(),
500        }
501    }
502
503    fn finish(&mut self) {
504        let elapsed = self.phase_started_at.elapsed();
505        match self.phase.take() {
506            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
507            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
508            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
509            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
510            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
511            None => {}
512        }
513    }
514
515    fn enter(&mut self, phase: RouteOpenPhase) {
516        self.finish();
517        self.phase = Some(phase);
518        self.phase_started_at = Instant::now();
519    }
520}
521
522struct RouteBindReservationGuard {
523    forwarding: Arc<ForwardingTable>,
524    endpoint: ModuleEndpointId,
525    relay_corr: u64,
526    armed: bool,
527}
528
529struct ModuleControlRpcGuard {
530    forwarding: Arc<ForwardingTable>,
531    endpoint: ModuleEndpointId,
532    corr: u64,
533    armed: bool,
534}
535
536impl ModuleControlRpcGuard {
537    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
538        Self {
539            forwarding,
540            endpoint,
541            corr,
542            armed: true,
543        }
544    }
545
546    fn disarm(&mut self) {
547        self.armed = false;
548    }
549}
550
551impl Drop for ModuleControlRpcGuard {
552    fn drop(&mut self) {
553        if self.armed {
554            let _ = self
555                .forwarding
556                .cancel_module_control_rpc(self.endpoint, self.corr);
557        }
558    }
559}
560
561impl RouteBindReservationGuard {
562    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
563        Self {
564            forwarding,
565            endpoint,
566            relay_corr,
567            armed: true,
568        }
569    }
570
571    fn release_and_disarm(&mut self) {
572        if !self.armed {
573            return;
574        }
575        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
576            self.endpoint,
577            self.relay_corr,
578            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
579        ) {
580            send_goodbye_target_best_effort(
581                &self.forwarding.counters(),
582                &target,
583                "canceled route.bind",
584            );
585        }
586        self.armed = false;
587    }
588
589    fn disarm(&mut self) {
590        self.armed = false;
591    }
592}
593
594impl Drop for RouteBindReservationGuard {
595    fn drop(&mut self) {
596        self.release_and_disarm();
597    }
598}
599
600/// Per-target-module circuit breaker around the `route.bind` relay.
601///
602/// The connection reader is serial per connection, so a module whose `on_bind`
603/// sits on the ack blocks every LATER frame on the connections that call it,
604/// including calls to unrelated modules. This does not make any module's bind
605/// fast; it stops the daemon paying the full budget again and again for a
606/// condition it has already observed.
607///
608/// State is keyed by TARGET MODULE and shared by every connection: a wedged
609/// module wedges everyone, so what one connection learned should protect the
610/// rest.
611///
612/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
613/// relay to that module has actually timed out, and is removed again when a
614/// relay is accepted or the module reconnects, so it cannot grow with traffic
615/// or with modules that behave.
616///
617/// # Why a `std` mutex here is not the head-of-line defect again
618///
619/// Acquisition never awaits. The critical section is a hash lookup plus a few
620/// integer updates, with no I/O and no `.await` inside it, so a reader task
621/// cannot be descheduled behind it the way it can behind
622/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
623/// primitive, held for the same kind of work, as the refusal counter this very
624/// path already increments.
625///
626/// It is also NOT on the data-plane splice path: only `route.open` and module
627/// registration touch it, so bound-route frames gain no state check and no
628/// contention.
629#[derive(Debug, Clone, Default)]
630pub(crate) struct RouteBindBreakers {
631    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
632}
633
634#[derive(Debug, Clone, Default)]
635pub(crate) struct RouteBindConcurrency {
636    modules: Arc<Mutex<HashMap<String, usize>>>,
637}
638
639struct RouteBindConcurrencyGuard {
640    concurrency: RouteBindConcurrency,
641    module_id: String,
642}
643
644impl RouteBindConcurrency {
645    /// Admit without waiting. Waiting here would move the bind stall from the
646    /// module reply to a semaphore and restore reader head-of-line blocking.
647    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
648        let mut modules = self
649            .modules
650            .lock()
651            .expect("route.bind concurrency mutex poisoned");
652        let in_flight = modules.entry(module_id.to_string()).or_default();
653        if *in_flight >= limit {
654            return Err(*in_flight);
655        }
656        *in_flight += 1;
657        Ok(RouteBindConcurrencyGuard {
658            concurrency: self.clone(),
659            module_id: module_id.to_string(),
660        })
661    }
662}
663
664impl Drop for RouteBindConcurrencyGuard {
665    fn drop(&mut self) {
666        let mut modules = self
667            .concurrency
668            .modules
669            .lock()
670            .expect("route.bind concurrency mutex poisoned");
671        let remove = {
672            let in_flight = modules
673                .get_mut(&self.module_id)
674                .expect("admitted route.bind has a concurrency entry");
675            *in_flight -= 1;
676            *in_flight == 0
677        };
678        if remove {
679            modules.remove(&self.module_id);
680        }
681    }
682}
683
684#[derive(Debug, Default)]
685struct ModuleBreakerState {
686    /// Relay timeouts observed with no accepted relay in between.
687    consecutive_timeouts: u32,
688    /// `Some` while the breaker is open: the instant the cooldown expires and
689    /// the next arrival may probe. `None` means closed.
690    cooldown_until: Option<Instant>,
691    /// A half-open probe has been admitted and has not settled yet. This is
692    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
693    /// that read the cooldown, so concurrent opens arriving at the moment the
694    /// cooldown expires cannot all decide that they are the probe.
695    probe_in_flight: Option<Arc<()>>,
696}
697
698/// What the breaker decided for one `route.open`, before any relay work.
699enum RouteBindAdmission<'a> {
700    Admitted {
701        guard: RouteBindBreakerGuard<'a>,
702        /// This open is the single half-open probe, so the transition is worth
703        /// one log line.
704        probe: bool,
705    },
706    Refused {
707        consecutive_timeouts: u32,
708        /// What is left of the cooldown. Zero when the refusal is because the
709        /// one probe is already in flight rather than because the cooldown has
710        /// not elapsed.
711        retry_in: Duration,
712        probe_in_flight: bool,
713    },
714}
715
716/// An outstanding admission, which must be told how its relay settled.
717///
718/// `Drop` settles it as inconclusive, so an early return between admission and
719/// the relay -- or the whole handler being cancelled when the client
720/// disconnects -- releases a half-open probe slot instead of leaving the
721/// breaker wedged half-open with no further probes.
722struct RouteBindBreakerGuard<'a> {
723    breakers: RouteBindBreakers,
724    module_id: &'a str,
725    probe_token: Option<Arc<()>>,
726    settled: bool,
727}
728
729impl RouteBindBreakerGuard<'_> {
730    /// The module answered within the budget and took the bind. THE ONLY
731    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
732    /// breaker, which is a transition worth logging.
733    fn record_accepted(&mut self) -> bool {
734        self.settled = true;
735        self.breakers.record_accepted(self.module_id)
736    }
737
738    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
739    /// COUNTS TOWARD OPENING.
740    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
741        self.settled = true;
742        self.breakers.record_timeout(
743            self.module_id,
744            self.probe_token.as_ref(),
745            threshold,
746            cooldown,
747        )
748    }
749
750    /// Everything else: the module REJECTED the bind, its connection went away
751    /// mid-relay, or the waiter was cancelled.
752    ///
753    /// None of these is evidence that a module is slow, and each already has
754    /// its own refusal with its own code. A module that rejects a bind in
755    /// microseconds is healthy and must never be convicted for it; a module
756    /// that died has said nothing about the module that replaces it. So these
757    /// neither increment nor reset the count -- they only release a probe slot.
758    fn record_inconclusive(&mut self) {
759        self.settled = true;
760        self.breakers
761            .record_inconclusive(self.module_id, self.probe_token.as_ref());
762    }
763}
764
765impl Drop for RouteBindBreakerGuard<'_> {
766    fn drop(&mut self) {
767        if !self.settled {
768            self.breakers
769                .record_inconclusive(self.module_id, self.probe_token.as_ref());
770        }
771    }
772}
773
774/// The breaker moved to open, reported so the caller can log it outside the
775/// lock. Opening is rare and load-bearing; the refusals that follow are
776/// frequent and are counted rather than logged.
777struct BreakerOpened {
778    consecutive_timeouts: u32,
779    /// True when a failed probe re-opened an already-open breaker, which reads
780    /// very differently in a log from a first opening.
781    reopened_after_probe: bool,
782}
783
784impl RouteBindBreakers {
785    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
786        self.modules
787            .lock()
788            .expect("route.bind breaker mutex poisoned")
789    }
790
791    /// Decide whether this `route.open` may attempt its relay. Takes the map
792    /// lock and nothing else, and never awaits.
793    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
794        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
795            probe: probe_token.is_some(),
796            guard: RouteBindBreakerGuard {
797                breakers: self.clone(),
798                module_id,
799                probe_token,
800                settled: false,
801            },
802        };
803
804        let mut modules = self.lock();
805        let Some(state) = modules.get_mut(module_id) else {
806            return admitted(None);
807        };
808        let Some(cooldown_until) = state.cooldown_until else {
809            return admitted(None);
810        };
811        if state.probe_in_flight.is_some() {
812            return RouteBindAdmission::Refused {
813                consecutive_timeouts: state.consecutive_timeouts,
814                retry_in: Duration::ZERO,
815                probe_in_flight: true,
816            };
817        }
818        let now = Instant::now();
819        if now < cooldown_until {
820            return RouteBindAdmission::Refused {
821                consecutive_timeouts: state.consecutive_timeouts,
822                retry_in: cooldown_until - now,
823                probe_in_flight: false,
824            };
825        }
826        let token = Arc::new(());
827        state.probe_in_flight = Some(Arc::clone(&token));
828        admitted(Some(token))
829    }
830
831    fn record_accepted(&self, module_id: &str) -> bool {
832        self.lock()
833            .remove(module_id)
834            .is_some_and(|state| state.cooldown_until.is_some())
835    }
836
837    fn record_timeout(
838        &self,
839        module_id: &str,
840        probe_token: Option<&Arc<()>>,
841        threshold: u32,
842        cooldown: Duration,
843    ) -> Option<BreakerOpened> {
844        let mut modules = self.lock();
845        let state = modules.entry(module_id.to_string()).or_default();
846        let was_open = state.cooldown_until.is_some();
847        let was_probe = Self::owns_probe(state, probe_token);
848        if was_probe {
849            state.probe_in_flight = None;
850        }
851        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
852        if state.consecutive_timeouts < threshold {
853            return None;
854        }
855        state.cooldown_until = Some(Instant::now() + cooldown);
856        Some(BreakerOpened {
857            consecutive_timeouts: state.consecutive_timeouts,
858            reopened_after_probe: was_open && was_probe,
859        })
860    }
861
862    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
863        // A relay can finish after the breaker was reset or after it opened
864        // again and started a new probe. Only the guard whose token matches the
865        // active probe may release it, so a late relay never frees a newer probe.
866        state
867            .probe_in_flight
868            .as_ref()
869            .zip(token)
870            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
871    }
872
873    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
874        if let Some(state) = self.lock().get_mut(module_id) {
875            if Self::owns_probe(state, probe_token) {
876                state.probe_in_flight = None;
877            }
878        }
879    }
880
881    /// Discard what was learned about a module, because the process it was
882    /// learned about is gone. Returns the discarded count when it was non-zero.
883    ///
884    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
885    /// `module_id` is a configuration identity that outlives any particular
886    /// child; what the breaker observed was the process behind the module
887    /// connection of the moment. When a new connection registers under that id
888    /// the verdict's subject no longer exists, so the verdict is stale by
889    /// construction rather than merely likely to be wrong. Keeping it would
890    /// apply a dead process's record to a live one, which is the same defect
891    /// class this breaker exists to stop the daemon committing.
892    ///
893    /// A half-open probe in flight is discarded with the rest: it was a
894    /// question about the old process.
895    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
896        self.lock()
897            .remove(module_id)
898            .map(|state| state.consecutive_timeouts)
899            .filter(|discarded| *discarded > 0)
900    }
901
902    /// Open breakers, for the `server.describe` counters object. `None` when
903    /// none is open, so the key stays absent rather than present-and-empty.
904    ///
905    /// This is the operator's answer to "is this module refusing instantly or
906    /// is it fine?", which look identical from a client that retries and then
907    /// succeeds.
908    fn open_snapshot(&self) -> Option<serde_json::Value> {
909        let now = Instant::now();
910        let modules = self.lock();
911        let open = modules
912            .iter()
913            .filter_map(|(module_id, state)| {
914                let cooldown_until = state.cooldown_until?;
915                Some((
916                    module_id.clone(),
917                    serde_json::json!({
918                        "consecutive_timeouts": state.consecutive_timeouts,
919                        "cooldown_remaining_ms":
920                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
921                        "probe_in_flight": state.probe_in_flight.is_some(),
922                    }),
923                ))
924            })
925            .collect::<serde_json::Map<String, serde_json::Value>>();
926        (!open.is_empty()).then_some(serde_json::Value::Object(open))
927    }
928}
929
930impl ControlHandler {
931    pub fn new(registry: Arc<Registry>) -> Self {
932        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
933    }
934
935    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
936        let counters = forwarding.counters();
937        // Taken from the forwarding table rather than created here, so that the
938        // breaker a `route.open` consults is the same one a module's
939        // registration resets, however many handlers are built over one table.
940        let route_bind_breakers = forwarding.route_bind_breakers();
941        let route_bind_concurrency = forwarding.route_bind_concurrency();
942        let route_outages = forwarding.route_outages();
943        Self {
944            registry,
945            activity: None,
946            activity_streams: Arc::new(Mutex::new(HashMap::new())),
947            forwarding,
948            process_liveness: None,
949            supervisor: SupervisorHandle::new(),
950            subc_capabilities: Arc::from([
951                CAP_MANIFEST_REGISTRATION.to_string(),
952                CAP_CHANNEL_LIFECYCLE.to_string(),
953                CAP_PING_PONG.to_string(),
954                CAP_SESSION_ATTACH.to_string(),
955                CAP_ADMISSION_FACTS_RELAY.to_string(),
956                CAP_SCOPES_V1.to_string(),
957                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
958                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
959                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
960                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
961                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
962                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
963                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
964                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
965                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
966            ]),
967            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
968            route_bind_relay_timeouts: BTreeMap::new(),
969            route_bind_breakers,
970            route_bind_concurrency,
971            route_outages,
972            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
973            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
974            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
975            storage_config: None,
976            machine_id: None,
977            admission_facts_carrier_module_id: None,
978            admission_facts_targets: None,
979            scopes: Arc::new(RwLock::new(ScopeTable::new(
980                crate::daemon_config::default_scope_authority_owners(),
981            ))),
982            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
983            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
984            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
985            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
986            #[cfg(test)]
987            caller_lookup: None,
988            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
989            rescan: None,
990            connected_clients: ConnectedClients::new(),
991            counters,
992            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
993            daemon_shutdown: DaemonShutdownFlag::default(),
994            daemon_provenance: DaemonProvenanceFacts::default(),
995            #[cfg(test)]
996            control_dispatch_delay: None,
997            #[cfg(test)]
998            provenance_probe_override: None,
999            #[cfg(test)]
1000            route_bind_relay_sent_hold: None,
1001            #[cfg(test)]
1002            hello_registry_to_forwarding_hold: None,
1003        }
1004    }
1005
1006    /// Set the central storage policy: registering modules then receive their
1007    /// resolved storage descriptor in HELLO_ACK.
1008    pub fn with_storage_config(
1009        mut self,
1010        storage_config: Option<crate::daemon_config::StorageConfig>,
1011    ) -> Self {
1012        self.storage_config = storage_config;
1013        self
1014    }
1015
1016    /// Open the ledger on its own writer thread; an unavailable sink does not
1017    /// prevent the daemon from serving routing or supervisor requests.
1018    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1019        self.activity = Some(crate::activity::ActivityLedger::open(
1020            run_dir.join("activity.jsonl"),
1021            incarnation,
1022        ));
1023        self.supervisor.set_activity(self.activity.clone());
1024        self
1025    }
1026
1027    /// Set the machine id served to every registering module (HELLO_ACK) and on
1028    /// `server.describe`.
1029    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1030        self.machine_id = machine_id;
1031        self
1032    }
1033
1034    /// Configure the exact reserved module and target ids permitted to relay
1035    /// opaque admission facts. Config-file loading validates this authority;
1036    /// this builder keeps the same policy available to embedded test daemons.
1037    pub fn with_admission_facts_config(
1038        mut self,
1039        carrier_module_id: Option<String>,
1040        targets: Option<Vec<String>>,
1041    ) -> Self {
1042        self.admission_facts_carrier_module_id = carrier_module_id;
1043        self.admission_facts_targets = targets;
1044        self
1045    }
1046
1047    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1048    /// Replaces the scope table with an empty one under the new list, so call it
1049    /// while building the handler, before any module can sync.
1050    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1051        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1052        self.scope_authority_owners = owners;
1053        self
1054    }
1055
1056    /// Set the startup policy before installing the rescan context or serving requests.
1057    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1058        self.privileged_operator_prompt = mode;
1059        self
1060    }
1061
1062    /// Override the route.bind relay timeout. Used by tests that assert the
1063    /// timeout path so they don't block on the production-safe default.
1064    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1065        self.route_bind_relay_timeout = timeout;
1066        self
1067    }
1068
1069    /// Install per-module route.bind relay budget overrides. A module id
1070    /// listed here wins over the daemon-wide default set via
1071    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1072    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1073    /// the same `Duration` the bind path will use.
1074    pub fn with_route_bind_relay_timeouts(
1075        mut self,
1076        timeouts: impl IntoIterator<Item = (String, Duration)>,
1077    ) -> Self {
1078        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1079        self
1080    }
1081
1082    /// Resolve the route.bind relay budget for a specific target module id.
1083    /// Per-module overrides win; the daemon-wide value (set via
1084    /// `with_route_bind_relay_timeout` or the built-in default) is the
1085    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1086    /// the same resolution `handle_route_open` will use.
1087    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1088        self.route_bind_relay_timeouts
1089            .get(module_id)
1090            .copied()
1091            .unwrap_or(self.route_bind_relay_timeout)
1092    }
1093
1094    /// Override the per-module bind-relay breaker policy.
1095    ///
1096    /// Used by tests, which cannot spend three production budgets opening a
1097    /// breaker or twenty seconds waiting for its cooldown. The production
1098    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1099    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1100    /// reasoning for the numbers.
1101    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1102        self.route_bind_breaker_threshold = threshold.max(1);
1103        self.route_bind_breaker_cooldown = cooldown;
1104        self
1105    }
1106
1107    #[cfg(test)]
1108    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1109        self.health_probe_timeout = timeout;
1110        self
1111    }
1112
1113    #[cfg(test)]
1114    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1115        self.control_dispatch_delay = Some(delay);
1116        self
1117    }
1118
1119    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1120    /// announce it is parked, once to be released. Build the barrier for two
1121    /// parties, the test being the other one.
1122    #[cfg(test)]
1123    pub(crate) fn with_route_bind_relay_sent_hold(
1124        mut self,
1125        hold: Arc<tokio::sync::Barrier>,
1126    ) -> Self {
1127        self.route_bind_relay_sent_hold = Some(hold);
1128        self
1129    }
1130
1131    /// A HELLO waits on `hold` twice between reserving its registry entry and
1132    /// installing its forwarding endpoint: once to announce it is parked, once
1133    /// to be released. Build the barrier for two parties, the test being the
1134    /// other one, and run the HELLO on a thread of its own.
1135    #[cfg(test)]
1136    pub(crate) fn with_hello_registry_to_forwarding_hold(
1137        mut self,
1138        hold: Arc<std::sync::Barrier>,
1139    ) -> Self {
1140        self.hello_registry_to_forwarding_hold = Some(hold);
1141        self
1142    }
1143
1144    #[cfg(test)]
1145    fn hold_between_registry_and_forwarding(&self) {
1146        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1147            hold.wait();
1148            hold.wait();
1149        }
1150    }
1151
1152    pub fn with_process_liveness(
1153        mut self,
1154        process_liveness: Arc<dyn ModuleProcessLiveness>,
1155    ) -> Self {
1156        self.process_liveness = Some(process_liveness);
1157        self
1158    }
1159
1160    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1161        self.supervisor = supervisor;
1162        self.supervisor.set_activity(self.activity.clone());
1163        self
1164    }
1165
1166    pub fn with_daemon_provenance(
1167        mut self,
1168        pid: u32,
1169        started_at_ms: u64,
1170        executable_path: Option<PathBuf>,
1171        build_git_sha: Option<String>,
1172        build_lock_digest: Option<String>,
1173    ) -> Self {
1174        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1175        let process_start_time = process_start_time(pid);
1176        self.daemon_provenance = DaemonProvenanceFacts {
1177            build: DaemonBuildProvenance {
1178                build_git_sha,
1179                build_lock_digest,
1180            },
1181            pid: Some(pid),
1182            started_at_ms: Some(started_at_ms),
1183            start_clock: None,
1184            executable_path,
1185            executable_identity,
1186            process_start_time,
1187            probe: ExecutableIdentityProbe::default(),
1188        };
1189        self
1190    }
1191
1192    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1193        self.daemon_provenance.start_clock = Some(clock);
1194        self
1195    }
1196
1197    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1198    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1199        self.scopes
1200            .write()
1201            .unwrap_or_else(|poisoned| poisoned.into_inner())
1202            .set_wall_clock(Arc::new(clock));
1203        self
1204    }
1205
1206    #[cfg(test)]
1207    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1208        self.provenance_probe_override = Some(result);
1209        self
1210    }
1211
1212    /// Install the configured module set and its reserved capability bindings.
1213    /// Bindings are configuration-scoped and may point at a provider that has not
1214    /// been installed yet, so this does not require the bound module to exist.
1215    pub fn with_capability_config(
1216        self,
1217        modules: impl IntoIterator<Item = (String, bool)>,
1218        reserved_capabilities: BTreeMap<String, String>,
1219    ) -> Self {
1220        self.capability_evaluator
1221            .configure(modules, reserved_capabilities);
1222        self
1223    }
1224
1225    /// Share the supervisor's daemon-shutdown flag with this handler, so
1226    /// requirement changes after the daemon has begun its announced shutdown
1227    /// are not reported as faults.
1228    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1229        self.daemon_shutdown = flag;
1230        self
1231    }
1232
1233    pub fn with_supervisor_rescan(
1234        mut self,
1235        supervisor: Supervisor,
1236        config_path: impl Into<PathBuf>,
1237        configured_port: Option<u16>,
1238    ) -> Self {
1239        self.rescan = Some(SupervisorRescanContext {
1240            supervisor,
1241            config_path: config_path.into(),
1242            configured_port,
1243            storage_config: self.storage_config.clone(),
1244            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1245            admission_facts_targets: self.admission_facts_targets.clone(),
1246            scope_authority_owners: self.scope_authority_owners.clone(),
1247            privileged_operator_prompt: self.privileged_operator_prompt,
1248            generation: Arc::default(),
1249        });
1250        self
1251    }
1252
1253    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1254        self.connected_clients = connected_clients;
1255        self
1256    }
1257
1258    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1259        Arc::clone(&self.forwarding)
1260    }
1261
1262    pub(crate) fn counters(&self) -> DaemonCounters {
1263        self.counters.clone()
1264    }
1265
1266    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1267    /// requirement event without depending on an operator polling a status command.
1268    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1269        tokio::spawn(async move {
1270            loop {
1271                self.capability_evaluator
1272                    .wait_for_change_or_deadline()
1273                    .await;
1274                self.refresh_capability_requirements();
1275            }
1276        });
1277    }
1278
1279    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1280    /// deadline passed during sleep is enforced on the first tick after wake.
1281    /// While no live scope carries a deadline the loop waits for an accepted
1282    /// scope change instead of ticking, so a daemon without deadlines never
1283    /// wakes for expiry.
1284    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1285        tokio::spawn(async move {
1286            loop {
1287                let has_deadlines = self
1288                    .scopes
1289                    .read()
1290                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1291                    .has_deadlines();
1292                if !has_deadlines {
1293                    self.scope_deadline_added.notified().await;
1294                    continue;
1295                }
1296                tokio::time::sleep(Duration::from_secs(1)).await;
1297                if let Err(error) = self.sweep_expired_scopes() {
1298                    warn!(%error, "scope expiry sweep failed");
1299                }
1300            }
1301        });
1302    }
1303
1304    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1305        let mut table = self
1306            .scopes
1307            .write()
1308            .unwrap_or_else(|poisoned| poisoned.into_inner());
1309        let swept = table.sweep_expired();
1310        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1311        drop(table);
1312        self.close_scope_drained_routes(drained);
1313        Ok(())
1314    }
1315
1316    fn publish_scope_changes(
1317        &self,
1318        changes: &[crate::scopes::ScopeTagChange],
1319        expired: &[crate::scopes::ScopeExpired],
1320    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1321        if changes.is_empty() {
1322            return Ok(Vec::new());
1323        }
1324        let drained = self
1325            .forwarding
1326            .publish_scope_changes(changes)
1327            .map_err(RouterError::Forwarding)?;
1328        let mut counts = HashMap::new();
1329        for route in &drained {
1330            *counts
1331                .entry((
1332                    route.scope.owner.as_str(),
1333                    route.scope.scope_ref.as_str(),
1334                    route.scope.tag.scope_epoch,
1335                ))
1336                .or_insert(0usize) += 1;
1337        }
1338        for scope in expired {
1339            let routes_closed = counts
1340                .get(&(
1341                    scope.owner.as_str(),
1342                    scope.scope_ref.as_str(),
1343                    scope.scope_epoch,
1344                ))
1345                .copied()
1346                .unwrap_or(0);
1347            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1348                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1349                routes_closed, "scope expired");
1350        }
1351        Ok(drained)
1352    }
1353
1354    fn runtime_capability_snapshot(
1355        &self,
1356    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1357        let runtime = self
1358            .supervisor
1359            .list()
1360            .into_iter()
1361            .map(|module| {
1362                let status = module.status().map_err(|err| {
1363                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1364                })?;
1365                Ok(RuntimeModule {
1366                    module_id: status.module_id,
1367                    state: status.state,
1368                    enabled: status.enabled,
1369                })
1370            })
1371            .collect::<Result<Vec<_>, RouterError>>()?;
1372        // Include every registration, published or not. The capability
1373        // evaluator counts a module as providing its capabilities as soon as it
1374        // has registered, even before `catalog.list` shows it, so a HELLO in
1375        // progress does not change other modules' required-capability
1376        // verdicts. See `Registry::list_registered_modules`.
1377        let registrations = self.registry.list_registered_modules().map_err(|err| {
1378            RouterError::backend(
1379                0,
1380                0,
1381                format!("failed to list capability registrations: {err}"),
1382            )
1383        })?;
1384        let registrations = registrations
1385            .into_iter()
1386            .map(|registration| RegisteredModule {
1387                module_id: registration.manifest.module_id,
1388                module_version: registration.manifest.module_version,
1389                capabilities: registration.manifest.capabilities,
1390            })
1391            .collect();
1392        Ok((runtime, registrations))
1393    }
1394
1395    /// The capability side effects of a module becoming the active registration
1396    /// for its id: cache its manifest (warning if its claims drifted), run the
1397    /// deny census when its declarations call for one, and recompute the
1398    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1399    /// candidate's does not, and the supervisor does it at promotion instead,
1400    /// through [`crate::supervise::SwapPromotionObserver`].
1401    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1402        let cached_registration = RegisteredModule {
1403            module_id: registration.manifest.module_id.clone(),
1404            module_version: registration.manifest.module_version.clone(),
1405            capabilities: registration.manifest.capabilities.clone(),
1406        };
1407        if self.capability_evaluator.record_hello(&cached_registration) {
1408            warn!(
1409                module_id = %cached_registration.module_id,
1410                "capability claims drifted from the cached manifest"
1411            );
1412        }
1413        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1414            self.enforce_capability_denies();
1415        }
1416        self.refresh_capability_requirements();
1417    }
1418
1419    /// Point the shared supervisor handle at this handler for swap promotions.
1420    /// Called wherever a handler is put behind the `Arc` the router serves, so
1421    /// it can be held weakly.
1422    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1423        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1424            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1425        self.supervisor.set_swap_promotion_observer(observer);
1426    }
1427
1428    pub fn refresh_capability_requirements(&self) {
1429        match self.runtime_capability_snapshot() {
1430            Ok((runtime, registrations)) => {
1431                log_requirement_events(
1432                    self.capability_evaluator
1433                        .evaluate_now(&runtime, &registrations),
1434                    &self.daemon_shutdown,
1435                );
1436            }
1437            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1438        }
1439    }
1440
1441    /// Reconcile only live, attested route bindings after a capability deny edge
1442    /// or target claim was added. This is deliberately a control-plane census:
1443    /// the opaque forwarding hot path must not grow a per-frame capability check.
1444    fn enforce_capability_denies(&self) {
1445        let registrations = match self.registry.list_registered_modules() {
1446            Ok(snapshot) => snapshot,
1447            Err(err) => {
1448                warn!(error = %err, "failed to read registrations for capability deny census");
1449                return;
1450            }
1451        };
1452        let manifests = registrations
1453            .into_iter()
1454            .map(|registration| {
1455                (
1456                    registration.manifest.module_id.clone(),
1457                    registration.manifest,
1458                )
1459            })
1460            .collect::<BTreeMap<_, _>>();
1461        let census = match self.forwarding.route_census(None) {
1462            Ok(census) => census,
1463            Err(err) => {
1464                warn!(error = %err, "failed to read route census for capability deny enforcement");
1465                return;
1466            }
1467        };
1468
1469        for (target_module_id, routes) in census {
1470            let Some(target_manifest) = manifests.get(&target_module_id) else {
1471                continue;
1472            };
1473            let mut closed_routes = Vec::new();
1474            let mut module_goodbyes = Vec::new();
1475            for route in routes {
1476                let Principal::Reserved {
1477                    module_id: opening_module_id,
1478                } = &route.principal
1479                else {
1480                    continue;
1481                };
1482                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1483                    continue;
1484                };
1485                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1486                    continue;
1487                };
1488
1489                match self.forwarding.release_client_route(
1490                    route.goodbye_target.connection_id,
1491                    route.goodbye_target.channel,
1492                    route.goodbye_target.epoch,
1493                ) {
1494                    Ok(RouteRelease::Removed(module_goodbye)) => {
1495                        warn!(
1496                            opening_module_id,
1497                            target_module_id,
1498                            capability,
1499                            "force-closing route because an attested capability deny edge now matches"
1500                        );
1501                        closed_routes.push(route);
1502                        module_goodbyes.push(module_goodbye);
1503                    }
1504                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1505                    Err(err) => warn!(
1506                        opening_module_id,
1507                        target_module_id,
1508                        capability,
1509                        error = %err,
1510                        "failed to force-close capability-denied route"
1511                    ),
1512                }
1513            }
1514
1515            if closed_routes.is_empty() {
1516                continue;
1517            }
1518            send_route_control_pushes(
1519                &self.forwarding,
1520                closed_routes,
1521                ClientControlPush::RouteClosed {
1522                    module_id: target_module_id,
1523                    channels: Vec::new(),
1524                    reason: RouteCloseReason::CapabilityDenied,
1525                    drained: false,
1526                    abandoned: 0,
1527                    excluded_subscriptions: 0,
1528                    terminal: Some(false),
1529                },
1530            );
1531            self.emit_route_goodbyes(module_goodbyes);
1532        }
1533    }
1534
1535    /// Why a registered module is not accepting new route binds, or `None` when
1536    /// it is. This is the module's effective readiness: its declared readiness
1537    /// first, then every `need: required` capability it declares evaluating to
1538    /// `provided`. `route.open` and `catalog.list` both read it here so the
1539    /// catalog never reports a module routable that `route.open` would refuse.
1540    fn not_ready_reason(
1541        &self,
1542        registration: &crate::registry::ModuleRegistration,
1543    ) -> Option<NotReadyReason> {
1544        if !registration.ready {
1545            return Some(NotReadyReason {
1546                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1547                capability: None,
1548            });
1549        }
1550        self.first_unprovided_required_capability(registration)
1551            .map(|capability| NotReadyReason {
1552                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1553                capability: Some(capability),
1554            })
1555    }
1556
1557    /// The lexicographically first capability this registration declares
1558    /// `need: required` whose evaluator verdict is not `provided`.
1559    ///
1560    /// The verdicts are the capability evaluator's own; nothing here decides
1561    /// what "provided" means. The evaluator counts a capability provided as
1562    /// soon as a module claiming it has REGISTERED, not once that module is
1563    /// ready. That distinction is what keeps two modules that require each
1564    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1565    /// is ready", each would wait for the other to become ready first and
1566    /// neither ever would. Do not tighten it to readiness.
1567    ///
1568    /// A required capability with no verdict at all means this registration's
1569    /// HELLO or catalog.update landed after the last recompute; recompute once
1570    /// rather than let a missing verdict read as either answer. If it is still
1571    /// missing (the recompute itself failed) the capability counts as
1572    /// unprovided: the refusal is retryable, and routing a module whose
1573    /// required provider is unknown is the outcome this check exists to stop.
1574    fn first_unprovided_required_capability(
1575        &self,
1576        registration: &crate::registry::ModuleRegistration,
1577    ) -> Option<String> {
1578        let required = registration
1579            .manifest
1580            .capabilities
1581            .iter()
1582            .flat_map(|declarations| declarations.requires.iter())
1583            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1584            .map(|requirement| requirement.capability.as_str())
1585            .collect::<BTreeSet<_>>();
1586        if required.is_empty() {
1587            return None;
1588        }
1589        let module_id = registration.manifest.module_id.as_str();
1590        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1591        if required
1592            .iter()
1593            .any(|capability| verdict(capability).is_none())
1594        {
1595            self.refresh_capability_requirements();
1596        }
1597        required
1598            .into_iter()
1599            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1600            .map(str::to_string)
1601    }
1602
1603    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1604        self.capability_evaluator
1605            .statuses()
1606            .into_iter()
1607            .map(capability_requirement_status)
1608            .collect()
1609    }
1610
1611    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1612    /// registration-release watch. The signal is what the supervisor waits on
1613    /// before spawning a replacement, so it must only fire once forwarding
1614    /// teardown is also done (see [`Self::cleanup_connection`] /
1615    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1616    /// admitted but its forwarding endpoint could not be installed.
1617    fn deregister_connection(
1618        &self,
1619        connection_id: ConnectionId,
1620        reason: RegistrationEndReason,
1621    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1622        self.registry
1623            .deregister_connection_with_reason(connection_id, reason)
1624    }
1625
1626    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1627        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1628            return None;
1629        }
1630        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1631            parse_client_control_request(&frame.body)
1632        else {
1633            return None;
1634        };
1635        Some(target_module_id(&target).to_string())
1636    }
1637
1638    pub(crate) fn route_open_capacity_refusal(
1639        &self,
1640        ctx: &RouteCtx,
1641        frame: &Frame,
1642        target_module_id: &str,
1643        in_flight: usize,
1644        limit: usize,
1645    ) -> Result<Frame, RouterError> {
1646        self.route_open_admission_refusal_frame(
1647            ctx,
1648            frame,
1649            target_module_id,
1650            "open_admission_full",
1651            (in_flight, limit),
1652            format!(
1653                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1654            ),
1655        )
1656    }
1657
1658    fn route_open_target_capacity_refusal(
1659        &self,
1660        ctx: &RouteCtx,
1661        frame: &Frame,
1662        target_module_id: &str,
1663        in_flight: usize,
1664    ) -> Result<Frame, RouterError> {
1665        self.route_open_admission_refusal_frame(
1666            ctx,
1667            frame,
1668            target_module_id,
1669            "target_binds_full",
1670            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1671            format!(
1672                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1673            ),
1674        )
1675    }
1676
1677    /// Admission pressure clears as existing binds settle, so its refusal must
1678    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1679    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1680    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1681    /// cannot currently reach its target; `module_timeout` would falsely claim
1682    /// that a wait expired. A new, cleaner code would be terminal to deployed
1683    /// clients, so it requires a client-tolerance rollout before daemon emission.
1684    fn route_open_admission_refusal_frame(
1685        &self,
1686        ctx: &RouteCtx,
1687        frame: &Frame,
1688        target_module_id: &str,
1689        reason: &'static str,
1690        (in_flight, limit): (usize, usize),
1691        message: impl Into<String>,
1692    ) -> Result<Frame, RouterError> {
1693        let code = error_codes::TARGET_UNAVAILABLE;
1694        self.counters.increment_route_open_refused(code);
1695        info!(
1696            target: "control",
1697            code,
1698            reason,
1699            module_id = ?target_module_id,
1700            connection_id = ctx.connection_id.get(),
1701            in_flight,
1702            limit,
1703            "route.open refused"
1704        );
1705        control_error_frame(frame, code, message.into())
1706    }
1707
1708    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1709    ///
1710    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1711    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1712    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1713    #[cfg(test)]
1714    pub fn handle_control(
1715        &self,
1716        connection_id: ConnectionId,
1717        frame: Frame,
1718    ) -> Result<Vec<Frame>, RouterError> {
1719        match frame.header.ty {
1720            FrameType::Ping => Ok(vec![pong(&frame)?]),
1721            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1722            FrameType::Goodbye => self.handle_goodbye(connection_id),
1723            ty => Ok(vec![control_error_frame(
1724                &frame,
1725                "unsupported_control_frame",
1726                format!("unsupported channel-0 frame {ty:?}"),
1727            )?]),
1728        }
1729    }
1730
1731    pub async fn handle_control_frame(
1732        &self,
1733        ctx: &RouteCtx,
1734        frame: Frame,
1735    ) -> Result<Vec<Frame>, RouterError> {
1736        self.handle_control_frame_timed(ctx, frame, None).await
1737    }
1738
1739    pub(crate) fn record_caller_endpoints(
1740        &self,
1741        connection: ConnectionId,
1742        endpoints: Option<(SocketAddr, SocketAddr)>,
1743    ) {
1744        let mut callers = self
1745            .caller_endpoints
1746            .lock()
1747            .unwrap_or_else(|p| p.into_inner());
1748        callers.remove(&connection);
1749        if let Some(endpoints) = endpoints {
1750            callers.insert(connection, endpoints);
1751        }
1752    }
1753
1754    #[cfg(test)]
1755    pub(crate) fn with_caller_lookup_for_test(
1756        mut self,
1757        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1758    ) -> Self {
1759        self.caller_lookup = Some(Arc::new(lookup));
1760        self
1761    }
1762
1763    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1764        let endpoints = self
1765            .caller_endpoints
1766            .lock()
1767            .unwrap_or_else(|p| p.into_inner())
1768            .get(&connection)
1769            .copied();
1770        #[cfg(test)]
1771        let lookup = self.caller_lookup.clone();
1772        tokio::task::spawn_blocking(move || {
1773            #[cfg(test)]
1774            if let Some(lookup) = lookup {
1775                return lookup(endpoints);
1776            }
1777            endpoints
1778                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1779                .unwrap_or_default()
1780        })
1781        .await
1782        .unwrap_or_default()
1783    }
1784
1785    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1786        frame.header.channel == 0
1787            && frame.header.ty == FrameType::Request
1788            && self
1789                .forwarding
1790                .module_endpoint_for_connection(ctx.connection_id)
1791                .is_ok_and(|endpoint| endpoint.is_none())
1792            && parse_client_control_request(&frame.body)
1793                .ok()
1794                .is_some_and(|request| privileged_operation(&request).is_some())
1795    }
1796
1797    pub(crate) async fn handle_control_frame_timed(
1798        &self,
1799        ctx: &RouteCtx,
1800        frame: Frame,
1801        dispatch_started_at: Option<StdInstant>,
1802    ) -> Result<Vec<Frame>, RouterError> {
1803        match frame.header.ty {
1804            FrameType::Ping => Ok(vec![pong(&frame)?]),
1805            FrameType::Hello => {
1806                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1807            }
1808            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1809            FrameType::Cancel => {
1810                // A Cancel on channel 0 names a waiting operator.confirm or a
1811                // subscription. Successful cancellation has no ordinary reply.
1812                if self
1813                    .forwarding
1814                    .operator_confirms()
1815                    .cancel(ctx.connection_id, frame.header.corr)
1816                    || self
1817                        .supervisor
1818                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1819                    || self.cancel_activity_subscription(ctx, &frame)?
1820                {
1821                    Ok(Vec::new())
1822                } else {
1823                    Ok(vec![control_error_frame(
1824                        &frame,
1825                        "unknown_subscription",
1826                        "no subscription has this correlation id",
1827                    )?])
1828                }
1829            }
1830            FrameType::Request => {
1831                // This additive operation is not part of the existing exhaustive
1832                // module-control enum. Probe the op before decoding that enum.
1833                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1834                if op
1835                    .as_ref()
1836                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1837                {
1838                    return self.handle_activity_record(ctx, frame).await;
1839                }
1840                if op.as_ref().is_some_and(|probe| {
1841                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1842                }) {
1843                    return self.handle_activity_read(ctx, frame).await;
1844                }
1845                if op
1846                    .as_ref()
1847                    .is_some_and(|probe| probe.op == "operator.confirm")
1848                {
1849                    return self.handle_operator_confirm(ctx, frame);
1850                }
1851                if op
1852                    .as_ref()
1853                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1854                {
1855                    return self.handle_scope_served_by(ctx.connection_id, frame);
1856                }
1857                if self
1858                    .forwarding
1859                    .module_endpoint_for_connection(ctx.connection_id)
1860                    .map_err(RouterError::Forwarding)?
1861                    .is_some()
1862                {
1863                    if !is_known_module_request_op(&frame.body) {
1864                        return Ok(vec![control_error_frame(
1865                            &frame,
1866                            "unsupported_control_frame",
1867                            "module-originated channel-0 REQUEST is not supported",
1868                        )?]);
1869                    }
1870                    let request = match parse_module_control_request_from_module(&frame.body) {
1871                        Ok(request) => request,
1872                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1873                            return Ok(vec![control_error_frame(
1874                                &frame,
1875                                "unsupported_control_frame",
1876                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1877                            )?])
1878                        }
1879                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1880                            return Ok(vec![control_error_frame(
1881                                &frame,
1882                                "invalid_control_body",
1883                                format!("malformed module control body: {err}"),
1884                            )?])
1885                        }
1886                    };
1887                    let op = module_control_request_op(&request);
1888                    let corr = frame.header.corr;
1889                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1890                    let result =
1891                        self.handle_module_control_request(ctx.connection_id, frame, request);
1892                    log_slow_control_dispatch(
1893                        dispatch_started_at,
1894                        op,
1895                        ctx.connection_id,
1896                        corr,
1897                        None,
1898                    );
1899                    return result;
1900                }
1901
1902                if is_known_module_request_op(&frame.body) {
1903                    return Ok(vec![control_error_frame(
1904                        &frame,
1905                        "not_registered",
1906                        "catalog.update requires an active module registration owned by this connection",
1907                    )?]);
1908                }
1909
1910                let request = match parse_client_control_request(&frame.body) {
1911                    Ok(request) => request,
1912                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1913                        return Ok(vec![control_error_frame(
1914                            &frame,
1915                            "unknown_control_op",
1916                            format!("unknown client control op: {err}"),
1917                        )?])
1918                    }
1919                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1920                        return Ok(vec![control_error_frame(
1921                            &frame,
1922                            "invalid_control_body",
1923                            format!("malformed client control body: {err}"),
1924                        )?])
1925                    }
1926                };
1927                let op = client_control_request_op(&request);
1928                let corr = frame.header.corr;
1929                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1930                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1931                #[cfg(test)]
1932                if let Some(delay) = self.control_dispatch_delay {
1933                    tokio::time::sleep(delay).await;
1934                }
1935                let result = self
1936                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1937                    .await;
1938                if let Some(timings) = route_open_timings.as_mut() {
1939                    timings.finish();
1940                }
1941                log_slow_control_dispatch(
1942                    dispatch_started_at,
1943                    op,
1944                    ctx.connection_id,
1945                    corr,
1946                    route_open_timings.as_ref(),
1947                );
1948                result
1949            }
1950            FrameType::Push => {
1951                let Some(endpoint) = self
1952                    .forwarding
1953                    .module_endpoint_for_connection(ctx.connection_id)
1954                    .map_err(RouterError::Forwarding)?
1955                else {
1956                    return Ok(vec![control_error_frame(
1957                        &frame,
1958                        "unsupported_control_frame",
1959                        "client-originated channel-0 PUSH is not supported",
1960                    )?]);
1961                };
1962                self.handle_status_update(endpoint, frame)
1963            }
1964            FrameType::Response | FrameType::Error
1965                if self
1966                    .forwarding
1967                    .module_endpoint_for_connection(ctx.connection_id)
1968                    .map_err(RouterError::Forwarding)?
1969                    .is_some() =>
1970            {
1971                self.handle_module_relay_response(ctx.connection_id, frame)
1972            }
1973            ty => Ok(vec![control_error_frame(
1974                &frame,
1975                "unsupported_control_frame",
1976                format!("unsupported channel-0 frame {ty:?}"),
1977            )?]),
1978        }
1979    }
1980
1981    pub fn cleanup_connection(
1982        &self,
1983        connection_id: ConnectionId,
1984    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1985        self.caller_endpoints
1986            .lock()
1987            .unwrap_or_else(|p| p.into_inner())
1988            .remove(&connection_id);
1989        self.forwarding
1990            .operator_confirms()
1991            .connection_closed(connection_id);
1992        self.cleanup_connection_with_end_reason(
1993            connection_id,
1994            RegistrationEndReason::ConnectionClosed,
1995        )
1996    }
1997
1998    fn cleanup_connection_with_end_reason(
1999        &self,
2000        connection_id: ConnectionId,
2001        requested_reason: RegistrationEndReason,
2002    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2003        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2004            self.registry
2005                .get_module_by_connection(connection_id)?
2006                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2007                .and_then(|module| module.registration_end_reason().ok().flatten())
2008                .unwrap_or(requested_reason)
2009        } else {
2010            requested_reason
2011        };
2012        let crash_closed = self
2013            .registry
2014            .get_module_by_connection(connection_id)?
2015            .and_then(|registration| {
2016                self.forwarding
2017                    .module_endpoint_for_connection(connection_id)
2018                    .ok()
2019                    .flatten()
2020                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2021                    .map(|routes| (registration.manifest.module_id, routes))
2022            });
2023        let crash_closed = crash_closed.map(|(module_id, routes)| {
2024            let terminal = match self.supervisor.get(&module_id) {
2025                None => false,
2026                Some(module) => match module.will_recover_after_connection_loss() {
2027                    Ok(will_recover) => !will_recover,
2028                    Err(err) => {
2029                        warn!(
2030                            %module_id,
2031                            error = %err,
2032                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2033                        );
2034                        false
2035                    }
2036                },
2037            };
2038            // The forwarding table gates all providers at the start of daemon
2039            // shutdown, before their connections are closed. An ordinary
2040            // module disconnect still reports crash if that gate is not set.
2041            let (reason, drained) = match self
2042                .forwarding
2043                .daemon_drain_result_for_connection(connection_id)
2044            {
2045                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2046                Ok(None) => (RouteCloseReason::Crash, false),
2047                Err(err) => {
2048                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2049                    (RouteCloseReason::Crash, false)
2050                }
2051            };
2052            (module_id, routes, reason, drained, terminal)
2053        });
2054        let registrations = self.deregister_connection(connection_id, end_reason);
2055        let cleanup = if crash_closed.is_some() {
2056            self.forwarding.cleanup_connection_counted(connection_id)
2057        } else {
2058            // A module connection's teardown needs the count of abandoned
2059            // route.bind relays for its route.closed notice below. Any other
2060            // connection, such as a client's, sends no such notice and needs
2061            // only its routes released, so it uses the route-only wrapper and
2062            // reports zero.
2063            self.forwarding
2064                .cleanup_connection(connection_id)
2065                .map(|released| crate::forwarding::ConnectionCleanup {
2066                    released,
2067                    abandoned_relays: 0,
2068                })
2069        };
2070        // The route.closed push waits for forwarding teardown because only
2071        // teardown knows how many pending route.bind relays it aborted. It still
2072        // goes out before the GOODBYEs for the released routes, and its targets
2073        // were captured above, before teardown removed those routes.
2074        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2075            let abandoned = cleanup
2076                .as_ref()
2077                .map_or(0, |cleanup| cleanup.abandoned_relays);
2078            send_route_control_pushes(
2079                &self.forwarding,
2080                routes,
2081                ClientControlPush::RouteClosed {
2082                    module_id,
2083                    channels: Vec::new(),
2084                    reason,
2085                    drained,
2086                    abandoned,
2087                    excluded_subscriptions: 0,
2088                    terminal: Some(terminal),
2089                },
2090            );
2091        }
2092        if let Ok(cleanup) = cleanup {
2093            self.emit_route_goodbyes(cleanup.released);
2094        }
2095        // Signal the registration-release watch only now that BOTH registry and
2096        // forwarding teardown are done, so a supervisor waiting to spawn a
2097        // replacement never observes release while old routes still exist.
2098        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2099            crate::supervise::notify_registration_release();
2100            self.capability_evaluator.wake_deadline_loop();
2101            self.refresh_capability_requirements();
2102        }
2103        self.supervisor.remove_spawn_subscribers(connection_id);
2104        self.activity_streams
2105            .lock()
2106            .unwrap_or_else(|p| p.into_inner())
2107            .retain(|(id, _), task| {
2108                if *id == connection_id {
2109                    task.abort();
2110                    false
2111                } else {
2112                    true
2113                }
2114            });
2115        // Sync authority dies with its connection, so the owner's next
2116        // connection can take it; the owner's scopes stay as they are.
2117        self.hello_launch_nonces
2118            .lock()
2119            .unwrap_or_else(|poisoned| poisoned.into_inner())
2120            .forget(connection_id);
2121        self.scopes
2122            .write()
2123            .unwrap_or_else(|poisoned| poisoned.into_inner())
2124            .release_connection(connection_id);
2125        registrations
2126    }
2127
2128    pub(crate) fn handle_route_goodbye(
2129        &self,
2130        connection_id: ConnectionId,
2131        route_channel: u16,
2132        route_epoch: u32,
2133    ) -> Result<bool, RouterError> {
2134        debug!(
2135            connection_id = connection_id.get(),
2136            route_channel, route_epoch, "handling route GOODBYE"
2137        );
2138        let RouteRelease::Removed(released_route) = self
2139            .forwarding
2140            .release_client_route(connection_id, route_channel, route_epoch)
2141            .map_err(RouterError::Forwarding)?
2142        else {
2143            return Ok(false);
2144        };
2145        self.emit_route_goodbyes(vec![released_route]);
2146        Ok(true)
2147    }
2148
2149    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2150        for released in released_routes {
2151            let frame = match Frame::build_with_version(
2152                released.negotiated_ver,
2153                FrameType::Goodbye,
2154                control_flags(),
2155                released.channel,
2156                released.epoch,
2157                0,
2158                Vec::new(),
2159            ) {
2160                Ok(frame) => frame,
2161                Err(err) => {
2162                    warn!(
2163                        route_channel = released.channel,
2164                        error = %err,
2165                        "failed to build route GOODBYE frame"
2166                    );
2167                    continue;
2168                }
2169            };
2170            if !released.close_on_delivery_failure() {
2171                crate::forwarding::send_module_route_goodbye(
2172                    &self.counters,
2173                    &released.sink,
2174                    frame,
2175                    released.module_id.as_deref(),
2176                    "client route released",
2177                );
2178                continue;
2179            }
2180            if let Err(err) = released.sink.try_send(frame) {
2181                warn!(
2182                    target_connection_id = released.connection_id.get(),
2183                    route_channel = released.channel,
2184                    error = %err,
2185                    "route GOODBYE was not delivered to client; closing target connection"
2186                );
2187                if self
2188                    .forwarding
2189                    .escalate_client_delivery_failure(
2190                        released.connection_id,
2191                        released.channel,
2192                        released.epoch,
2193                        CloseReason::new(
2194                            "route_goodbye_delivery_failed",
2195                            format!(
2196                                "failed to enqueue route GOODBYE for channel {}: {err}",
2197                                released.channel
2198                            ),
2199                        ),
2200                        crate::forwarding::UndeliveredFrame {
2201                            module_id: released.module_id.as_deref(),
2202                            sink: &released.sink,
2203                        },
2204                    )
2205                    .unwrap_or(false)
2206                {
2207                    self.counters.increment_goodbye_relay_client_failed();
2208                }
2209            }
2210        }
2211    }
2212
2213    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2214    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2215    /// own commit failed after the module had already accepted). Without this, a
2216    /// module that accepts late keeps a binding subc has torn down, so a later
2217    /// frame on that module channel could misdeliver if the channel is reused.
2218    ///
2219    /// Never closes the shared module connection on failure: a dropped notification
2220    /// only wastes a bounded amount of warm module-side state, which the module's
2221    /// own idle reaper reclaims. Only call this once the route.bind relay was
2222    /// actually enqueued to the module — if the relay send itself failed, the
2223    /// module never created a binding and there is nothing to tear down.
2224    fn send_abandoned_route_bind_goodbye(
2225        &self,
2226        module_sink: &crate::FrameSink,
2227        negotiated_ver: u8,
2228        module_channel: u16,
2229        module_epoch: u32,
2230    ) {
2231        let frame = match Frame::build_with_version(
2232            negotiated_ver,
2233            FrameType::Goodbye,
2234            control_flags(),
2235            module_channel,
2236            module_epoch,
2237            0,
2238            Vec::new(),
2239        ) {
2240            Ok(frame) => frame,
2241            Err(err) => {
2242                warn!(
2243                    route_channel = module_channel,
2244                    error = %err,
2245                    "failed to build GOODBYE for abandoned route.bind"
2246                );
2247                return;
2248            }
2249        };
2250        crate::forwarding::send_module_route_goodbye(
2251            &self.counters,
2252            module_sink,
2253            frame,
2254            None,
2255            "abandoned route.bind",
2256        );
2257    }
2258
2259    fn handle_hello(
2260        &self,
2261        connection_id: ConnectionId,
2262        sink: Option<crate::FrameSink>,
2263        frame: Frame,
2264    ) -> Result<Vec<Frame>, RouterError> {
2265        debug!(
2266            connection_id = connection_id.get(),
2267            corr = frame.header.corr,
2268            "handling HELLO"
2269        );
2270        // A module connection has one identity for its entire lifetime. A second
2271        // registration would leave the old registry owner behind while replacing
2272        // its forwarding endpoint and launch nonce.
2273        if self
2274            .registry
2275            .get_module_by_connection(connection_id)
2276            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2277            .is_some()
2278        {
2279            return Ok(vec![control_error_frame(
2280                &frame,
2281                "invalid_hello",
2282                "connection is already registered as a module",
2283            )?]);
2284        }
2285        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2286            Ok(value) => value,
2287            Err(err) => {
2288                return Ok(vec![control_error_frame(
2289                    &frame,
2290                    "invalid_hello",
2291                    format!("malformed HELLO body: {err}"),
2292                )?])
2293            }
2294        };
2295        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2296            return Ok(vec![control_error_frame(
2297                &frame,
2298                "invalid_capability_grammar",
2299                err.to_string(),
2300            )?]);
2301        }
2302        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2303            return Ok(vec![control_error_frame(
2304                &frame,
2305                "invalid_manifest",
2306                err.to_string(),
2307            )?]);
2308        }
2309        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2310            return Ok(vec![control_error_frame(
2311                &frame,
2312                "invalid_event_declaration",
2313                err.to_string(),
2314            )?]);
2315        }
2316        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2317            return Ok(vec![control_error_frame(
2318                &frame,
2319                "invalid_manifest",
2320                err.to_string(),
2321            )?]);
2322        }
2323        if let Some(provenance) = hello_value
2324            .get("manifest")
2325            .and_then(|manifest| manifest.get("provenance"))
2326        {
2327            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2328                return Ok(vec![control_error_frame(
2329                    &frame,
2330                    "invalid_manifest",
2331                    format!("malformed manifest provenance: {err}"),
2332                )?]);
2333            }
2334        }
2335        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2336            Ok(hello) => hello,
2337            Err(err) => {
2338                return Ok(vec![control_error_frame(
2339                    &frame,
2340                    "invalid_hello",
2341                    format!("malformed HELLO body: {err}"),
2342                )?])
2343            }
2344        };
2345
2346        if hello.protocol_ver != hello.manifest.protocol_ver {
2347            return Ok(vec![control_error_frame(
2348                &frame,
2349                "invalid_manifest",
2350                format!(
2351                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2352                    hello.protocol_ver, hello.manifest.protocol_ver
2353                ),
2354            )?]);
2355        }
2356
2357        if hello.manifest.module_id.trim().is_empty() {
2358            return Ok(vec![control_error_frame(
2359                &frame,
2360                "invalid_manifest",
2361                "manifest module_id must not be empty",
2362            )?]);
2363        }
2364
2365        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2366            Ok(negotiated_ver) => negotiated_ver,
2367            Err(message) => {
2368                return Ok(vec![control_error_frame(
2369                    &frame,
2370                    "version_unsupported",
2371                    message,
2372                )?])
2373            }
2374        };
2375
2376        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2377        // swap is open for this id, the only HELLO admitted as a second process
2378        // is the one carrying the candidate's launch nonce (the swap token), and
2379        // it registers into the candidate slot rather than being refused as a
2380        // duplicate. Run after the reserved gate, a reserved module's candidate
2381        // would be refused `reserved_module` for presenting a nonce that gate
2382        // does not know. See `SupervisorHandle::swap_hello_admission`.
2383        let swap_admission = self
2384            .supervisor
2385            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2386        if swap_admission == SwapHelloAdmission::Refused {
2387            warn!(
2388                module_id = %hello.manifest.module_id,
2389                connection_id = connection_id.get(),
2390                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2391            );
2392            return Ok(vec![control_error_frame(
2393                &frame,
2394                "swap_token_invalid",
2395                format!(
2396                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2397                    hello.manifest.module_id
2398                ),
2399            )?]);
2400        }
2401        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2402
2403        // A swap candidate proves its identity with the candidate nonce above.
2404        // Other registrations receive this catalog fact only when reserved-module
2405        // policy required and accepted a nonce; unreserved registrations require
2406        // no nonce check.
2407        let nonce_verified = if swap_candidate {
2408            true
2409        } else {
2410            match self.supervisor.reserved_hello_verification(
2411                &hello.manifest.module_id,
2412                hello.launch_nonce.as_deref(),
2413            ) {
2414                Ok(verified) => verified,
2415                Err(rejection) => {
2416                    let message = match rejection {
2417                        ReservedHelloRejection::Exact { module_id } => format!(
2418                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2419                        ),
2420                        ReservedHelloRejection::Prefix {
2421                            prefix,
2422                            owner_module_id,
2423                        } => format!(
2424                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2425                            hello.manifest.module_id
2426                        ),
2427                    };
2428                    return Ok(vec![control_error_frame(
2429                        &frame,
2430                        "reserved_module",
2431                        message,
2432                    )?]);
2433                }
2434            }
2435        };
2436
2437        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2438            &hello.manifest.module_id,
2439            hello.manifest.capabilities.as_ref(),
2440        );
2441        if let Some(refusal) = reserved_capability_refusals.first() {
2442            let capability = refusal.capability.clone();
2443            let bound_module = refusal.claimants[0].clone();
2444            log_duplicate_claim_events(reserved_capability_refusals);
2445            return Ok(vec![control_error_frame(
2446                &frame,
2447                "reserved_capability",
2448                format!(
2449                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2450                    capability, bound_module, hello.manifest.module_id
2451                ),
2452            )?]);
2453        }
2454
2455        // A connection that already opened client routes must not also register as
2456        // a module: cleanup would then release only one side and leak the other.
2457        if self
2458            .forwarding
2459            .connection_has_client_routes(connection_id)
2460            .map_err(RouterError::Forwarding)?
2461        {
2462            return Ok(vec![control_error_frame(
2463                &frame,
2464                "invalid_hello",
2465                "connection has open client routes and cannot also register as a module",
2466            )?]);
2467        }
2468
2469        // Kept for scope sync authority, which goes only to the connection that
2470        // presented the module's current launch nonce. Recorded before the
2471        // registration is attempted: a connection whose registration then fails
2472        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2473        self.hello_launch_nonces
2474            .lock()
2475            .unwrap_or_else(|poisoned| poisoned.into_inner())
2476            .record(connection_id, hello.launch_nonce.as_deref());
2477        let control_ops = effective_module_control_ops(hello.control_ops);
2478        // Built before anything is registered so an encoding failure leaves no
2479        // registry or forwarding state behind.
2480        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2481        if swap_candidate {
2482            return self.register_swap_candidate(
2483                connection_id,
2484                sink,
2485                &frame,
2486                hello.manifest,
2487                negotiated_ver,
2488                control_ops,
2489                hello_ack,
2490            );
2491        }
2492        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2493        // is refused, but `catalog.list` does not show the module and the
2494        // catalog generation does not move until it is published below, once
2495        // its forwarding endpoint exists. Listing it now would advertise a
2496        // module that `route.open` still refuses for having no forwarding
2497        // connection.
2498        let registration = match self
2499            .registry
2500            .register_unpublished_verified_with_control_ops(
2501                hello.manifest,
2502                negotiated_ver,
2503                connection_id,
2504                control_ops,
2505                nonce_verified,
2506            ) {
2507            Ok(registration) => registration,
2508            Err(RegistryError::DuplicateModuleId { module_id }) => {
2509                return Ok(vec![control_error_frame(
2510                    &frame,
2511                    "duplicate_module_id",
2512                    format!(
2513                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2514                    ),
2515                )?])
2516            }
2517            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2518                return Ok(vec![control_error_frame(
2519                    &frame,
2520                    "invalid_module_id",
2521                    err.to_string(),
2522                )?])
2523            }
2524            Err(err) => {
2525                return Ok(vec![control_error_frame(
2526                    &frame,
2527                    "registry_error",
2528                    err.to_string(),
2529                )?])
2530            }
2531        };
2532        #[cfg(test)]
2533        self.hold_between_registry_and_forwarding();
2534
2535        let reply = if let Some(sink) = sink {
2536            // The forwarding table's module store is also the daemon-to-module
2537            // control-RPC lane, so every HELLO gets a live endpoint even when the
2538            // manifest has no routable provider role. Non-routable modules still
2539            // cannot receive route.bind in production: `handle_route_open` checks
2540            // the registry manifest with `target_has_required_role` before the
2541            // only production call to `begin_route_bind_relay_for` below that
2542            // route.open path. The remaining direct relay callers are unit tests
2543            // and benchmark harnesses that construct forwarding state explicitly.
2544            //
2545            // The HELLO_ACK is queued by the forwarding table itself, before the
2546            // endpoint becomes visible, and is NOT returned as a reply. A module
2547            // reads HELLO_ACK first and exits on anything else; a reply is only
2548            // written after this handler returns, by which time a route.open on
2549            // another connection could already have queued a route.bind request
2550            // for this module ahead of it.
2551            let concurrency = manifest_concurrency(&registration.manifest);
2552            if let Err(err) = self.forwarding.register_module_connection_acked(
2553                connection_id,
2554                registration.manifest.module_id.clone(),
2555                negotiated_ver,
2556                concurrency,
2557                sink,
2558                hello_ack,
2559            ) {
2560                // Forwarding registration failed, so there is no forwarding
2561                // state to tear down. Remove the registry entry, which was never
2562                // published and so never listed or counted in the catalog
2563                // generation, and signal the release watch directly.
2564                if matches!(
2565                    self.deregister_connection(
2566                        connection_id,
2567                        RegistrationEndReason::RegistrationFailed,
2568                    ),
2569                    Ok(r) if !r.is_empty()
2570                ) {
2571                    crate::supervise::notify_registration_release();
2572                }
2573                return Ok(vec![control_error_frame(
2574                    &frame,
2575                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2576                        "invalid_hello"
2577                    } else {
2578                        forwarding_error_code(&err)
2579                    },
2580                    err.to_string(),
2581                )?]);
2582            }
2583            Vec::new()
2584        } else {
2585            // No sink means no forwarding endpoint, so nothing can be routed
2586            // ahead of the ack; it goes out as the reply.
2587            vec![hello_ack]
2588        };
2589
2590        // The module is routable now (or, without a sink, as routable as it
2591        // will ever be), so the catalog may list it. Publishing bumps the
2592        // catalog generation in the same registry critical section that makes
2593        // the module listed.
2594        if !self
2595            .registry
2596            .publish_registration(connection_id)
2597            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2598        {
2599            debug!(
2600                module_id = %registration.manifest.module_id,
2601                connection_id = connection_id.get(),
2602                "registration was removed before it could be published"
2603            );
2604        }
2605
2606        // Exposure over assumption: Concurrency's serde default is pinned to the
2607        // pre-field behavior (ModuleManaged), so a management surface that is
2608        // genuinely Serial and just never declared it inherits concurrent
2609        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2610        // turns "no module has been bitten yet" into the checkable claim "no
2611        // module is exposed" -- one read of the boot log instead of a fleet
2612        // audit. Detected from the raw HELLO bytes because the serde default
2613        // deliberately erases the absent/declared distinction from the type.
2614        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2615            info!(
2616                module_id = %registration.manifest.module_id,
2617                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2618            );
2619        }
2620
2621        self.apply_registration_capabilities(&registration);
2622
2623        self.supervisor
2624            .record_activity_registration(&registration.manifest.module_id);
2625
2626        info!(
2627            module_id = %registration.manifest.module_id,
2628            module_version = %registration.manifest.module_version,
2629            negotiated_ver,
2630            routable_provider = manifest_provides_routable_role(&registration.manifest),
2631            connection_id = connection_id.get(),
2632            "module registered"
2633        );
2634
2635        Ok(reply)
2636    }
2637
2638    /// Register a HELLO the swap gate admitted into the candidate slot of the
2639    /// registry and of forwarding, where it is reachable over its own
2640    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2641    /// nothing routes to it until the supervisor cuts over.
2642    ///
2643    /// Registry first (unpublished), then forwarding, then publication, the
2644    /// same order as an ordinary HELLO; a forwarding failure removes the
2645    /// registry entry again. The swap supervisor treats a published, ready
2646    /// candidate as registered and promotes it, and promotion needs the
2647    /// candidate's forwarding endpoint, so the candidate is published only
2648    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2649    /// recompute capability verdicts or run the capability deny census
2650    /// (`enforce_capability_denies`): both are about the module in the active
2651    /// slot, which `route.open` can target. A candidate sits in the candidate
2652    /// slot, so it neither provides capabilities to other modules nor can be
2653    /// routed to until promotion moves it into the active slot.
2654    #[allow(clippy::too_many_arguments)]
2655    fn register_swap_candidate(
2656        &self,
2657        connection_id: ConnectionId,
2658        sink: Option<crate::FrameSink>,
2659        frame: &Frame,
2660        manifest: ModuleManifest,
2661        negotiated_ver: u8,
2662        control_ops: Vec<String>,
2663        hello_ack: Frame,
2664    ) -> Result<Vec<Frame>, RouterError> {
2665        let module_id = manifest.module_id.clone();
2666        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2667            manifest,
2668            negotiated_ver,
2669            connection_id,
2670            control_ops,
2671        ) {
2672            Ok(registration) => registration,
2673            Err(RegistryError::DuplicateModuleId { module_id }) => {
2674                return Ok(vec![control_error_frame(
2675                    frame,
2676                    "duplicate_module_id",
2677                    format!(
2678                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2679                    ),
2680                )?])
2681            }
2682            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2683                return Ok(vec![control_error_frame(
2684                    frame,
2685                    "invalid_module_id",
2686                    err.to_string(),
2687                )?])
2688            }
2689            Err(err) => {
2690                return Ok(vec![control_error_frame(
2691                    frame,
2692                    "registry_error",
2693                    err.to_string(),
2694                )?])
2695            }
2696        };
2697        #[cfg(test)]
2698        self.hold_between_registry_and_forwarding();
2699        let reply = if let Some(sink) = sink {
2700            // Same ordering as an ordinary HELLO: the forwarding table queues
2701            // the HELLO_ACK before the candidate endpoint is inserted, because
2702            // a module exits if its first frame after HELLO is anything else.
2703            let concurrency = manifest_concurrency(&registration.manifest);
2704            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2705                connection_id,
2706                module_id.clone(),
2707                negotiated_ver,
2708                concurrency,
2709                sink,
2710                hello_ack,
2711            ) {
2712                if matches!(
2713                    self.deregister_connection(
2714                        connection_id,
2715                        RegistrationEndReason::RegistrationFailed,
2716                    ),
2717                    Ok(r) if !r.is_empty()
2718                ) {
2719                    crate::supervise::notify_registration_release();
2720                }
2721                return Ok(vec![control_error_frame(
2722                    frame,
2723                    forwarding_error_code(&err),
2724                    err.to_string(),
2725                )?]);
2726            }
2727            Vec::new()
2728        } else {
2729            vec![hello_ack]
2730        };
2731        if !self
2732            .registry
2733            .publish_registration(connection_id)
2734            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2735        {
2736            debug!(
2737                module_id = %module_id,
2738                connection_id = connection_id.get(),
2739                "swap candidate registration was removed before it could be published"
2740            );
2741        }
2742        self.supervisor.mark_swap_candidate_admitted(&module_id);
2743        info!(
2744            module_id = %module_id,
2745            module_version = %registration.manifest.module_version,
2746            negotiated_ver,
2747            ready = registration.ready,
2748            connection_id = connection_id.get(),
2749            "swap candidate registered; not routable until cutover"
2750        );
2751        Ok(reply)
2752    }
2753
2754    fn build_hello_ack(
2755        &self,
2756        frame: &Frame,
2757        negotiated_ver: u8,
2758        module_id: &str,
2759    ) -> Result<Frame, RouterError> {
2760        let ack = ModuleHelloAckBody {
2761            negotiated_ver,
2762            subc_ops: module_subc_ops(),
2763            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2764            storage: self
2765                .storage_config
2766                .as_ref()
2767                .map(|cfg| cfg.descriptor_for(module_id)),
2768            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2769        };
2770        let body = serde_json::to_vec(&ack).map_err(|err| {
2771            RouterError::backend(
2772                0,
2773                frame.header.corr,
2774                format!("failed to encode HELLO_ACK: {err}"),
2775            )
2776        })?;
2777
2778        Frame::build_with_version(
2779            negotiated_ver,
2780            FrameType::HelloAck,
2781            control_flags(),
2782            0,
2783            0,
2784            frame.header.corr,
2785            body,
2786        )
2787        .map_err(RouterError::FrameBuild)
2788    }
2789
2790    async fn handle_client_control_request(
2791        &self,
2792        ctx: &RouteCtx,
2793        frame: Frame,
2794        request: ClientControlRequest,
2795        route_open_timings: &mut Option<RouteOpenTimings>,
2796    ) -> Result<Vec<Frame>, RouterError> {
2797        if let Some(operation) = privileged_operation(&request) {
2798            let caller = self.resolve_caller(ctx.connection_id).await;
2799            let op = client_control_request_op(&request);
2800            let text = prompt_text(operation, &caller);
2801            match self.privileged_operator_prompt {
2802                PrivilegedOperatorPrompt::Log => {
2803                    info!(target: "subc_daemon::privileged_prompt", operation = op,
2804                        target = ?operation, caller = %caller.description(), summary = %text,
2805                        "privileged request would prompt the operator");
2806                }
2807                PrivilegedOperatorPrompt::Enforce => {
2808                    let key =
2809                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2810                    let outcome = self
2811                        .forwarding
2812                        .operator_confirms()
2813                        .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2814                        .await;
2815                    if outcome.code != "confirmed" {
2816                        return Ok(vec![outcome.frame(&frame)]);
2817                    }
2818                }
2819            }
2820        }
2821        match request {
2822            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2823            ClientControlRequest::CatalogList { module_id } => {
2824                self.handle_catalog_list(frame, module_id)
2825            }
2826            ClientControlRequest::RouteOpen {
2827                target,
2828                identity,
2829                consumer_identity,
2830                consumer_capabilities,
2831                role_versions,
2832                admission_facts,
2833                scope,
2834            } => {
2835                self.handle_route_open(
2836                    ctx,
2837                    frame,
2838                    RouteOpenRequest {
2839                        target,
2840                        identity,
2841                        consumer_identity,
2842                        consumer_capabilities,
2843                        role_versions,
2844                        admission_facts,
2845                        scope,
2846                    },
2847                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2848                )
2849                .await
2850            }
2851            ClientControlRequest::RoutePoll {
2852                route_channel,
2853                route_epoch,
2854                kind,
2855            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2856            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2857            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2858                self.handle_supervisor_spawn_snapshot(frame)
2859            }
2860            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2861                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2862            }
2863            ClientControlRequest::SupervisorRestart {
2864                module_id,
2865                drain_timeout_ms,
2866            } => {
2867                self.log_supervisor_request_received(
2868                    ctx,
2869                    frame.header.corr,
2870                    ops::SUPERVISOR_RESTART,
2871                    Some(&module_id),
2872                    None,
2873                )?;
2874                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2875                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2876                    .await
2877            }
2878            ClientControlRequest::SupervisorSwap {
2879                module_id,
2880                ready_timeout_ms,
2881            } => {
2882                self.log_supervisor_request_received(
2883                    ctx,
2884                    frame.header.corr,
2885                    ops::SUPERVISOR_SWAP,
2886                    Some(&module_id),
2887                    None,
2888                )?;
2889                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2890                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2891                    .await
2892            }
2893            ClientControlRequest::SupervisorReload { module_id } => {
2894                self.log_supervisor_request_received(
2895                    ctx,
2896                    frame.header.corr,
2897                    ops::SUPERVISOR_RELOAD,
2898                    Some(&module_id),
2899                    None,
2900                )?;
2901                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2902                self.handle_supervisor_reload(frame, module_id, requester)
2903                    .await
2904            }
2905            ClientControlRequest::SupervisorRescan { preview } => {
2906                if !preview {
2907                    self.log_supervisor_request_received(
2908                        ctx,
2909                        frame.header.corr,
2910                        ops::SUPERVISOR_RESCAN,
2911                        None,
2912                        None,
2913                    )?;
2914                }
2915                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2916                self.handle_supervisor_rescan(
2917                    frame,
2918                    if preview {
2919                        RescanMode::Preview
2920                    } else {
2921                        RescanMode::Execute
2922                    },
2923                    requester,
2924                )
2925                .await
2926            }
2927            ClientControlRequest::SupervisorRescanChecked {
2928                preview,
2929                candidate,
2930                expect,
2931            } => {
2932                self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2933                    .await
2934            }
2935            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2936                self.log_supervisor_request_received(
2937                    ctx,
2938                    frame.header.corr,
2939                    ops::SUPERVISOR_RELEASE_RESERVED,
2940                    Some(&module_id),
2941                    None,
2942                )?;
2943                self.handle_supervisor_release_reserved(frame, module_id)
2944                    .await
2945            }
2946            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2947                self.log_supervisor_request_received(
2948                    ctx,
2949                    frame.header.corr,
2950                    ops::SUPERVISOR_SET_ENABLED,
2951                    Some(&module_id),
2952                    Some(enabled),
2953                )?;
2954                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2955                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2956                    .await
2957            }
2958            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2959                self.handle_supervisor_health_probe(frame, module_id).await
2960            }
2961            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2962            ClientControlRequest::SupervisorRoutes { module_id } => {
2963                self.handle_supervisor_routes(frame, module_id)
2964            }
2965            ClientControlRequest::SupervisorProvenance { module_id } => {
2966                self.handle_supervisor_provenance(frame, module_id).await
2967            }
2968            ClientControlRequest::SupervisorStderrTail {
2969                module_id,
2970                max_lines,
2971                max_bytes,
2972            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2973            ClientControlRequest::SupervisorTerminals { module_id } => {
2974                self.handle_supervisor_terminals(frame, module_id).await
2975            }
2976        }
2977    }
2978
2979    fn log_supervisor_request_received(
2980        &self,
2981        ctx: &RouteCtx,
2982        corr: u64,
2983        op: &'static str,
2984        module_id: Option<&str>,
2985        enabled: Option<bool>,
2986    ) -> Result<(), RouterError> {
2987        let caller = self.supervisor_requester(ctx, corr)?;
2988        let caller = principal_label(&caller);
2989        // `Option` fields are recorded only when present, so a request with no
2990        // module id (rescan) or no enabled flag simply omits that field.
2991        info!(
2992            target: "control",
2993            op,
2994            module_id,
2995            enabled,
2996            connection_id = ctx.connection_id.get(),
2997            caller = %caller,
2998            "supervisor request received"
2999        );
3000        Ok(())
3001    }
3002
3003    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3004        Ok(self
3005            .registry
3006            .get_module_by_connection(ctx.connection_id)
3007            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3008            .map(|registration| Principal::Reserved {
3009                module_id: registration.manifest.module_id,
3010            })
3011            .unwrap_or(Principal::Direct))
3012    }
3013
3014    fn handle_module_control_request(
3015        &self,
3016        connection_id: ConnectionId,
3017        frame: Frame,
3018        request: ModuleControlRequestFromModule,
3019    ) -> Result<Vec<Frame>, RouterError> {
3020        match request {
3021            ModuleControlRequestFromModule::CatalogUpdate {
3022                provides,
3023                capabilities,
3024                ready,
3025            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3026            ModuleControlRequestFromModule::LiveRoots {} => {
3027                let registered = self
3028                    .registry
3029                    .get_module_by_connection(connection_id)
3030                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3031                let Some(registration) = registered else {
3032                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3033                };
3034                let response = self
3035                    .forwarding
3036                    .live_roots(&registration.manifest.module_id)
3037                    .map_err(RouterError::Forwarding)?;
3038                Ok(vec![control_response_body_frame(
3039                    &frame,
3040                    &response,
3041                    "ModuleControlResponseToModule::LiveRoots",
3042                )?])
3043            }
3044            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3045                self.handle_scope_sync(connection_id, frame, generation, scopes)
3046            }
3047            ModuleControlRequestFromModule::ScopeApply {
3048                generation,
3049                upsert,
3050                end,
3051            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3052            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3053                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3054            }
3055        }
3056    }
3057
3058    fn handle_scope_served_by(
3059        &self,
3060        connection_id: ConnectionId,
3061        frame: Frame,
3062    ) -> Result<Vec<Frame>, RouterError> {
3063        let registered = self
3064            .registry
3065            .get_module_by_connection(connection_id)
3066            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3067        let Some(registered) = registered else {
3068            return Ok(vec![control_error_frame(
3069                &frame,
3070                "not_registered",
3071                "scope.served_by requires an active module registration owned by this connection",
3072            )?]);
3073        };
3074        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3075        {
3076            Ok(request) => request,
3077            Err(err) => {
3078                return Ok(vec![control_error_frame(
3079                    &frame,
3080                    "invalid_control_body",
3081                    format!("malformed scope.served_by body: {err}"),
3082                )?])
3083            }
3084        };
3085        let answer = self
3086            .scopes
3087            .read()
3088            .unwrap_or_else(|poisoned| poisoned.into_inner())
3089            .served_by(
3090                &registered.manifest.module_id,
3091                &request.scope_ref,
3092                request.scope_epoch,
3093                &request.module_id,
3094            );
3095        let reply = subc_control::ScopeServedByReply::new(
3096            answer,
3097            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3098        );
3099        Ok(vec![control_response_body_frame(
3100            &frame,
3101            &reply,
3102            "ScopeServedByReply",
3103        )?])
3104    }
3105
3106    fn handle_operator_confirm(
3107        &self,
3108        ctx: &RouteCtx,
3109        frame: Frame,
3110    ) -> Result<Vec<Frame>, RouterError> {
3111        use crate::operator_confirm::{audit, Outcome};
3112        let registration = self
3113            .registry
3114            .get_module_by_connection(ctx.connection_id)
3115            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3116        let Some(registration) = registration else {
3117            let outcome = Outcome::refusal("not_registered");
3118            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3119            return Ok(vec![outcome.frame(&frame)]);
3120        };
3121        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3122            Ok(request) => request,
3123            Err(_) => {
3124                let outcome = Outcome::refusal("invalid_control_body");
3125                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3126                return Ok(vec![outcome.frame(&frame)]);
3127            }
3128        };
3129        let module_id = registration.manifest.module_id;
3130        // Read the launch nonce (under its own lock) before taking the forwarding
3131        // table's lock below: holding forwarding while waiting on another daemon
3132        // lock risks a lock-order deadlock with paths that take them the other way.
3133        let nonce = self
3134            .hello_launch_nonces
3135            .lock()
3136            .unwrap_or_else(|p| p.into_inner())
3137            .nonce(ctx.connection_id)
3138            .map(str::to_owned);
3139        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3140            self.supervisor
3141                .spawned_consumer_authorized(&module_id, nonce)
3142        });
3143        let confirms = self.forwarding.operator_confirms();
3144        self.forwarding
3145            .with_operator_route(
3146                ctx.connection_id,
3147                request.route_channel,
3148                request.route_epoch,
3149                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3150            )
3151            .map_err(RouterError::Forwarding)
3152    }
3153
3154    /// Resolve folders before taking the scope-table lock: a slow filesystem
3155    /// must not hold up admissions for unrelated scoped routes.
3156    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3157        let mut scopes = ScopeRecords::from(scopes);
3158        for (index, record) in scopes.records.iter_mut().enumerate() {
3159            if let Some(refusal) = Self::canonicalize_workdir(record) {
3160                scopes.refusals.insert(index, refusal);
3161            }
3162        }
3163        scopes
3164    }
3165
3166    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3167        let workdir = record.workdir.as_mut()?;
3168        let path = std::path::Path::new(workdir);
3169        if !path.is_absolute() {
3170            return Some((
3171                error_codes::INVALID_WORKDIR,
3172                "workdir must be absolute".to_string(),
3173            ));
3174        }
3175        match ProjectRootId::from_path_allowing_missing(path) {
3176            Ok(root) => match root.as_path().to_str() {
3177                Some(canonical) => {
3178                    *workdir = canonical.to_string();
3179                    None
3180                }
3181                None => Some((
3182                    error_codes::INVALID_WORKDIR,
3183                    "canonical workdir is not UTF-8".to_string(),
3184                )),
3185            },
3186            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3187        }
3188    }
3189
3190    /// `scope.sync`: the owner is the module registered on this connection.
3191    /// A connection with no registration (every client connection, `direct`
3192    /// included) is refused `not_registered` before the table is consulted.
3193    fn handle_scope_sync(
3194        &self,
3195        connection_id: ConnectionId,
3196        frame: Frame,
3197        generation: u64,
3198        scopes: Vec<ScopeRecord>,
3199    ) -> Result<Vec<Frame>, RouterError> {
3200        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3201    }
3202
3203    fn handle_scope_change(
3204        &self,
3205        connection_id: ConnectionId,
3206        frame: Frame,
3207        generation: u64,
3208        scopes: Vec<ScopeRecord>,
3209        end: Option<Vec<ScopeEnd>>,
3210    ) -> Result<Vec<Frame>, RouterError> {
3211        let is_apply = end.is_some();
3212        let Some(registration) = self
3213            .registry
3214            .get_module_by_connection(connection_id)
3215            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3216        else {
3217            return Ok(vec![control_error_frame(
3218                &frame,
3219                "not_registered",
3220                "scope.sync and scope.apply require an active module registration owned by this connection",
3221            )?]);
3222        };
3223        let owner = registration.manifest.module_id;
3224        let scopes = Self::prepare_scope_records(scopes);
3225        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3226        let is_current_launch = |connection: ConnectionId| {
3227            self.hello_launch_nonces
3228                .lock()
3229                .unwrap_or_else(|poisoned| poisoned.into_inner())
3230                .presented(connection, current_nonce.as_deref())
3231        };
3232        // Lock order is the scope table, then the forwarding table: the new
3233        // tags are published, and the routes the change closes are selected,
3234        // while the scope table is still write-locked, so no admission can read
3235        // a record whose tag is not yet published.
3236        let mut table = self
3237            .scopes
3238            .write()
3239            .unwrap_or_else(|poisoned| poisoned.into_inner());
3240        let outcome = match end {
3241            Some(end) => table.apply(
3242                &owner,
3243                connection_id,
3244                is_current_launch,
3245                generation,
3246                scopes,
3247                end,
3248            ),
3249            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3250        };
3251        let drained = match &outcome {
3252            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3253            Err(_) => Vec::new(),
3254        };
3255        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3256        drop(table);
3257        if has_deadlines {
3258            self.scope_deadline_added.notify_one();
3259        }
3260        match outcome {
3261            Ok(applied) => {
3262                let counts = ScopeOutcomeCounts::of(&applied.results);
3263                info!(
3264                    owner = %owner,
3265                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3266                    generation,
3267                    records = applied.results.len(),
3268                    created = counts.created,
3269                    replaced = counts.replaced,
3270                    updated = counts.updated,
3271                    unchanged = counts.unchanged,
3272                    refused = counts.refused,
3273                    ended = applied.ended.len(),
3274                    tag_changes = applied.tag_changes.len(),
3275                    routes_closed = drained.len(),
3276                    "scope change accepted"
3277                );
3278                // An accepted scope change can still refuse individual records, and the
3279                // owner is the only party that sees the reply. Name them here so
3280                // an operator can tell a refused session from a missing one
3281                // without the owner's logs. Capped so a call that refuses
3282                // thousands cannot flood the log; the count above is complete.
3283                for refused in applied
3284                    .results
3285                    .iter()
3286                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3287                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3288                {
3289                    warn!(
3290                        owner = %owner,
3291                        generation,
3292                        scope_ref = %refused.scope_ref,
3293                        scope_epoch = refused.scope_epoch,
3294                        code = refused.code.as_deref().unwrap_or(""),
3295                        "scope record refused"
3296                    );
3297                }
3298                self.close_scope_drained_routes(drained);
3299                let response = if is_apply {
3300                    ModuleControlResponseToModule::ScopeApply {
3301                        generation,
3302                        results: applied.results,
3303                        end_results: applied.end_results,
3304                        ended: applied.ended,
3305                    }
3306                } else {
3307                    ModuleControlResponseToModule::ScopeSync {
3308                        generation,
3309                        results: applied.results,
3310                        ended: applied.ended,
3311                    }
3312                };
3313                Ok(vec![control_response_body_frame(
3314                    &frame,
3315                    &response,
3316                    "scope change response",
3317                )?])
3318            }
3319            Err(refusal) => {
3320                info!(
3321                    owner = %owner,
3322                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3323                    generation,
3324                    code = refusal.code,
3325                    "scope change refused"
3326                );
3327                Ok(vec![control_error_frame(
3328                    &frame,
3329                    refusal.code,
3330                    refusal.message,
3331                )?])
3332            }
3333        }
3334    }
3335
3336    /// Tell both ends of each route a scope change closed. The module gets a
3337    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3338    /// the client's route handle. The client also gets `route.closed` with the
3339    /// scope reason, one push per module and reason, so it can tell a revoked
3340    /// route from an ordinary close and not reopen it.
3341    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3342        if drained.is_empty() {
3343            return;
3344        }
3345        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3346            BTreeMap::new();
3347        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3348        for route in drained {
3349            warn!(
3350                module_id = %route.module_id,
3351                reason = ?route.reason,
3352                client_connection_id = route.client.connection_id.get(),
3353                route_channel = route.client.channel,
3354                "closing route because its scope changed"
3355            );
3356            pushes
3357                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3358                .or_insert_with(|| (route.reason, Vec::new()))
3359                .1
3360                .push(EndpointRoute {
3361                    goodbye_target: route.client.clone(),
3362                    principal: Principal::Unverified,
3363                    bound_at: Instant::now(),
3364                    draining: false,
3365                    drain_reason: None,
3366                });
3367            goodbyes.push(route.module);
3368            goodbyes.push(route.client);
3369        }
3370        for ((module_id, _), (reason, routes)) in pushes {
3371            send_route_control_pushes(
3372                &self.forwarding,
3373                routes,
3374                ClientControlPush::RouteClosed {
3375                    module_id,
3376                    channels: Vec::new(),
3377                    reason,
3378                    drained: false,
3379                    abandoned: 0,
3380                    excluded_subscriptions: 0,
3381                    terminal: Some(false),
3382                },
3383            );
3384        }
3385        self.emit_route_goodbyes(goodbyes);
3386    }
3387
3388    /// `scope.describe`: any registered module may read any scope, because a
3389    /// provider must read the scope a route it serves is stamped with.
3390    fn handle_scope_describe(
3391        &self,
3392        connection_id: ConnectionId,
3393        frame: Frame,
3394        owner: Principal,
3395        scope_ref: String,
3396    ) -> Result<Vec<Frame>, RouterError> {
3397        let registered = self
3398            .registry
3399            .get_module_by_connection(connection_id)
3400            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3401        let Some(registered) = registered else {
3402            return Ok(vec![control_error_frame(
3403                &frame,
3404                "not_registered",
3405                "scope.describe requires an active module registration owned by this connection",
3406            )?]);
3407        };
3408        let mut description = self
3409            .scopes
3410            .read()
3411            .unwrap_or_else(|poisoned| poisoned.into_inner())
3412            .describe(&owner, &scope_ref);
3413        // Describe readers also decode strict ScopeAttributes. Attribution is
3414        // safe to omit unless the requesting module declares it understands it.
3415        if !registered
3416            .manifest
3417            .capabilities
3418            .as_ref()
3419            .is_some_and(|caps| {
3420                caps.provides
3421                    .iter()
3422                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3423            })
3424        {
3425            if let Some(stamp) = description.stamp.as_mut() {
3426                stamp.attributes.on_behalf_of_agent = None;
3427            }
3428        }
3429        let owner_configured = match &owner {
3430            // Ask whether the owner is configured (`is_configured`), not
3431            // whether it is on the roster (`get(..).is_some()`): a supervised
3432            // module's process can register and describe a scope before the
3433            // supervisor has put it on the roster.
3434            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3435            _ => false,
3436        };
3437        let response = ModuleControlResponseToModule::ScopeDescribe {
3438            status: description.status,
3439            scope_epoch: description.scope_epoch,
3440            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3441            owner_synced: description.owner_synced,
3442            owner_configured,
3443            scope: description.stamp,
3444        };
3445        Ok(vec![control_response_body_frame(
3446            &frame,
3447            &response,
3448            "ModuleControlResponseToModule::ScopeDescribe",
3449        )?])
3450    }
3451
3452    fn handle_catalog_update(
3453        &self,
3454        connection_id: ConnectionId,
3455        frame: Frame,
3456        provides: Vec<ProviderRole>,
3457        capabilities: Option<CapabilityDeclarations>,
3458        ready: Option<bool>,
3459    ) -> Result<Vec<Frame>, RouterError> {
3460        self.refresh_capability_requirements();
3461        let Some(registration) = self
3462            .registry
3463            .get_module_by_connection(connection_id)
3464            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3465        else {
3466            return Ok(vec![control_error_frame(
3467                &frame,
3468                "not_registered",
3469                "catalog.update requires an active module registration owned by this connection",
3470            )?]);
3471        };
3472
3473        if let Some(message) =
3474            catalog_update_frozen_field_message(&registration.manifest, &provides)
3475        {
3476            return Ok(vec![control_error_frame(
3477                &frame,
3478                "catalog_update_frozen_field",
3479                message,
3480            )?]);
3481        }
3482
3483        let mut candidate = registration.manifest.clone();
3484        candidate.provides = provides.clone();
3485        candidate.capabilities = capabilities
3486            .clone()
3487            .or_else(|| registration.manifest.capabilities.clone());
3488        if let Some(rich_ui) = &candidate.rich_ui {
3489            if let Err(err) = rich_ui.validate() {
3490                return Ok(vec![control_error_frame(
3491                    &frame,
3492                    "invalid_manifest",
3493                    err.to_string(),
3494                )?]);
3495            }
3496        }
3497        if let Err(err) = candidate.validate_capability_grammar() {
3498            return Ok(vec![control_error_frame(
3499                &frame,
3500                "invalid_capability_grammar",
3501                err.to_string(),
3502            )?]);
3503        }
3504
3505        // Updates must honor the same reserved owner as initial registration;
3506        // otherwise an empty HELLO could acquire the claim after admission.
3507        let mut conflicts = self
3508            .capability_evaluator
3509            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3510        if let Some(conflict) = conflicts.first() {
3511            let message = format!(
3512                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3513                conflict.capability, conflict.claimants[0], candidate.module_id
3514            );
3515            for conflict in &mut conflicts {
3516                conflict.source = DuplicateClaimSource::CatalogUpdate;
3517            }
3518            log_duplicate_claim_events(conflicts);
3519            return Ok(vec![control_error_frame(
3520                &frame,
3521                "reserved_capability",
3522                message,
3523            )?]);
3524        }
3525
3526        let updated = self
3527            .registry
3528            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3529            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3530        if updated.is_none() {
3531            return Ok(vec![control_error_frame(
3532                &frame,
3533                "not_registered",
3534                "catalog.update requires an active module registration owned by this connection",
3535            )?]);
3536        }
3537        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3538            log_duplicate_claim_events(
3539                self.capability_evaluator
3540                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3541            );
3542        }
3543        if capability_census_trigger(
3544            registration.manifest.capabilities.as_ref(),
3545            updated
3546                .as_ref()
3547                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3548        ) {
3549            self.enforce_capability_denies();
3550        }
3551        self.refresh_capability_requirements();
3552
3553        let response = ModuleControlResponseToModule::CatalogUpdate {};
3554        control_response_body_frame(
3555            &frame,
3556            &response,
3557            "ModuleControlResponseToModule::CatalogUpdate",
3558        )
3559        .map(|frame| vec![frame])
3560    }
3561
3562    pub(crate) fn with_activity_ledger_config(
3563        mut self,
3564        run_dir: &Path,
3565        incarnation: String,
3566        config: crate::activity::ActivityConfig,
3567    ) -> Self {
3568        self.activity = Some(crate::activity::ActivityLedger::open_config(
3569            run_dir.join("activity.jsonl"),
3570            incarnation,
3571            config,
3572        ));
3573        self.supervisor.set_activity(self.activity.clone());
3574        self
3575    }
3576
3577    pub(crate) async fn configure_activity(
3578        &self,
3579        config: crate::activity::ActivityConfig,
3580    ) -> Result<(), &'static str> {
3581        match &self.activity {
3582            Some(ledger) => ledger.configure(config).await,
3583            None => Ok(()),
3584        }
3585    }
3586
3587    fn cancel_activity_subscription(
3588        &self,
3589        ctx: &RouteCtx,
3590        frame: &Frame,
3591    ) -> Result<bool, RouterError> {
3592        let task = self
3593            .activity_streams
3594            .lock()
3595            .unwrap_or_else(|p| p.into_inner())
3596            .remove(&(ctx.connection_id, frame.header.corr));
3597        let Some(task) = task else {
3598            return Ok(false);
3599        };
3600        task.abort();
3601        let end = Frame::build_with_version(
3602            response_version(frame),
3603            FrameType::StreamEnd,
3604            control_flags(),
3605            0,
3606            0,
3607            frame.header.corr,
3608            Vec::new(),
3609        )
3610        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3611        let sink = ctx.egress.clone();
3612        tokio::spawn(async move {
3613            let _ = sink.send(end).await;
3614        });
3615        Ok(true)
3616    }
3617
3618    async fn handle_activity_read(
3619        &self,
3620        ctx: &RouteCtx,
3621        frame: Frame,
3622    ) -> Result<Vec<Frame>, RouterError> {
3623        use subc_control::{
3624            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3625            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3626        };
3627        #[derive(Deserialize)]
3628        struct Identity {
3629            #[serde(default)]
3630            consumer_identity: Option<ConsumerIdentity>,
3631        }
3632        // A registered module connection already identifies its caller. Ignore
3633        // consumer_identity there so a module cannot borrow a mirror's identity.
3634        // Only separate consumer connections use the Direct/nonce trust boundary.
3635        let principal = match self
3636            .forwarding
3637            .module_id_for_connection(ctx.connection_id)
3638            .map_err(RouterError::Forwarding)?
3639        {
3640            Some(module_id) => Ok(Principal::Reserved { module_id }),
3641            None => {
3642                let identity: Identity = match serde_json::from_slice(&frame.body) {
3643                    Ok(identity) => identity,
3644                    Err(_) => {
3645                        return Ok(vec![control_error_frame(
3646                            &frame,
3647                            ACTIVITY_MALFORMED,
3648                            "malformed activity read",
3649                        )?])
3650                    }
3651                };
3652                self.route_open_principal(&frame, identity.consumer_identity)?
3653            }
3654        };
3655        let admitted = matches!(principal, Ok(Principal::Direct))
3656            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3657        if !admitted {
3658            return Ok(vec![control_error_frame(
3659                &frame,
3660                ACTIVITY_READ_FORBIDDEN,
3661                "activity reads require an operator or trusted mirror principal",
3662            )?]);
3663        }
3664        let Some(ledger) = &self.activity else {
3665            return Ok(vec![control_error_frame(
3666                &frame,
3667                ACTIVITY_UNAVAILABLE,
3668                "activity ledger unavailable",
3669            )?]);
3670        };
3671        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3672            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3673        {
3674            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3675                Ok(request) => ledger.query(request).await,
3676                Err(_) => Err(ACTIVITY_MALFORMED),
3677            };
3678            return match result {
3679                Ok(reply) => Ok(vec![control_response_body_frame(
3680                    &frame,
3681                    &reply,
3682                    ops::ACTIVITY_QUERY,
3683                )?]),
3684                Err(code) => Ok(vec![control_error_frame(
3685                    &frame,
3686                    code,
3687                    "activity query refused",
3688                )?]),
3689            };
3690        }
3691        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3692            Ok(request) => request,
3693            Err(_) => {
3694                return Ok(vec![control_error_frame(
3695                    &frame,
3696                    ACTIVITY_MALFORMED,
3697                    "malformed activity subscription",
3698                )?])
3699            }
3700        };
3701        let mut subscription = match ledger.subscribe(request.since).await {
3702            Ok(subscription) => subscription,
3703            Err(code) => {
3704                return Ok(vec![control_error_frame(
3705                    &frame,
3706                    code,
3707                    "activity subscription refused",
3708                )?])
3709            }
3710        };
3711        let key = (ctx.connection_id, frame.header.corr);
3712        let sink = ctx.egress.clone();
3713        let mut streams = self
3714            .activity_streams
3715            .lock()
3716            .unwrap_or_else(|p| p.into_inner());
3717        streams.retain(|_, task| !task.is_finished());
3718        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3719            return Ok(vec![control_error_frame(
3720                &frame,
3721                "duplicate_subscription",
3722                "activity subscription correlation id already in use",
3723            )?]);
3724        }
3725        let task = tokio::spawn(async move {
3726            loop {
3727                let entry = if let Some(entry) = subscription.replay.pop_front() {
3728                    Some(entry)
3729                } else {
3730                    subscription.receiver.recv().await
3731                };
3732                let Some(entry) = entry else {
3733                    break;
3734                };
3735                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3736                    return;
3737                };
3738                let Ok(next) = Frame::build_with_version(
3739                    response_version(&frame),
3740                    FrameType::StreamData,
3741                    control_flags(),
3742                    0,
3743                    0,
3744                    frame.header.corr,
3745                    body,
3746                ) else {
3747                    return;
3748                };
3749                if sink.send(next).await.is_err() {
3750                    return;
3751                }
3752            }
3753            // Dropping a full subscriber's sender drains its accepted entries
3754            // before the terminal error, preserving their order on the wire.
3755            if let Some(code) = subscription.terminal_code() {
3756                if let Ok(end) =
3757                    control_error_frame(&frame, code, "activity subscriber fell behind")
3758                {
3759                    let _ = sink.send(end).await;
3760                }
3761            }
3762        });
3763        streams.insert(key, task.abort_handle());
3764        Ok(Vec::new())
3765    }
3766
3767    async fn handle_activity_record(
3768        &self,
3769        ctx: &RouteCtx,
3770        frame: Frame,
3771    ) -> Result<Vec<Frame>, RouterError> {
3772        use subc_control::{
3773            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3774            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3775        };
3776        let result = async {
3777            let module = self
3778                .forwarding
3779                .module_id_for_connection(ctx.connection_id)
3780                .ok()
3781                .flatten()
3782                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3783            let request: ActivityRecordRequest =
3784                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3785            crate::activity::validate(&module, &request)?;
3786            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3787            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3788            let at_ms = std::time::SystemTime::now()
3789                .duration_since(std::time::UNIX_EPOCH)
3790                .unwrap_or_default()
3791                .as_millis() as u64;
3792            ledger.record(module, request, actor, at_ms).await
3793        }
3794        .await;
3795        match result {
3796            Ok(entry) => Ok(vec![control_response_body_frame(
3797                &frame,
3798                &ActivityRecordResponse { seq: entry.seq },
3799                "activity.record",
3800            )?]),
3801            Err(code) => Ok(vec![control_error_frame(
3802                &frame,
3803                code,
3804                "activity recording refused",
3805            )?]),
3806        }
3807    }
3808
3809    fn activity_actor(
3810        &self,
3811        connection: ConnectionId,
3812        module: &str,
3813        on: Option<&subc_control::ActivityOn>,
3814    ) -> Result<subc_control::ActivityActor, &'static str> {
3815        use crate::forwarding::{DataRoute, DataRouteState};
3816        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3817        let principal = |id: &str| Principal::Reserved {
3818            module_id: id.to_owned(),
3819        };
3820        let mut actor = ActivityActor {
3821            principal: principal(module),
3822            scope_owner: None,
3823            agent_id: None,
3824            run_id: None,
3825            flow_id: None,
3826            project_root: None,
3827            workdir: None,
3828            project_id: None,
3829        };
3830        let scope = match on {
3831            None => return Ok(actor),
3832            Some(ActivityOn::Scope(scope)) => {
3833                let selector = subc_protocol::scope::ScopeSelector::new(
3834                    principal(module),
3835                    scope.scope_ref.clone(),
3836                    Some(scope.epoch),
3837                );
3838                self.scopes
3839                    .read()
3840                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3841                    .attribute(&principal(module), module, &selector, true)
3842                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3843            }
3844            Some(ActivityOn::Route(route)) => {
3845                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3846                    .forwarding
3847                    .lookup_data_route(connection, route.channel, route.epoch)
3848                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3849                else {
3850                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3851                };
3852                if binding.module_id != module {
3853                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3854                }
3855                actor.principal = binding.principal.clone();
3856                actor.project_root = binding
3857                    .project_root
3858                    .as_ref()
3859                    .map(|root| root.as_path().to_path_buf());
3860                let Some(bound) = &binding.scope else {
3861                    return Ok(actor);
3862                };
3863                let owner = principal(&bound.owner);
3864                let selector = subc_protocol::scope::ScopeSelector::new(
3865                    owner.clone(),
3866                    bound.scope_ref.clone(),
3867                    Some(bound.tag.scope_epoch),
3868                );
3869                let admitted = self
3870                    .scopes
3871                    .read()
3872                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3873                    .attribute(&owner, module, &selector, true)
3874                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3875                if admitted.tag != bound.tag {
3876                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3877                }
3878                admitted
3879            }
3880        };
3881        actor.scope_owner = Some(scope.stamp.owner);
3882        actor.agent_id = scope.stamp.attributes.agent_id;
3883        actor.run_id = scope.stamp.attributes.run_id;
3884        actor.flow_id = scope.stamp.attributes.flow_id;
3885        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3886        actor.project_id = scope.stamp.project_id;
3887        Ok(actor)
3888    }
3889
3890    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3891        self.refresh_capability_requirements();
3892        // A bare connection count is ambiguous between many clients holding a
3893        // route each and one client accumulating hundreds, so publish the
3894        // concentration alongside it. Route state is best-effort here: a
3895        // diagnostic endpoint must still answer if the forwarding lock is
3896        // contended.
3897        let mut counters = self.counters.snapshot();
3898        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3899            obj.insert(
3900                "activity_unparseable_lines".into(),
3901                ledger.read_errors().into(),
3902            );
3903            obj.insert(
3904                "activity_lifecycle_append_failures".into(),
3905                ledger.lifecycle_failures().into(),
3906            );
3907            obj.insert(
3908                "activity_started_digest_skips".into(),
3909                ledger.lifecycle_digest_skips().into(),
3910            );
3911        }
3912        if let (Ok((connections_with_routes, max)), Some(obj)) = (
3913            self.forwarding.client_route_concentration(),
3914            counters.as_object_mut(),
3915        ) {
3916            obj.insert(
3917                "client_connections_with_routes".into(),
3918                connections_with_routes.into(),
3919            );
3920            obj.insert("max_routes_on_one_connection".into(), max.into());
3921        }
3922        // A module that is being fast-refused and a module that is fine look
3923        // identical from a client that retries and succeeds, so name the open
3924        // breakers here. This rides the existing free-form counters object
3925        // rather than a new wire field, so no sibling that deserializes
3926        // `ServerDescribe` has to be rebuilt to keep reading it.
3927        if let (Some(open_breakers), Some(obj)) = (
3928            self.route_bind_breakers.open_snapshot(),
3929            counters.as_object_mut(),
3930        ) {
3931            obj.insert("route_bind_breakers_open".into(), open_breakers);
3932        }
3933        let response = ClientControlResponse::ServerDescribe {
3934            protocol_ver: PROTOCOL_VERSION,
3935            subc_ops: subc_ops(),
3936            capabilities: self.describe_capabilities(),
3937            connected_clients: self.connected_clients.count(),
3938            counters: Some(counters),
3939            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3940            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3941            capability_requirements: self.capability_requirement_statuses(),
3942            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3943            config_path: self
3944                .rescan
3945                .as_ref()
3946                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3947        };
3948        Ok(vec![control_response_body_frame(
3949            &frame,
3950            &response,
3951            "ClientControlResponse::ServerDescribe",
3952        )?])
3953    }
3954
3955    fn handle_catalog_list(
3956        &self,
3957        frame: Frame,
3958        module_id: Option<String>,
3959    ) -> Result<Vec<Frame>, RouterError> {
3960        let (generation, modules) = self
3961            .registry
3962            .list_modules_with_nonce_verification()
3963            .map_err(|err| {
3964                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3965            })?;
3966        let entries = modules
3967            .into_iter()
3968            .filter(|(registration, _)| {
3969                module_id
3970                    .as_deref()
3971                    .map(|wanted| registration.manifest.module_id == wanted)
3972                    .unwrap_or(true)
3973            })
3974            .map(|(registration, nonce_verified)| {
3975                let not_ready = self.not_ready_reason(&registration);
3976                let roles = registration.manifest.provides;
3977                CatalogEntry::new(
3978                    registration.manifest.module_id,
3979                    roles,
3980                    registration.control_ops,
3981                )
3982                .with_ready(not_ready.is_none())
3983                .with_not_ready(not_ready)
3984                .with_module_version(Some(registration.manifest.module_version))
3985                .with_capabilities(registration.manifest.capabilities)
3986                .with_self_signals(registration.manifest.self_signals)
3987                .with_events(registration.manifest.events)
3988                .with_operation_delivery(registration.manifest.operation_delivery)
3989                .with_rich_ui(registration.manifest.rich_ui)
3990                .with_nonce_verified(nonce_verified)
3991            })
3992            .collect();
3993        let response = ClientControlResponse::CatalogList {
3994            generation,
3995            modules: entries,
3996            subc_ops: subc_ops(),
3997        };
3998        Ok(vec![control_response_body_frame(
3999            &frame,
4000            &response,
4001            "ClientControlResponse::CatalogList",
4002        )?])
4003    }
4004
4005    fn route_open_principal(
4006        &self,
4007        frame: &Frame,
4008        consumer_identity: Option<ConsumerIdentity>,
4009    ) -> Result<Result<Principal, Frame>, RouterError> {
4010        let Some(consumer_identity) = consumer_identity else {
4011            return Ok(Ok(Principal::Direct));
4012        };
4013
4014        if self.supervisor.spawned_consumer_authorized(
4015            &consumer_identity.module_id,
4016            &consumer_identity.launch_nonce,
4017        ) {
4018            return Ok(Ok(Principal::Reserved {
4019                module_id: consumer_identity.module_id,
4020            }));
4021        }
4022
4023        Ok(Err(control_error_frame(
4024            frame,
4025            "bad_consumer_identity",
4026            format!(
4027                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4028                consumer_identity.module_id
4029            ),
4030        )?))
4031    }
4032
4033    /// Ordinary `route.open` refusals go through here; admission and breaker
4034    /// refusals log separately with their capacity or breaker state. The daemon can
4035    /// attest which code it sent: without the event, a client's "the daemon
4036    /// refused me" and the daemon's own view could only be reconciled by
4037    /// argument. Malformed input (`invalid_project_root`) does not come here;
4038    /// rejecting a request that was never a valid open is not a refusal of one.
4039    fn route_open_refusal_frame(
4040        &self,
4041        ctx: &RouteCtx,
4042        frame: &Frame,
4043        module_id: &str,
4044        reason: &'static str,
4045        code: &'static str,
4046        message: impl Into<String>,
4047    ) -> Result<Frame, RouterError> {
4048        self.observe_route_open_refusal(ctx, module_id, reason, code);
4049        control_error_frame(frame, code, message.into())
4050    }
4051
4052    /// Refuse a `route.open` because the target module's bind-relay breaker is
4053    /// open, without attempting the relay.
4054    ///
4055    /// The wire code is `module_timeout`, which is the truth (the module has
4056    /// not been answering binds) and which both SDKs already classify as
4057    /// retryable with capped backoff. Reusing it is what keeps this change out
4058    /// of both SDKs; the daemon-side distinction lives in the counter key
4059    /// instead.
4060    ///
4061    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4062    /// While a breaker is open this fires on every open to that module, and the
4063    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4064    /// produced 261 lines about a single module inside 3000 lines of daemon
4065    /// log. The rare transitions are logged at warn/info instead and the volume
4066    /// is carried by the counter, so the evidence survives without the flood.
4067    /// The debug line keeps a per-refusal record reachable for whoever turns
4068    /// the level up.
4069    fn route_open_breaker_refusal_frame(
4070        &self,
4071        ctx: &RouteCtx,
4072        frame: &Frame,
4073        module_id: &str,
4074        consecutive_timeouts: u32,
4075        retry_in: Duration,
4076        probe_in_flight: bool,
4077    ) -> Result<Frame, RouterError> {
4078        self.counters
4079            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4080        debug!(
4081            target: "control",
4082            code = "module_timeout",
4083            module_id = ?module_id,
4084            connection_id = ctx.connection_id.get(),
4085            consecutive_timeouts,
4086            retry_in_ms = retry_in.as_millis() as u64,
4087            probe_in_flight,
4088            "route.open refused by open bind-relay breaker"
4089        );
4090        // Say what a caller can act on. An open bind-relay breaker means the
4091        // module timed out accepting several new routes in a row. The module
4092        // is still running and its established routes keep working; only new
4093        // route.open requests are refused until the cooldown ends and one
4094        // test route (the probe) gets through. A message that only counts
4095        // failed relays reads as "the module is down" to a worker that sees it.
4096        let detail = if probe_in_flight {
4097            "one test route is already being tried; retry once it settles".to_string()
4098        } else {
4099            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4100        };
4101        control_error_frame(
4102            frame,
4103            "module_timeout",
4104            format!(
4105                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4106                 timed out in a row); {detail}; its established routes are unaffected"
4107            ),
4108        )
4109    }
4110
4111    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4112    /// requester's bytes (an unknown target is whatever the client sent) and
4113    /// is Debug-formatted so control characters land in the log escaped
4114    /// rather than as terminal sequences for whoever tails it.
4115    ///
4116    /// `reason` names the check that refused, because one wire code has
4117    /// several senders: after a module registers, `target_unavailable` can
4118    /// come from a missing role, an inactive registration, a supervisor that
4119    /// has not marked the process live, a missing forwarding connection, or a
4120    /// failed relay, and a log that records only the code cannot say which of
4121    /// them fired. It is a static, daemon-chosen label per branch, so it is
4122    /// safe to print plainly and stays a closed set.
4123    fn observe_route_open_refusal(
4124        &self,
4125        ctx: &RouteCtx,
4126        module_id: &str,
4127        reason: &'static str,
4128        code: &'static str,
4129    ) {
4130        self.counters.increment_route_open_refused(code);
4131        info!(
4132            target: "control",
4133            code,
4134            reason,
4135            module_id = ?module_id,
4136            connection_id = ctx.connection_id.get(),
4137            "route.open refused"
4138        );
4139        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4140            self.route_outages.record_not_serving(module_id, reason);
4141        }
4142    }
4143
4144    /// Record an ACCEPTED route.open.
4145    ///
4146    /// Refusals have been logged and counted since the attestation work; accepts
4147    /// were invisible, so the daemon knew every principal it stamped and wrote
4148    /// none of them down. The party that attests the identity was the only party
4149    /// not recording it, which left a credential vault unable to name the sender
4150    /// of a call that reached it (claustrum #43) and left the launch-nonce
4151    /// concurrency question unanswerable from the outside.
4152    ///
4153    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4154    /// `code`/`module_id`/`connection_id` returns both directions of the same
4155    /// decision rather than two shapes a reader has to join by hand.
4156    ///
4157    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4158    /// and the difference carries information rather than being an
4159    /// inconsistency. This line is only reachable after a successful bind to a
4160    /// REGISTERED module, so the value has already passed HELLO validation
4161    /// including the path-hazard refusal and cannot contain control bytes. A
4162    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4163    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4164    ///
4165    /// Bare is also what every other daemon line already emits (`module
4166    /// registered`, `configured module supervised`). Shipping `?module_id` here
4167    /// made this instrument the only one in the file whose ids did not answer
4168    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4169    /// line whose whole purpose is being grepped beside its sibling.
4170    ///
4171    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4172    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4173    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4174    /// one are recorded identically. A test written against that harness passes
4175    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4176    /// green assertion that cannot fail. The same limit applies to the escaping
4177    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4178    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4179    /// Fencing either needs the real formatter, not the capture layer.
4180    ///
4181    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4182    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4183    /// to record. The ephemeral port would decay within minutes and answer only a
4184    /// live question. The identity question is instead answered by counting
4185    /// distinct live connections presenting one module's `consumer_identity` --
4186    /// "is anyone else holding this secret" rather than "is this the right
4187    /// process".
4188    fn observe_route_open_accept(
4189        &self,
4190        ctx: &RouteCtx,
4191        module_id: &str,
4192        principal: &str,
4193        scope: &str,
4194    ) {
4195        self.route_outages.record_accepted(module_id);
4196        self.counters.increment_route_open_accepted(principal);
4197        info!(
4198            target: "control",
4199            principal,
4200            module_id,
4201            connection_id = ctx.connection_id.get(),
4202            scope = %scope,
4203            "route.open accepted"
4204        );
4205    }
4206
4207    fn supervised_absent_route_open_refusal_frame(
4208        &self,
4209        ctx: &RouteCtx,
4210        frame: &Frame,
4211        module_id: &str,
4212        code: &'static str,
4213        status: &crate::supervise::ModuleStatus,
4214    ) -> Result<Frame, RouterError> {
4215        self.counters.increment_route_open_refused(code);
4216        info!(
4217            target: "control",
4218            code,
4219            reason = "supervised_not_registered",
4220            module_id = ?module_id,
4221            connection_id = ctx.connection_id.get(),
4222            state = %status.state,
4223            enabled = status.enabled,
4224            live = status.live,
4225            "route.open refused"
4226        );
4227        // A supervised module whose process has not registered is not
4228        // serving, whatever the reason; the supervisor knows this id, so it is
4229        // safe to track.
4230        self.route_outages
4231            .record_not_serving(module_id, "supervised_not_registered");
4232        let message = if let Some(pid) = status.waiting_for_previous_pid {
4233            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4234                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4235            })
4236        } else {
4237            format!(
4238                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4239                status.state, status.enabled, status.live
4240            )
4241        };
4242        control_error_frame(frame, code, message)
4243    }
4244
4245    async fn handle_route_open(
4246        &self,
4247        ctx: &RouteCtx,
4248        frame: Frame,
4249        request: RouteOpenRequest,
4250        timings: &mut RouteOpenTimings,
4251    ) -> Result<Vec<Frame>, RouterError> {
4252        let RouteOpenRequest {
4253            target,
4254            mut identity,
4255            consumer_identity,
4256            consumer_capabilities,
4257            role_versions,
4258            admission_facts,
4259            scope,
4260        } = request;
4261        let target_module_id = target_module_id(&target).to_string();
4262        timings.module_id.clone_from(&target_module_id);
4263        if self
4264            .registry
4265            .get_module_by_connection(ctx.connection_id)
4266            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4267            .is_some()
4268        {
4269            return Ok(vec![control_error_frame(
4270                &frame,
4271                "invalid_request",
4272                "module connections cannot open client routes",
4273            )?]);
4274        }
4275        debug!(
4276            connection_id = ctx.connection_id.get(),
4277            corr = frame.header.corr,
4278            module_id = %target_module_id,
4279            "handling route.open"
4280        );
4281
4282        // A malformed declaration is refused first, before anything about the
4283        // target is looked up: the same body would be refused against any
4284        // module, so the caller learns nothing by retrying or waiting. An empty
4285        // map declares nothing and travels as no field at all, so a provider
4286        // only ever sees a missing field or a non-empty one.
4287        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4288        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4289            self.observe_route_open_refusal(
4290                ctx,
4291                &target_module_id,
4292                "invalid_role_versions",
4293                error_codes::INVALID_REQUEST,
4294            );
4295            return Ok(vec![control_error_body_frame(
4296                &frame,
4297                ErrorBody {
4298                    code: error_codes::INVALID_REQUEST.to_string(),
4299                    message: error.to_string(),
4300                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4301                },
4302            )?]);
4303        }
4304
4305        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4306            if status.waiting_for_previous_pid.is_some() {
4307                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4308                    ctx,
4309                    &frame,
4310                    &target_module_id,
4311                    "module_warming",
4312                    &status,
4313                )?]);
4314            }
4315        }
4316
4317        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4318        // opposite. Below, a caller learns whether a module is unregistered,
4319        // supervised-but-down (with state/enabled/live), or registered without the
4320        // requested role. Elsewhere that is an enumeration leak: a probe learning
4321        // the shape of a fleet it cannot otherwise see.
4322        //
4323        // It is not one here, and the reason is the ACCESS MODEL rather than
4324        // anything about these errors. Reaching route.open requires the
4325        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4326        // connection file, so any caller who completes it already runs as this
4327        // user -- and can read subc.jsonc for the module list and `ck module
4328        // status` for live state. The reply discloses nothing the caller cannot
4329        // read more easily from disk, while the precision is load-bearing:
4330        // `unknown_module` is retryable and a missing role is not.
4331        //
4332        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4333        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4334        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4335        let Some(registration) = self
4336            .registry
4337            .get_module(&target_module_id)
4338            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4339        else {
4340            if let Some((status, warming)) =
4341                self.supervisor_status(&target_module_id, frame.header.corr)?
4342            {
4343                // BEFORE the two availability codes below, because for a module
4344                // that speaks no subc wire both of them are false comfort: they
4345                // say "not right now" and are retried, and this module will
4346                // never register no matter how long the caller waits. The
4347                // absence here is the declaration being honoured, not a module
4348                // that is late.
4349                if status.protocol == ModuleProtocol::None {
4350                    return Ok(vec![self.route_open_refusal_frame(
4351                        ctx,
4352                        &frame,
4353                        &target_module_id,
4354                        "protocol_none",
4355                        error_codes::MODULE_NO_PROTOCOL,
4356                        format!(
4357                            "module_id '{target_module_id}' is declared protocol: none; \
4358                             it speaks no subc wire and serves no routes"
4359                        ),
4360                    )?]);
4361                }
4362                let code = if warming {
4363                    "module_warming"
4364                } else {
4365                    "target_unavailable"
4366                };
4367                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4368                    ctx,
4369                    &frame,
4370                    &target_module_id,
4371                    code,
4372                    &status,
4373                )?]);
4374            }
4375            if let Some(removed_ago_ms) =
4376                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4377            {
4378                return Ok(vec![self.route_open_refusal_frame(
4379                    ctx,
4380                    &frame,
4381                    &target_module_id,
4382                    "removed",
4383                    error_codes::MODULE_REMOVED,
4384                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4385                )?]);
4386            }
4387            return Ok(vec![self.route_open_refusal_frame(
4388                ctx,
4389                &frame,
4390                &target_module_id,
4391                "not_registered",
4392                error_codes::UNKNOWN_MODULE,
4393                format!("module_id '{target_module_id}' is not registered"),
4394            )?]);
4395        };
4396
4397        // Best-effort only: registry readiness and forwarding reservation use
4398        // different locks, so a module can flip readiness between this read and
4399        // the relay. Modules must still tolerate an `on_bind` while not ready.
4400        if !registration.ready {
4401            self.counters
4402                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4403            info!(
4404                target: "control",
4405                code = error_codes::MODULE_WARMING,
4406                module_id = ?target_module_id,
4407                connection_id = ctx.connection_id.get(),
4408                reason = "declared_not_ready",
4409                "route.open refused"
4410            );
4411            // The module is registered but says it cannot take work, which is
4412            // an outage from the caller's side even though its process is up.
4413            self.route_outages
4414                .record_not_serving(&target_module_id, "declared_not_ready");
4415            return Ok(vec![control_error_body_frame(
4416                &frame,
4417                ErrorBody {
4418                    code: error_codes::MODULE_WARMING.to_string(),
4419                    message: format!(
4420                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4421                    ),
4422                    detail: Some(serde_json::json!({
4423                        "reason": "declared_not_ready"
4424                    })),
4425                },
4426            )?]);
4427        }
4428
4429        // Effective readiness, second half: a module that declares a capability
4430        // `need: required` is not routable while that capability has no
4431        // registered provider. It is enforced HERE, as a retryable routing
4432        // refusal, and deliberately not as spawn ordering or a boot block. The
4433        // module is still started and registered and can make its own calls;
4434        // spawn ordering is a promise that cannot be kept once a provider
4435        // crashes at runtime, and refusing to boot would stop the whole
4436        // machine, including the tools needed to fix its configuration.
4437        //
4438        // "Provided" is the evaluator's verdict, which counts a provider as
4439        // soon as it has REGISTERED, not once it is ready. Two modules that
4440        // require each other's capabilities are therefore both routable once
4441        // both register; counting readiness instead would deadlock them.
4442        //
4443        // Only new opens are refused. Routes already bound when a provider
4444        // goes away stay bound: nothing here tears them down, and the module
4445        // answers them as it can. Like the readiness read above this is
4446        // best-effort against a provider registering or leaving concurrently.
4447        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
4448            self.counters
4449                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4450            info!(
4451                target: "control",
4452                code = error_codes::MODULE_WARMING,
4453                module_id = ?target_module_id,
4454                connection_id = ctx.connection_id.get(),
4455                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4456                capability = %capability,
4457                "route.open refused"
4458            );
4459            return Ok(vec![control_error_body_frame(
4460                &frame,
4461                ErrorBody {
4462                    code: error_codes::MODULE_WARMING.to_string(),
4463                    message: format!(
4464                        "module_id '{target_module_id}' requires capability '{capability}', \
4465                         which no registered module provides; retry"
4466                    ),
4467                    detail: Some(serde_json::json!({
4468                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4469                        "capability": capability,
4470                    })),
4471                },
4472            )?]);
4473        }
4474
4475        if !target_has_required_role(&target, &registration.manifest.provides) {
4476            return Ok(vec![self.route_open_refusal_frame(
4477                ctx,
4478                &frame,
4479                &target_module_id,
4480                "role_not_provided",
4481                "target_unavailable",
4482                format!("module_id '{target_module_id}' does not provide the requested target"),
4483            )?]);
4484        }
4485
4486        if registration.state != ChannelState::Active {
4487            return Ok(vec![self.route_open_refusal_frame(
4488                ctx,
4489                &frame,
4490                &target_module_id,
4491                "registration_not_active",
4492                "target_unavailable",
4493                format!("module_id '{target_module_id}' is not active"),
4494            )?]);
4495        }
4496
4497        if self
4498            .forwarding
4499            .module_is_draining(&target_module_id)
4500            .map_err(RouterError::Forwarding)?
4501        {
4502            return Ok(vec![self.route_open_refusal_frame(
4503                ctx,
4504                &frame,
4505                &target_module_id,
4506                "reloading",
4507                "module_reloading",
4508                format!("module_id '{target_module_id}' is reloading"),
4509            )?]);
4510        }
4511
4512        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4513            process_liveness.process_live(&target_module_id) == Some(false)
4514        }) {
4515            // A module the supervisor is restarting or reloading can still hold
4516            // a registration: the old process before its connection closes, or
4517            // a new one that registered while the supervisor was draining. The
4518            // forwarding table does not see that as draining, but the consumer
4519            // should still be told to retry soon, exactly as for the drain
4520            // above, rather than that the target is unavailable.
4521            if process_liveness.process_replacing(&target_module_id) {
4522                return Ok(vec![self.route_open_refusal_frame(
4523                    ctx,
4524                    &frame,
4525                    &target_module_id,
4526                    "reloading",
4527                    "module_reloading",
4528                    format!("module_id '{target_module_id}' is reloading"),
4529                )?]);
4530            }
4531            return Ok(vec![self.route_open_refusal_frame(
4532                ctx,
4533                &frame,
4534                &target_module_id,
4535                "supervisor_not_live",
4536                "target_unavailable",
4537                format!("module_id '{target_module_id}' is not live"),
4538            )?]);
4539        }
4540
4541        if !self
4542            .forwarding
4543            .has_live_module_connection(&target_module_id)
4544            .map_err(RouterError::Forwarding)?
4545        {
4546            return Ok(vec![self.route_open_refusal_frame(
4547                ctx,
4548                &frame,
4549                &target_module_id,
4550                "no_forwarding_connection",
4551                "target_unavailable",
4552                format!("module_id '{target_module_id}' has no live forwarding connection"),
4553            )?]);
4554        }
4555
4556        if let Some(error) =
4557            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4558        {
4559            self.observe_route_open_refusal(
4560                ctx,
4561                &target_module_id,
4562                "op_not_allowed",
4563                "op_not_allowed",
4564            );
4565            return Ok(vec![error]);
4566        }
4567
4568        let principal = match self.route_open_principal(&frame, consumer_identity)? {
4569            Ok(principal) => principal,
4570            Err(error) => {
4571                self.observe_route_open_refusal(
4572                    ctx,
4573                    &target_module_id,
4574                    "bad_consumer_identity",
4575                    "bad_consumer_identity",
4576                );
4577                return Ok(vec![error]);
4578            }
4579        };
4580        timings.principal = principal_label(&principal);
4581
4582        // This is attested, control-plane policy for supervised module origins.
4583        // Keep it before route reservation and out of the opaque forwarding hot
4584        // path: data frames must never acquire a per-frame capability check.
4585        if let Principal::Reserved {
4586            module_id: opening_module_id,
4587        } = &principal
4588        {
4589            if let Some(opening_registration) = self
4590                .registry
4591                .get_module(opening_module_id)
4592                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4593            {
4594                if let Some(capability) =
4595                    denied_capability(&opening_registration.manifest, &registration.manifest)
4596                {
4597                    warn!(
4598                        opening_module_id,
4599                        target_module_id,
4600                        capability,
4601                        "refusing route.open because an attested capability deny edge matches"
4602                    );
4603                    return Ok(vec![self.route_open_refusal_frame(
4604                        ctx,
4605                        &frame,
4606                        &target_module_id,
4607                        "capability_deny_edge",
4608                        "capability_forbidden",
4609                        format!(
4610                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4611                        ),
4612                    )?]);
4613                }
4614            }
4615        }
4616
4617        if admission_facts.is_some() {
4618            let carrier_matches = matches!(
4619                &principal,
4620                Principal::Reserved { module_id }
4621                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4622            );
4623            if !carrier_matches {
4624                return Ok(vec![self.route_open_refusal_frame(
4625                    ctx,
4626                    &frame,
4627                    &target_module_id,
4628                    "admission_facts_carrier_not_permitted",
4629                    "admission_facts_not_permitted",
4630                    "admission facts may only be carried by the configured reserved module",
4631                )?]);
4632            }
4633
4634            let target_allowed = self
4635                .admission_facts_targets
4636                .as_ref()
4637                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4638            if !target_allowed {
4639                return Ok(vec![self.route_open_refusal_frame(
4640                    ctx,
4641                    &frame,
4642                    &target_module_id,
4643                    "admission_facts_target_not_listed",
4644                    "admission_facts_target_not_allowed",
4645                    format!(
4646                        "admission facts are not permitted for target module_id '{target_module_id}'"
4647                    ),
4648                )?]);
4649            }
4650
4651            // Keep the value opaque to subc. The downstream admission validator owns
4652            // schema and semantic checks; this daemon only enforces carrier authority
4653            // and the configured destination allowlist.
4654        }
4655
4656        // Scope admission, on the attested principal above and never on the
4657        // request body. The tag read here travels with the pending bind and is
4658        // compared with the published one at commit, so a sync between here
4659        // and the module's ack refuses the open instead of binding a stamp
4660        // that is no longer true.
4661        let (bound_scope, mut scope_stamp) = match &scope {
4662            None => (None, None),
4663            Some(selector) => {
4664                let owner_configured = match &selector.owner {
4665                    // A reserved owner counts as configured from before its
4666                    // process is spawned (see `SupervisorHandle::is_configured`).
4667                    // So an owner that has not synced its scopes yet is refused
4668                    // as retryable (`scope_not_synced`), not as one that will
4669                    // never sync.
4670                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4671                    _ => false,
4672                };
4673                let mut scopes = self
4674                    .scopes
4675                    .write()
4676                    .unwrap_or_else(|poisoned| poisoned.into_inner());
4677                let admitted = scopes
4678                    .admit(&principal, &target_module_id, selector, owner_configured)
4679                    .and_then(|admission| {
4680                        crate::scopes::check_target_flow_support(
4681                            &admission.stamp,
4682                            &target_module_id,
4683                            registration.manifest.capabilities.as_ref(),
4684                        )?;
4685                        crate::scopes::check_target_agent_run_support(
4686                            &admission.stamp,
4687                            &target_module_id,
4688                            registration.manifest.capabilities.as_ref(),
4689                        )?;
4690                        let bound = scopes.reserve_bind(
4691                            &admission,
4692                            &principal,
4693                            ctx.connection_id,
4694                            selector.hold_presence,
4695                        );
4696                        Ok((admission, bound))
4697                    });
4698                match admitted {
4699                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4700                    Err(refusal) => {
4701                        return Ok(vec![self.route_open_refusal_frame(
4702                            ctx,
4703                            &frame,
4704                            &target_module_id,
4705                            refusal.code,
4706                            refusal.code,
4707                            refusal.message,
4708                        )?]);
4709                    }
4710                }
4711            }
4712        };
4713
4714        // Bind admits a root that no longer exists on disk, because refusing here
4715        // closes the only exit from a paused run: cancel needs a bound route, and a
4716        // renamed or reclaimed directory makes that route unopenable forever. The
4717        // run itself is intact and still addressable by its recorded identity.
4718        //
4719        // This does NOT relax the rule the strict constructor protects. That rule is
4720        // that no root is ever aliased into NEW durable state -- a missing component
4721        // can reappear as a symlink elsewhere, which would move the identity and
4722        // split a session's history across two of them. The engine now refuses the
4723        // two operations that create such state (send and import) at admission,
4724        // which is a narrower way to hold the same invariant: reads and terminations
4725        // are admitted, writes are not. That refusal had to ship before this line
4726        // changed, or there is an interval where a send commits under a provisional
4727        // identity -- the exact failure the original policy existed to prevent.
4728        //
4729        // Resolution follows realpath rather than lexical cleanup: the longest
4730        // existing ancestor is canonicalized and the missing tail re-appended, so a
4731        // live root is unchanged and a vanished leaf keeps the identity it was
4732        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
4733        // same caller the moment the directory vanished, which strands the run more
4734        // quietly than refusing it.
4735        timings.enter(RouteOpenPhase::ProjectRoot);
4736        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4737            Ok(project_root) => project_root,
4738            Err(err) => {
4739                return Ok(vec![control_error_frame(
4740                    &frame,
4741                    "invalid_project_root",
4742                    err.to_string(),
4743                )?])
4744            }
4745        };
4746        timings.enter(RouteOpenPhase::Admission);
4747        identity.project_root = project_root.as_path().to_path_buf();
4748
4749        // Last gate before any relay work, and deliberately after the cheap
4750        // registry and availability checks above: those name a more precise
4751        // condition (unknown, removed, reloading) and a caller is better served
4752        // by the precise code than by this one.
4753        //
4754        // Everything below this point costs an egress permit, a reserved handle
4755        // pair and, if the module does not answer, the whole relay budget. The
4756        // reader no longer waits for that budget, so cap each target explicitly;
4757        // serial dispatch used to provide the accidental cap of one relay per
4758        // connection. Admission is a mutex-protected count and never waits.
4759        let _concurrency_guard = match self
4760            .route_bind_concurrency
4761            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4762        {
4763            Ok(guard) => guard,
4764            Err(in_flight) => {
4765                return Ok(vec![self.route_open_target_capacity_refusal(
4766                    ctx,
4767                    &frame,
4768                    &target_module_id,
4769                    in_flight,
4770                )?]);
4771            }
4772        };
4773
4774        // A module that has already burned the whole budget `threshold` times
4775        // in a row does not get to charge it again until a probe says it recovered.
4776        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4777            RouteBindAdmission::Admitted { guard, probe } => {
4778                if probe {
4779                    info!(
4780                        module_id = %target_module_id,
4781                        connection_id = ctx.connection_id.get(),
4782                        "route.bind breaker half-open: admitting one probe"
4783                    );
4784                }
4785                guard
4786            }
4787            RouteBindAdmission::Refused {
4788                consecutive_timeouts,
4789                retry_in,
4790                probe_in_flight,
4791            } => {
4792                return Ok(vec![self.route_open_breaker_refusal_frame(
4793                    ctx,
4794                    &frame,
4795                    &target_module_id,
4796                    consecutive_timeouts,
4797                    retry_in,
4798                    probe_in_flight,
4799                )?]);
4800            }
4801        };
4802
4803        // Resolve the per-module budget here so the wait matches the operator's
4804        // intent for this specific target. A per-module override in
4805        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
4806        // daemons) wins over the daemon-wide default.
4807        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4808        let relay_deadline = Instant::now() + route_bind_relay_timeout;
4809        timings.enter(RouteOpenPhase::EgressReservation);
4810        let pending = match self
4811            .forwarding
4812            .begin_route_bind_relay_for(
4813                ctx.connection_id,
4814                ctx.egress.clone(),
4815                response_version(&frame),
4816                frame.header.corr,
4817                &target_module_id,
4818                principal.clone(),
4819                bound_scope,
4820                Some(project_root),
4821                relay_deadline,
4822            )
4823            .await
4824        {
4825            Ok(pending) => pending,
4826            Err(err) => {
4827                return Ok(vec![self.route_open_refusal_frame(
4828                    ctx,
4829                    &frame,
4830                    &target_module_id,
4831                    "relay_reservation_failed",
4832                    forwarding_error_code(&err),
4833                    err.to_string(),
4834                )?])
4835            }
4836        };
4837        timings.enter(RouteOpenPhase::Admission);
4838        let crate::forwarding::PendingRouteBindRelay {
4839            endpoint,
4840            module_sink,
4841            negotiated_ver,
4842            client_channel,
4843            client_epoch,
4844            module_channel,
4845            module_epoch,
4846            corr: relay_corr,
4847            receiver,
4848        } = pending;
4849        let mut reservation =
4850            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4851
4852        // Reserving egress can wait while a module reconnects or a swap cuts
4853        // over. Check the relay's captured connection, not the earlier by-id
4854        // lookup: the original module's flow/run capabilities cannot authorize
4855        // its replacement. The captured sink stays bound to that connection.
4856        let relay_registration = self
4857            .registry
4858            .get_module_by_connection(endpoint.connection_id)
4859            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4860        let relay_capabilities = relay_registration
4861            .as_ref()
4862            .and_then(|registration| registration.manifest.capabilities.as_ref());
4863        if let Some(selector) = &scope {
4864            // Egress reservation may wait across a scope update. Recheck the
4865            // proof against the current record before sending the captured bind.
4866            if let Err(refusal) = self
4867                .scopes
4868                .read()
4869                .unwrap_or_else(|poisoned| poisoned.into_inner())
4870                .check_bind_proof(&principal, selector)
4871            {
4872                reservation.release_and_disarm();
4873                return Ok(vec![self.route_open_refusal_frame(
4874                    ctx,
4875                    &frame,
4876                    &target_module_id,
4877                    refusal.code,
4878                    refusal.code,
4879                    refusal.message,
4880                )?]);
4881            }
4882        }
4883        if let Some(stamp) = scope_stamp
4884            .as_ref()
4885            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4886        {
4887            if let Err(refusal) = crate::scopes::check_target_flow_support(
4888                stamp,
4889                &target_module_id,
4890                relay_capabilities,
4891            )
4892            .and_then(|()| {
4893                crate::scopes::check_target_agent_run_support(
4894                    stamp,
4895                    &target_module_id,
4896                    relay_capabilities,
4897                )
4898            }) {
4899                reservation.release_and_disarm();
4900                return Ok(vec![self.route_open_refusal_frame(
4901                    ctx,
4902                    &frame,
4903                    &target_module_id,
4904                    refusal.code,
4905                    refusal.code,
4906                    refusal.message,
4907                )?]);
4908            }
4909        }
4910
4911        // Older readers may reject unknown fields. Select optional fields from
4912        // the captured endpoint's declaration, including on unscoped binds.
4913        if !relay_capabilities.is_some_and(|caps| {
4914            caps.provides
4915                .iter()
4916                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
4917        }) {
4918            if let Some(stamp) = scope_stamp.as_mut() {
4919                stamp.workdir = None;
4920                stamp.project_id = None;
4921            }
4922        }
4923        if !relay_capabilities.is_some_and(|caps| {
4924            caps.provides
4925                .iter()
4926                .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4927        }) {
4928            if let Some(stamp) = scope_stamp.as_mut() {
4929                stamp.attributes.on_behalf_of_agent = None;
4930            }
4931        }
4932        let call_mode = if relay_capabilities
4933            .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4934        {
4935            Some(derive_call_mode(scope_stamp.as_ref()))
4936        } else {
4937            if let Some(stamp) = scope_stamp.as_mut() {
4938                stamp.expires_at_ms = None;
4939            }
4940            None
4941        };
4942
4943        debug!(
4944            connection_id = ctx.connection_id.get(),
4945            client_channel,
4946            client_epoch,
4947            module_channel,
4948            module_epoch,
4949            "reserved route handle pair"
4950        );
4951        // Rendered BEFORE the move into the relay, because the accept arm below
4952        // is where it is logged and the principal is gone by then.
4953        let route_principal_label = principal_label(&principal);
4954        let scope_label = scope_stamp
4955            .as_ref()
4956            .map_or_else(|| "none".to_string(), route_open_scope_label);
4957        let relay = ModuleControlRequest::RouteBind {
4958            route_channel: module_channel,
4959            epoch: module_epoch,
4960            target,
4961            identity,
4962            principal: Some(principal),
4963            consumer_capabilities,
4964            role_versions,
4965            admission_facts,
4966            scope: scope_stamp,
4967            call_mode,
4968        };
4969        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4970            RouterError::backend(
4971                0,
4972                frame.header.corr,
4973                format!("failed to encode route.bind request: {err}"),
4974            )
4975        })?;
4976        let relay_frame = Frame::build_with_version(
4977            negotiated_ver,
4978            FrameType::Request,
4979            control_flags(),
4980            0,
4981            0,
4982            relay_corr,
4983            relay_body,
4984        )
4985        .map_err(RouterError::FrameBuild)?;
4986
4987        timings.enter(RouteOpenPhase::BindSend);
4988        let sent = module_sink.send(relay_frame).await;
4989        timings.enter(RouteOpenPhase::Admission);
4990        if let Err(err) = sent {
4991            reservation.release_and_disarm();
4992            return Ok(vec![self.route_open_refusal_frame(
4993                ctx,
4994                &frame,
4995                &target_module_id,
4996                "relay_send_failed",
4997                "target_unavailable",
4998                err.to_string(),
4999            )?]);
5000        }
5001
5002        #[cfg(test)]
5003        if let Some(hold) = &self.route_bind_relay_sent_hold {
5004            // First wait: tell the test the bind is queued. Second wait: stay
5005            // here until the test has driven whatever it wants to happen
5006            // between queueing the bind and marking it enqueued.
5007            hold.wait().await;
5008            hold.wait().await;
5009        }
5010
5011        // A missing relay is not by itself an abandoned bind: the module may
5012        // already have acked it and the route may be committed. Only a relay
5013        // dropped without any module reply needs this GOODBYE; one a reply
5014        // settled has already had its GOODBYE decided by the reply path.
5015        if self
5016            .forwarding
5017            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5018            .map_err(RouterError::Forwarding)?
5019            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5020        {
5021            self.send_abandoned_route_bind_goodbye(
5022                &module_sink,
5023                negotiated_ver,
5024                module_channel,
5025                module_epoch,
5026            );
5027        }
5028
5029        timings.enter(RouteOpenPhase::BindAckWait);
5030        let outcome = timeout_at(relay_deadline, receiver).await;
5031        timings.enter(RouteOpenPhase::Admission);
5032        match outcome {
5033            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5034                reservation.disarm();
5035                if breaker.record_accepted() {
5036                    info!(
5037                        module_id = %target_module_id,
5038                        "route.bind breaker closed: the probe was accepted"
5039                    );
5040                }
5041                self.observe_route_open_accept(
5042                    ctx,
5043                    &target_module_id,
5044                    &route_principal_label,
5045                    &scope_label,
5046                );
5047                Ok(Vec::new())
5048            }
5049            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5050                reservation.release_and_disarm();
5051                // A module that says no in microseconds is healthy. Rejection
5052                // is a different condition with its own refusal and must not
5053                // move the breaker.
5054                breaker.record_inconclusive();
5055                // The daemon's own commit re-check refused the bind because the
5056                // scope ended or changed after admission. The module accepted;
5057                // counting it as a module rejection would blame the module.
5058                let scope_code = match body.code.as_str() {
5059                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5060                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5061                    _ => None,
5062                };
5063                if let Some(code) = scope_code {
5064                    self.observe_route_open_refusal(
5065                        ctx,
5066                        &target_module_id,
5067                        "scope_changed_before_commit",
5068                        code,
5069                    );
5070                    return Ok(vec![control_error_body_frame(&frame, body)?]);
5071                }
5072                self.counters
5073                    .increment_route_open_refused("module_rejected");
5074                info!(
5075                    target: "control",
5076                    code = "module_rejected",
5077                    module_code = ?body.code,
5078                    module_id = ?target_module_id,
5079                    connection_id = ctx.connection_id.get(),
5080                    "route.open refused"
5081                );
5082                Ok(vec![control_error_body_frame(&frame, body)?])
5083            }
5084            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5085                reservation.release_and_disarm();
5086                breaker.record_inconclusive();
5087                // Fires when the module's connection closes while a relayed
5088                // bind is pending -- typically a caller racing a module restart
5089                // whose bind was relayed BEFORE the drain mark went up. Logged
5090                // because the caller sees only its own error and the fleet has
5091                // already spent one diagnosis round unable to tell this arm
5092                // from a relay timeout without daemon-side evidence.
5093                tracing::warn!(
5094                    module_id = %target_module_id,
5095                    "route.bind relay abandoned: {message}"
5096                );
5097                Ok(vec![self.route_open_refusal_frame(
5098                    ctx,
5099                    &frame,
5100                    &target_module_id,
5101                    "relay_abandoned",
5102                    "target_unavailable",
5103                    message,
5104                )?])
5105            }
5106            Ok(Err(_)) => {
5107                reservation.release_and_disarm();
5108                breaker.record_inconclusive();
5109                Ok(vec![self.route_open_refusal_frame(
5110                    ctx,
5111                    &frame,
5112                    &target_module_id,
5113                    "relay_waiter_canceled",
5114                    "target_unavailable",
5115                    "route.bind relay waiter was canceled before the module responded",
5116                )?])
5117            }
5118            Err(_) => {
5119                reservation.release_and_disarm();
5120                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5121                // answer at all is the one condition a fast refusal can
5122                // usefully stand in for; every other arm already answered.
5123                if let Some(opened) = breaker.record_timeout(
5124                    self.route_bind_breaker_threshold,
5125                    self.route_bind_breaker_cooldown,
5126                ) {
5127                    warn!(
5128                        module_id = %target_module_id,
5129                        consecutive_timeouts = opened.consecutive_timeouts,
5130                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5131                        reopened_after_probe = opened.reopened_after_probe,
5132                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5133                    );
5134                }
5135                // The generous budget just burned to no answer: the module is
5136                // registered and its connection is up, but its bind handler sat
5137                // on the ack for the full budget (warm-on-bind, cold configure,
5138                // or a wedged handler). Every earlier unavailability shape
5139                // fast-refuses BEFORE the relay, so this arm firing means the
5140                // slowness is module-side -- log it so the per-module timeline
5141                // is reconstructable without client audit rows.
5142                tracing::warn!(
5143                    module_id = %target_module_id,
5144                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5145                    "route.bind relay timed out: module did not ack within budget"
5146                );
5147                Ok(vec![self.route_open_refusal_frame(
5148                    ctx,
5149                    &frame,
5150                    &target_module_id,
5151                    "relay_timed_out",
5152                    "module_timeout",
5153                    format!(
5154                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5155                        route_bind_relay_timeout
5156                    ),
5157                )?])
5158            }
5159        }
5160    }
5161
5162    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5163        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5164            snapshot: self.supervisor.spawn_snapshot(),
5165        };
5166        Ok(vec![control_response_body_frame(
5167            &frame,
5168            &response,
5169            "ClientControlResponse::SupervisorSpawnSnapshot",
5170        )?])
5171    }
5172
5173    fn handle_supervisor_spawn_subscribe(
5174        &self,
5175        ctx: &RouteCtx,
5176        frame: Frame,
5177        since: Option<SpawnCursor>,
5178    ) -> Result<Vec<Frame>, RouterError> {
5179        match self.supervisor.subscribe_spawns(
5180            ctx.connection_id,
5181            frame.header.corr,
5182            response_version(&frame),
5183            since,
5184            ctx.egress.clone(),
5185        ) {
5186            Ok(()) => Ok(Vec::new()),
5187            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5188                Ok(vec![control_error_body_frame(
5189                    &frame,
5190                    ErrorBody {
5191                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5192                        message: "spawn cursor belongs to a different daemon incarnation"
5193                            .to_string(),
5194                        detail: Some(serde_json::json!({
5195                            "current_daemon_incarnation": current
5196                        })),
5197                    },
5198                )?])
5199            }
5200            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5201                &frame,
5202                ErrorBody {
5203                    code: "spawn_cursor_too_old".to_string(),
5204                    message: "spawn cursor predates the retained event ring".to_string(),
5205                    detail: Some(serde_json::json!({
5206                        "oldest_retained_cursor": oldest
5207                    })),
5208                },
5209            )?]),
5210            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5211                0,
5212                frame.header.corr,
5213                format!("failed to open supervisor spawn subscription: {error}"),
5214            )),
5215        }
5216    }
5217
5218    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5219        let generation = self
5220            .registry
5221            .generation()
5222            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5223        let mut modules = Vec::new();
5224        for module in self.supervisor.list() {
5225            let status = module.status_for_control("list").map_err(|err| {
5226                RouterError::backend(
5227                    0,
5228                    frame.header.corr,
5229                    format!("failed to read supervisor status: {err}"),
5230                )
5231            })?;
5232            let (configured, _) = module.configuration().map_err(|err| {
5233                RouterError::backend(
5234                    0,
5235                    frame.header.corr,
5236                    format!("failed to read module configuration: {err}"),
5237                )
5238            })?;
5239            // Status and configuration snapshots release their locks before the image probe awaits.
5240            let image = module.running_image_agreement().await;
5241            // Read per request so the figure is current when the operator asks;
5242            // the daemon samples nothing in between.
5243            let resources = Some(module.child_resource_usage());
5244            let mut verdict =
5245                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5246            verdict.spec = Some(launch_spec_agreement(
5247                &configured,
5248                status.process_alive,
5249                status.launch_spec_digest.as_ref(),
5250            ));
5251            let pending_reload = Some(verdict);
5252            let configured_enabled = self
5253                .capability_evaluator
5254                .configured_enabled(&status.module_id);
5255            // Keep the retired policy field on the wire for one release so
5256            // existing status consumers still receive the platform policy.
5257            modules.push(
5258                SupervisorEntry::new(
5259                    status.module_id,
5260                    status.state.to_string(),
5261                    status.enabled,
5262                    status.live,
5263                    status.health.status,
5264                )
5265                .with_configured_enabled(configured_enabled)
5266                .with_launch_nonce_env(Some(!cfg!(unix)))
5267                .with_launch_nonce_source(status.launch_nonce_source)
5268                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5269                .with_protocol(status.protocol)
5270                .with_pending_reload(pending_reload)
5271                .with_last_probe_ms(status.health.last_probe_ms)
5272                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5273                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5274                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5275                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5276                .with_restart_count(Some(status.restart_count))
5277                .with_max_restarts(Some(status.max_restarts))
5278                .with_lifetime_restarts(Some(status.lifetime_restarts))
5279                .with_spawn_generation(Some(status.spawn_generation))
5280                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5281                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5282                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5283                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5284                .with_resources(resources),
5285            );
5286        }
5287        let response = ClientControlResponse::SupervisorList {
5288            generation,
5289            modules,
5290        };
5291        Ok(vec![control_response_body_frame(
5292            &frame,
5293            &response,
5294            "ClientControlResponse::SupervisorList",
5295        )?])
5296    }
5297
5298    fn handle_supervisor_stderr_tail(
5299        &self,
5300        frame: Frame,
5301        module_id: String,
5302        max_lines: Option<u32>,
5303        max_bytes: Option<u32>,
5304    ) -> Result<Vec<Frame>, RouterError> {
5305        let Some(module) = self.supervisor.get(&module_id) else {
5306            return Ok(vec![control_error_frame(
5307                &frame,
5308                "unknown_module",
5309                format!("module_id '{module_id}' is not supervised"),
5310            )?]);
5311        };
5312
5313        let snapshot = module.stderr_tail(
5314            max_lines.map(|value| value as usize),
5315            max_bytes.map(|value| value as usize),
5316        );
5317
5318        let response = ClientControlResponse::SupervisorStderrTail {
5319            module_id,
5320            tail: StderrTail {
5321                capture: match snapshot.capture {
5322                    CaptureState::Captured => StderrCaptureState::Captured,
5323                    CaptureState::Incomplete { reason } => {
5324                        StderrCaptureState::Incomplete { reason }
5325                    }
5326                    CaptureState::NotCaptured { reason } => {
5327                        StderrCaptureState::NotCaptured { reason }
5328                    }
5329                },
5330                entries: snapshot
5331                    .entries
5332                    .into_iter()
5333                    .map(|entry| match entry {
5334                        TailEntry::Line {
5335                            text,
5336                            truncated,
5337                            at_ms,
5338                        } => StderrTailEntry::Line {
5339                            text,
5340                            truncated,
5341                            at_ms,
5342                        },
5343                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5344                    })
5345                    .collect(),
5346                dropped_lines: snapshot.dropped_lines,
5347            },
5348        };
5349        Ok(vec![control_response_body_frame(
5350            &frame,
5351            &response,
5352            "ClientControlResponse::SupervisorStderrTail",
5353        )?])
5354    }
5355
5356    async fn handle_supervisor_terminals(
5357        &self,
5358        frame: Frame,
5359        module_id: String,
5360    ) -> Result<Vec<Frame>, RouterError> {
5361        let Some(module) = self.supervisor.get(&module_id) else {
5362            return Ok(vec![control_error_frame(
5363                &frame,
5364                "unknown_module",
5365                format!("module_id '{module_id}' is not supervised"),
5366            )?]);
5367        };
5368
5369        // The journal read runs on a blocking thread: it can be megabytes of
5370        // file I/O and must not occupy a runtime worker.
5371        let terminals = module
5372            .read_durable_terminal_history()
5373            .await
5374            .map_err(|error| {
5375                RouterError::backend(
5376                    0,
5377                    frame.header.corr,
5378                    format!("failed to read terminal history: {error}"),
5379                )
5380            })?;
5381        let response = ClientControlResponse::SupervisorTerminals {
5382            module_id,
5383            terminals,
5384        };
5385        Ok(vec![control_response_body_frame(
5386            &frame,
5387            &response,
5388            "ClientControlResponse::SupervisorTerminals",
5389        )?])
5390    }
5391
5392    fn handle_supervisor_routes(
5393        &self,
5394        frame: Frame,
5395        module_id: Option<String>,
5396    ) -> Result<Vec<Frame>, RouterError> {
5397        let modules = self
5398            .forwarding
5399            .route_census(module_id.as_deref())
5400            .map_err(RouterError::Forwarding)?
5401            .into_iter()
5402            .map(|(module_id, routes)| SupervisorRouteModule {
5403                module_id,
5404                routes: routes
5405                    .into_iter()
5406                    .map(|route| SupervisorRoute {
5407                        consumer: match route.principal {
5408                            Principal::Reserved { module_id } => {
5409                                SupervisorRouteConsumer::Reserved { module_id }
5410                            }
5411                            Principal::Direct | Principal::Unverified => {
5412                                SupervisorRouteConsumer::Direct {
5413                                    connection_id: route.goodbye_target.connection_id.get(),
5414                                }
5415                            }
5416                        },
5417                        age_ms: Instant::now()
5418                            .saturating_duration_since(route.bound_at)
5419                            .as_millis()
5420                            .try_into()
5421                            .unwrap_or(u64::MAX),
5422                        draining: route.draining,
5423                        drain_reason: route.drain_reason,
5424                    })
5425                    .collect(),
5426            })
5427            .collect();
5428        let response = ClientControlResponse::SupervisorRoutes { modules };
5429        Ok(vec![control_response_body_frame(
5430            &frame,
5431            &response,
5432            "ClientControlResponse::SupervisorRoutes",
5433        )?])
5434    }
5435
5436    async fn handle_supervisor_provenance(
5437        &self,
5438        frame: Frame,
5439        module_id: Option<String>,
5440    ) -> Result<Vec<Frame>, RouterError> {
5441        let mut selected = if let Some(module_id) = module_id {
5442            let Some(module) = self.supervisor.get(&module_id) else {
5443                return Ok(vec![control_error_frame(
5444                    &frame,
5445                    "unknown_module",
5446                    format!("module_id '{module_id}' is not supervised"),
5447                )?]);
5448            };
5449            vec![module]
5450        } else {
5451            self.supervisor.list()
5452        };
5453
5454        let mut modules = Vec::with_capacity(selected.len());
5455        for module in selected.drain(..) {
5456            let (status, observed_image) = module
5457                .status_and_running_image_agreement()
5458                .await
5459                .map_err(|err| {
5460                    RouterError::backend(
5461                        0,
5462                        frame.header.corr,
5463                        format!("failed to read supervisor status: {err}"),
5464                    )
5465                })?;
5466            let module_declared = self
5467                .registry
5468                .get_module(&status.module_id)
5469                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5470                .and_then(|registration| registration.manifest.provenance)
5471                .map(|build| ModuleDeclaredProvenance::Reported { build })
5472                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5473            #[cfg(test)]
5474            let running_image = match &self.provenance_probe_override {
5475                Some(result) => result.clone(),
5476                None => observed_image,
5477            };
5478            #[cfg(not(test))]
5479            let running_image = observed_image;
5480            modules.push(SupervisorModuleProvenance {
5481                module_id: status.module_id,
5482                module_declared,
5483                daemon_observed: SupervisorObservedProcess {
5484                    launch_nonce_source: status.launch_nonce_source,
5485                    pid: status.pid,
5486                    spawned_at_ms: status.spawned_at_ms,
5487                    spawned_from: status.spawned_from,
5488                    running_image,
5489                },
5490            });
5491        }
5492        let daemon = SupervisorDaemonProvenance {
5493            daemon_build: self.daemon_provenance.build.clone(),
5494            daemon_observed: DaemonObservedProcess {
5495                pid: self.daemon_provenance.pid,
5496                started_at_ms: self
5497                    .daemon_provenance
5498                    .start_clock
5499                    .map(|clock| clock.started_at_ms())
5500                    .or(self.daemon_provenance.started_at_ms),
5501                running_image: self
5502                    .daemon_provenance
5503                    .probe
5504                    .observe(
5505                        self.daemon_provenance.pid,
5506                        self.daemon_provenance.executable_path.as_deref(),
5507                        self.daemon_provenance.executable_identity.clone(),
5508                        self.daemon_provenance.process_start_time,
5509                    )
5510                    .await,
5511            },
5512        };
5513        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5514        Ok(vec![control_response_body_frame(
5515            &frame,
5516            &response,
5517            "ClientControlResponse::SupervisorProvenance",
5518        )?])
5519    }
5520
5521    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5522        self.refresh_capability_requirements();
5523        let generation = self
5524            .registry
5525            .generation()
5526            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5527        let modules = self
5528            .supervisor
5529            .list()
5530            .into_iter()
5531            .map(|module| {
5532                let status = module.status_for_control("health").map_err(|err| {
5533                    RouterError::backend(
5534                        0,
5535                        frame.header.corr,
5536                        format!("failed to read supervisor health: {err}"),
5537                    )
5538                })?;
5539                let module_id = status.module_id;
5540                let capability_detail = self
5541                    .capability_evaluator
5542                    .required_problem_detail(&module_id);
5543                Ok(SupervisorHealthEntry {
5544                    module_id,
5545                    status: status.health.status,
5546                    detail: append_capability_problem_detail(
5547                        status.health.detail,
5548                        capability_detail,
5549                    ),
5550                    metrics: status.health.metrics,
5551                    consecutive_failures: status.health.consecutive_failures,
5552                    late_answer_count: status.health.late_answer_count,
5553                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5554                    last_action: status.health.last_action,
5555                    last_action_ms: status.health.last_action_ms,
5556                    last_probe_ms: status.health.last_probe_ms,
5557                })
5558            })
5559            .collect::<Result<Vec<_>, RouterError>>()?;
5560        let response = ClientControlResponse::SupervisorHealth {
5561            generation,
5562            modules,
5563        };
5564        Ok(vec![control_response_body_frame(
5565            &frame,
5566            &response,
5567            "ClientControlResponse::SupervisorHealth",
5568        )?])
5569    }
5570
5571    async fn handle_supervisor_restart(
5572        &self,
5573        frame: Frame,
5574        module_id: String,
5575        drain_timeout_ms: Option<u64>,
5576        requester: Principal,
5577    ) -> Result<Vec<Frame>, RouterError> {
5578        let operation_lock = self.supervisor.operation_lock();
5579        let _operation_guard = operation_lock.lock().await;
5580        let Some(module) = self.supervisor.get(&module_id) else {
5581            return Ok(vec![control_error_frame(
5582                &frame,
5583                "unknown_module",
5584                format!("module_id '{module_id}' is not supervised"),
5585            )?]);
5586        };
5587
5588        self.route_outages.mark_operator_action(&module_id);
5589        if let Err(err) = module
5590            .restart_requested(drain_timeout_ms, Some(requester))
5591            .await
5592        {
5593            self.route_outages
5594                .operator_action_ended_unrefused(&module_id);
5595            let (code, message) = match err {
5596                crate::supervise::SuperviseError::Disabled { .. } => {
5597                    ("module_disabled", err.to_string())
5598                }
5599                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5600                    ("swap_in_progress", err.to_string())
5601                }
5602                _ => (
5603                    "target_unavailable",
5604                    format!("failed to restart module_id '{module_id}': {err}"),
5605                ),
5606            };
5607            return Ok(vec![control_error_frame(&frame, code, message)?]);
5608        }
5609
5610        let response = ClientControlResponse::SupervisorAck {
5611            module_id,
5612            applied: true,
5613        };
5614        Ok(vec![control_response_body_frame(
5615            &frame,
5616            &response,
5617            "ClientControlResponse::SupervisorAck",
5618        )?])
5619    }
5620
5621    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
5622    /// when the old process has finished draining: a caller whose own lane
5623    /// rides the old process must get its reply before that drain waits on it.
5624    async fn handle_supervisor_swap(
5625        &self,
5626        frame: Frame,
5627        module_id: String,
5628        ready_timeout_ms: Option<u64>,
5629        requester: Principal,
5630    ) -> Result<Vec<Frame>, RouterError> {
5631        // The daemon-wide operation lock is held only to resolve the handle,
5632        // not across the swap. The swap can take its whole readiness budget,
5633        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
5634        // holding it here would park an operator's stop behind the swap it is
5635        // meant to abort. A rescan or stop that reaches the module during the
5636        // swap is served by the swap itself (see `supervise_swap`).
5637        let module = {
5638            let operation_lock = self.supervisor.operation_lock();
5639            let _operation_guard = operation_lock.lock().await;
5640            self.supervisor.get(&module_id)
5641        };
5642        let Some(module) = module else {
5643            return Ok(vec![control_error_frame(
5644                &frame,
5645                "unknown_module",
5646                format!("module_id '{module_id}' is not supervised"),
5647            )?]);
5648        };
5649
5650        self.route_outages.mark_operator_action(&module_id);
5651        if let Err(err) = module
5652            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5653            .await
5654        {
5655            self.route_outages
5656                .operator_action_ended_unrefused(&module_id);
5657            use crate::supervise::SuperviseError;
5658            let message = err.to_string();
5659            let error = match err {
5660                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5661                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5662                    code: "swap_refused".to_string(),
5663                    message,
5664                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5665                },
5666                SuperviseError::SwapFailed {
5667                    arm,
5668                    candidate_exit,
5669                    ..
5670                } => ErrorBody {
5671                    code: "swap_failed".to_string(),
5672                    message,
5673                    detail: Some(serde_json::json!({
5674                        "arm": arm.as_str(),
5675                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5676                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5677                    })),
5678                },
5679                _ => ErrorBody::new(
5680                    "target_unavailable",
5681                    format!("failed to swap module_id '{module_id}': {message}"),
5682                ),
5683            };
5684            return Ok(vec![control_error_body_frame(&frame, error)?]);
5685        }
5686        // A completed swap kept the incumbent serving until cutover, so it
5687        // usually opened no outage; a mark left behind would make the next,
5688        // unrelated outage read as requested.
5689        self.route_outages
5690            .operator_action_ended_unrefused(&module_id);
5691
5692        let response = ClientControlResponse::SupervisorAck {
5693            module_id,
5694            applied: true,
5695        };
5696        Ok(vec![control_response_body_frame(
5697            &frame,
5698            &response,
5699            "ClientControlResponse::SupervisorAck",
5700        )?])
5701    }
5702
5703    async fn handle_supervisor_reload(
5704        &self,
5705        frame: Frame,
5706        module_id: String,
5707        requester: Principal,
5708    ) -> Result<Vec<Frame>, RouterError> {
5709        let operation_lock = self.supervisor.operation_lock();
5710        let _operation_guard = operation_lock.lock().await;
5711        let Some(module) = self.supervisor.get(&module_id) else {
5712            return Ok(vec![control_error_frame(
5713                &frame,
5714                "unknown_module",
5715                format!("module_id '{module_id}' is not supervised"),
5716            )?]);
5717        };
5718
5719        self.route_outages.mark_operator_action(&module_id);
5720        if let Err(err) = module.reload_requested(Some(requester)).await {
5721            self.route_outages
5722                .operator_action_ended_unrefused(&module_id);
5723            let (code, message) = match err {
5724                crate::supervise::SuperviseError::Disabled { .. } => {
5725                    ("module_disabled", err.to_string())
5726                }
5727                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5728                    ("swap_in_progress", err.to_string())
5729                }
5730                _ => (
5731                    "reload_failed",
5732                    format!("failed to reload module_id '{module_id}': {err}"),
5733                ),
5734            };
5735            return Ok(vec![control_error_frame(&frame, code, message)?]);
5736        }
5737
5738        let response = ClientControlResponse::SupervisorAck {
5739            module_id,
5740            applied: true,
5741        };
5742        Ok(vec![control_response_body_frame(
5743            &frame,
5744            &response,
5745            "ClientControlResponse::SupervisorAck",
5746        )?])
5747    }
5748
5749    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
5750    /// the configuration comes from and whether to apply it; every mode then
5751    /// runs the same comparison, so a preview describes the apply it previews.
5752    async fn handle_supervisor_rescan(
5753        &self,
5754        frame: Frame,
5755        mode: RescanMode,
5756        requester: Principal,
5757    ) -> Result<Vec<Frame>, RouterError> {
5758        let preview = mode.previews();
5759        let Some(context) = self.rescan.clone() else {
5760            return Ok(vec![control_error_frame(
5761                &frame,
5762                "rescan_unavailable",
5763                "the daemon was not started with a reloadable config path".to_string(),
5764            )?]);
5765        };
5766
5767        let operation_lock = self.supervisor.operation_lock();
5768        let _operation_guard = operation_lock.lock().await;
5769        // Loaded under the operation lock, so no other rescan can change the
5770        // staged configuration between a checked apply's comparison and the
5771        // apply itself.
5772        let loaded = match self.load_rescan_config(&mode, &context) {
5773            Ok(config) => config,
5774            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5775        };
5776        // `load` reports a missing file as Ok(None), which is correct at boot
5777        // (no config, nothing to supervise) and catastrophic here: rescan treats
5778        // "not in the config" as "remove it", so an absent file would read as an
5779        // empty module list and retire the entire running fleet. An editor
5780        // writing via write-new-then-rename, or a half-finished edit, is enough
5781        // to open that window. Refuse instead: a config that cannot be read
5782        // carries no instruction to remove anything.
5783        let Some((config, activity_config)) = loaded else {
5784            return Ok(vec![control_error_frame(
5785                &frame,
5786                "invalid_daemon_config",
5787                format!(
5788                    "daemon config not found at {}; refusing to rescan (an absent config would \
5789                     retire every supervised module)",
5790                    context.config_path.display()
5791                ),
5792            )?]);
5793        };
5794        let (
5795            configured_port,
5796            storage_config,
5797            admission_facts_carrier_module_id,
5798            admission_facts_targets,
5799            scope_authority_owners,
5800            privileged_operator_prompt,
5801            modules,
5802            reserved_capabilities,
5803        ) = (
5804            config.port,
5805            config.storage,
5806            config.admission_facts_carrier_module_id,
5807            config.admission_facts_targets,
5808            config.scope_authority_owners,
5809            config.privileged_operator_prompt,
5810            config.modules,
5811            config.reserved_capabilities,
5812        );
5813
5814        // Collect the sections rescan cannot apply, so the REPLY carries them.
5815        //
5816        // The warning below has always been correct and has always gone only to
5817        // the journal -- addressed to whoever reads logs, while the person who
5818        // just edited the config is looking at the CLI. Naming each section
5819        // individually rather than setting a flag: "something outside modules
5820        // changed" sends the operator back to diffing their own file, which is
5821        // the work this is meant to save.
5822        let mut restart_required = Vec::new();
5823        for section in RestartRequiredSection::ALL {
5824            let changed = match section {
5825                RestartRequiredSection::Port => configured_port != context.configured_port,
5826                RestartRequiredSection::Storage => storage_config != context.storage_config,
5827                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5828                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5829                }
5830                RestartRequiredSection::AdmissionFactsTargets => {
5831                    admission_facts_targets != context.admission_facts_targets
5832                }
5833                RestartRequiredSection::ScopeAuthorityOwners => {
5834                    scope_authority_owners != context.scope_authority_owners
5835                }
5836                RestartRequiredSection::PrivilegedOperatorPrompt => {
5837                    privileged_operator_prompt != context.privileged_operator_prompt
5838                }
5839            };
5840            if changed {
5841                restart_required.push(section.label().to_string());
5842            }
5843        }
5844        if !restart_required.is_empty() {
5845            warn!(
5846                config_path = %context.config_path.display(),
5847                sections = %restart_required.join(", "),
5848                "daemon config changed outside the modules section; restart the daemon to apply those changes"
5849            );
5850        }
5851
5852        for configured in &modules {
5853            if let Err(err) = validate_spec(&configured.module_spec()) {
5854                return Ok(vec![control_error_frame(
5855                    &frame,
5856                    "invalid_daemon_config",
5857                    format!("supervisor rescan rejected daemon config: {err}"),
5858                )?]);
5859            }
5860        }
5861
5862        let configured_capabilities = modules
5863            .iter()
5864            .map(|module| (module.module_id.clone(), module.enabled))
5865            .collect::<Vec<_>>();
5866        let preview_capability_warnings = if preview {
5867            let (_, registrations) = self.runtime_capability_snapshot()?;
5868            let current_modules = self
5869                .supervisor
5870                .list()
5871                .into_iter()
5872                .map(|module| module.module_id().to_string())
5873                .collect::<BTreeSet<_>>();
5874            let resulting_modules = configured_capabilities.clone();
5875            let removed = current_modules
5876                .into_iter()
5877                .filter(|module_id| {
5878                    !resulting_modules
5879                        .iter()
5880                        .any(|(configured_id, _)| configured_id == module_id)
5881                })
5882                .collect::<Vec<_>>();
5883            // A candidate preview also names providers the candidate disables.
5884            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
5885            let disabled = match &mode {
5886                RescanMode::Candidate(_) => resulting_modules
5887                    .iter()
5888                    .filter(|(_, enabled)| !enabled)
5889                    .map(|(module_id, _)| module_id.clone())
5890                    .collect::<Vec<_>>(),
5891                _ => Vec::new(),
5892            };
5893            self.capability_evaluator.preview_warnings(
5894                resulting_modules,
5895                &removed,
5896                &disabled,
5897                &registrations,
5898            )
5899        } else {
5900            Vec::new()
5901        };
5902        let reconciled = match &mode {
5903            RescanMode::Candidate(candidate) => self
5904                .preview_candidate(&context, modules, candidate)
5905                .map(|result| (result, Vec::new())),
5906            _ => {
5907                self.reconcile_supervised_modules(
5908                    &context.supervisor,
5909                    modules,
5910                    preview,
5911                    requester.clone(),
5912                )
5913                .await
5914            }
5915        };
5916        let (result, changed) = match reconciled {
5917            Ok(result) => result,
5918            Err(message) => {
5919                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5920            }
5921        };
5922        if !preview {
5923            if let Some(ledger) = &self.activity {
5924                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
5925                ledger.config_changed(&result.added, &result.removed, &changed, requester);
5926            }
5927            if let Err(code) = self.configure_activity(activity_config).await {
5928                warn!(
5929                    code,
5930                    "activity retention could not be applied during rescan"
5931                );
5932            }
5933            self.capability_evaluator
5934                .configure(configured_capabilities, reserved_capabilities);
5935            self.capability_evaluator.wake_deadline_loop();
5936            self.refresh_capability_requirements();
5937        }
5938        let mut result = result;
5939        result.restart_required = restart_required;
5940        result.capability_warnings = preview_capability_warnings;
5941        let response = ClientControlResponse::SupervisorRescan { result };
5942        Ok(vec![control_response_body_frame(
5943            &frame,
5944            &response,
5945            "ClientControlResponse::SupervisorRescan",
5946        )?])
5947    }
5948
5949    async fn handle_supervisor_release_reserved(
5950        &self,
5951        frame: Frame,
5952        module_id: String,
5953    ) -> Result<Vec<Frame>, RouterError> {
5954        let Some(context) = self.rescan.clone() else {
5955            return Ok(vec![control_error_frame(
5956                &frame,
5957                "release_unavailable",
5958                "reserved-id release requires a daemon started with a reloadable config path",
5959            )?]);
5960        };
5961        let operation_lock = self.supervisor.operation_lock();
5962        let _operation_guard = operation_lock.lock().await;
5963        let loaded = match crate::daemon_config::load(&context.config_path) {
5964            Ok(Some(config)) => config,
5965            Ok(None) => {
5966                return Ok(vec![control_error_frame(
5967                    &frame,
5968                    "invalid_daemon_config",
5969                    format!(
5970                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5971                        context.config_path.display()
5972                    ),
5973                )?])
5974            }
5975            Err(err) => {
5976                return Ok(vec![control_error_frame(
5977                    &frame,
5978                    "invalid_daemon_config",
5979                    format!("unable to verify reserved-id release against daemon config: {err}"),
5980                )?])
5981            }
5982        };
5983        if loaded
5984            .modules
5985            .iter()
5986            .any(|configured| configured.module_id == module_id)
5987        {
5988            return Ok(vec![control_error_frame(
5989                &frame,
5990                "reserved_module_configured",
5991                format!(
5992                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5993                ),
5994            )?]);
5995        }
5996        if !self.supervisor.release_retained_reserved_gate(&module_id) {
5997            return Ok(vec![control_error_frame(
5998                &frame,
5999                "reserved_gate_not_retained",
6000                format!(
6001                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6002                ),
6003            )?]);
6004        }
6005
6006        let response = ClientControlResponse::SupervisorAck {
6007            module_id,
6008            applied: true,
6009        };
6010        Ok(vec![control_response_body_frame(
6011            &frame,
6012            &response,
6013            "ClientControlResponse::SupervisorAck",
6014        )?])
6015    }
6016
6017    /// Compare the configured module set with the supervised one, changing
6018    /// nothing. The staged spec each supervised module holds is what the
6019    /// configured entry is compared with.
6020    fn plan_rescan(
6021        &self,
6022        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6023    ) -> Result<RescanPlan, String> {
6024        let mut current = BTreeMap::new();
6025        for module in self.supervisor.list() {
6026            let (spec, health) = module.configuration().map_err(|err| {
6027                format!(
6028                    "failed to read configuration for module_id '{}': {err}",
6029                    module.module_id()
6030                )
6031            })?;
6032            let enabled = module
6033                .status()
6034                .map_err(|err| {
6035                    format!(
6036                        "failed to read status for module_id '{}': {err}",
6037                        module.module_id()
6038                    )
6039                })?
6040                .enabled;
6041            current.insert(
6042                module.module_id().to_string(),
6043                (module, spec, health, enabled),
6044            );
6045        }
6046        let configured = configured_modules
6047            .into_iter()
6048            .map(|module| (module.module_id.clone(), module))
6049            .collect::<BTreeMap<_, _>>();
6050
6051        let added = configured
6052            .keys()
6053            .filter(|module_id| !current.contains_key(*module_id))
6054            .cloned()
6055            .collect::<Vec<_>>();
6056        let removed = current
6057            .keys()
6058            .filter(|module_id| !configured.contains_key(*module_id))
6059            .cloned()
6060            .collect::<Vec<_>>();
6061        let mut changed_pending_reload = Vec::new();
6062        let mut configuration_changes = BTreeSet::new();
6063        let mut enabled_changes = BTreeSet::new();
6064        let mut unchanged = 0_u32;
6065
6066        for (module_id, configured_module) in &configured {
6067            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6068            else {
6069                continue;
6070            };
6071            // Compare the whole launch spec so a future launch field cannot
6072            // accidentally become a live-only policy change. Health is stored
6073            // separately and applies live without replacing the process.
6074            let launch_changed = *current_spec != configured_module.module_spec();
6075            let configuration_changed =
6076                launch_changed || *current_health != configured_module.health;
6077            let enabled_changed = *current_enabled != configured_module.enabled;
6078            if configuration_changed {
6079                configuration_changes.insert(module_id.clone());
6080            }
6081            if launch_changed {
6082                changed_pending_reload.push(module_id.clone());
6083            }
6084            if enabled_changed {
6085                enabled_changes.insert(module_id.clone());
6086            }
6087            if !configuration_changed && !enabled_changed {
6088                unchanged = unchanged.saturating_add(1);
6089            }
6090        }
6091        Ok(RescanPlan {
6092            current,
6093            configured,
6094            added,
6095            removed,
6096            changed_pending_reload,
6097            configuration_changes,
6098            enabled_changes,
6099            unchanged,
6100        })
6101    }
6102
6103    /// Reconcile the running module set against the configured one.
6104    ///
6105    /// With `preview` set, the diff is computed and returned WITHOUT applying any
6106    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
6107    /// deliberately shares this function with the executing path rather than
6108    /// computing the same diff somewhere else -- two implementations of one
6109    /// decision agree until they do not, and the whole value of a preview is that
6110    /// it describes the operation that will actually run.
6111    async fn reconcile_supervised_modules(
6112        &self,
6113        supervisor: &Supervisor,
6114        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6115        preview: bool,
6116        requester: Principal,
6117    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6118        let plan = self.plan_rescan(configured_modules)?;
6119
6120        // Everything above this point is pure computation over two snapshots.
6121        // Everything below MUTATES. The preview returns here so the boundary is a
6122        // single early return rather than a condition repeated at each mutation
6123        // site, where one missed guard would apply part of a change the caller was
6124        // told would not happen.
6125        if preview {
6126            return Ok((plan.preview_result(), Vec::new()));
6127        }
6128        let changed = plan
6129            .configuration_changes
6130            .union(&plan.enabled_changes)
6131            .cloned()
6132            .collect::<Vec<_>>();
6133        let RescanPlan {
6134            current,
6135            configured,
6136            added,
6137            removed,
6138            changed_pending_reload,
6139            configuration_changes,
6140            enabled_changes,
6141            unchanged,
6142        } = plan;
6143        // The staged generation moves before the first change rather than
6144        // after the last, so a rescan that fails partway still invalidates
6145        // every earlier `supervisor.rescan_checked` preview: part of its diff
6146        // may have run.
6147        if !(added.is_empty()
6148            && removed.is_empty()
6149            && configuration_changes.is_empty()
6150            && enabled_changes.is_empty())
6151        {
6152            if let Some(context) = &self.rescan {
6153                context
6154                    .generation
6155                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6156            }
6157        }
6158
6159        for module_id in &removed {
6160            let module = &current
6161                .get(module_id)
6162                .expect("removed module came from current supervisor state")
6163                .0;
6164            module
6165                .retire_requested(Some(requester.clone()))
6166                .await
6167                .map_err(|err| {
6168                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
6169                })?;
6170            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
6171            //
6172            // `handle_route_open` resolves an absent module in three steps:
6173            // registry, then supervisor status, then tombstone. Retiring first
6174            // opens a window where ALL THREE ARE ABSENT -- the registry entry
6175            // went with the teardown above, the supervisor entry went with
6176            // `retire`, and the tombstone does not exist yet -- so a route.open
6177            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
6178            // it") for a module that was deliberately removed and whose caller
6179            // should get `module_removed` (TERMINAL, carrying a removal age).
6180            //
6181            // Writing the tombstone first closes it: during the window the
6182            // supervisor entry still answers, so the caller gets
6183            // `target_unavailable` -- retryable, and TRUE, because the module
6184            // is mid-teardown. After both statements it is `module_removed`.
6185            // No instant remains where a removed module reads as one that
6186            // never existed.
6187            //
6188            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6189            // the absence of a test beside a fix invites deletion: these are
6190            // two sync statements with no await between them, so reaching the
6191            // window needs a second worker thread to land exactly between them
6192            // and there is no hook to force it. MEASURED: the 25 daemon_config
6193            // tests pass identically with the old order and the new one, so
6194            // the existing suite cannot see this and a green run is not
6195            // evidence either way. What the suite does hold is the
6196            // post-condition -- a removed module answers `module_removed` --
6197            // which this preserves.
6198            //
6199            // Found by an Athena panel reading the shipped tree against a
6200            // design note (2026-09-19), as the one concrete instance of that
6201            // note's class that survived contact with source. Direction is
6202            // benign: retryable where terminal was intended, never the reverse.
6203            self.supervisor.record_rescan_removal(module_id);
6204            self.supervisor.retire(module_id);
6205            self.route_outages.forget(module_id);
6206        }
6207
6208        for module_id in configured.keys() {
6209            let Some((module, _, _, _)) = current.get(module_id) else {
6210                continue;
6211            };
6212            let configured_module = configured
6213                .get(module_id)
6214                .expect("configured module id came from configured map");
6215            if configuration_changes.contains(module_id) {
6216                module
6217                    .update_configuration(
6218                        configured_module.module_spec(),
6219                        configured_module.health.clone(),
6220                        configured_module.drain_timeout_ms,
6221                    )
6222                    .await
6223                    .map_err(|err| {
6224                        format!(
6225                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6226                        )
6227                    })?;
6228            }
6229            if enabled_changes.contains(module_id) {
6230                // A rescan that starts or stops a module applies an operator's
6231                // edit to the config, so the resulting outage was asked for.
6232                self.route_outages.mark_operator_action(module_id);
6233                module
6234                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6235                    .await
6236                    .map_err(|err| {
6237                        self.route_outages.operator_action_ended_unrefused(module_id);
6238                        format!(
6239                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6240                            configured_module.enabled
6241                        )
6242                    })?;
6243            }
6244        }
6245
6246        for module_id in &added {
6247            let configured_module = configured
6248                .get(module_id)
6249                .expect("added module id came from configured map");
6250            supervisor
6251                .supervise_configured_requested(
6252                    configured_module.module_spec(),
6253                    configured_module.enabled,
6254                    configured_module.health.clone(),
6255                    configured_module.drain_timeout_ms,
6256                    configured_module.restart,
6257                    Some(requester.clone()),
6258                )
6259                .map_err(|err| {
6260                    format!("failed to add module_id '{module_id}' during rescan: {err}")
6261                })?;
6262        }
6263
6264        Ok((
6265            SupervisorRescanResult {
6266                added,
6267                removed,
6268                changed_pending_reload,
6269                enabled_changes: enabled_changes.iter().cloned().collect(),
6270                unchanged,
6271                preview: false,
6272                // Filled by the caller, which is the only layer that can see the
6273                // previous config to diff against.
6274                restart_required: Vec::new(),
6275                capability_warnings: Vec::new(),
6276                module_actions: Vec::new(),
6277                earlier_pending: Vec::new(),
6278                expect: None,
6279            },
6280            changed,
6281        ))
6282    }
6283
6284    async fn handle_supervisor_set_enabled(
6285        &self,
6286        frame: Frame,
6287        module_id: String,
6288        enabled: bool,
6289        requester: Principal,
6290    ) -> Result<Vec<Frame>, RouterError> {
6291        let operation_lock = self.supervisor.operation_lock();
6292        let _operation_guard = operation_lock.lock().await;
6293        let Some(module) = self.supervisor.get(&module_id) else {
6294            return Ok(vec![control_error_frame(
6295                &frame,
6296                "unknown_module",
6297                format!("module_id '{module_id}' is not supervised"),
6298            )?]);
6299        };
6300
6301        // Enabling counts as well as disabling: a module an operator starts
6302        // is refused until it registers, and that wait was asked for.
6303        self.route_outages.mark_operator_action(&module_id);
6304        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6305            Ok(applied) => applied,
6306            Err(err) => {
6307                self.route_outages
6308                    .operator_action_ended_unrefused(&module_id);
6309                return Ok(vec![control_error_frame(
6310                    &frame,
6311                    "target_unavailable",
6312                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6313                )?]);
6314            }
6315        };
6316        if !applied {
6317            // Already in the requested state: nothing was made unavailable,
6318            // so the mark must not outlive this request.
6319            self.route_outages
6320                .operator_action_ended_unrefused(&module_id);
6321        }
6322
6323        self.capability_evaluator.wake_deadline_loop();
6324        self.refresh_capability_requirements();
6325        let response = ClientControlResponse::SupervisorAck { module_id, applied };
6326        Ok(vec![control_response_body_frame(
6327            &frame,
6328            &response,
6329            "ClientControlResponse::SupervisorAck",
6330        )?])
6331    }
6332
6333    async fn handle_supervisor_health_probe(
6334        &self,
6335        frame: Frame,
6336        module_id: String,
6337    ) -> Result<Vec<Frame>, RouterError> {
6338        self.refresh_capability_requirements();
6339        let Some(registration) = self
6340            .registry
6341            .get_module(&module_id)
6342            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6343        else {
6344            return Ok(vec![control_error_frame(
6345                &frame,
6346                "unknown_module",
6347                format!("module_id '{module_id}' is not registered"),
6348            )?]);
6349        };
6350
6351        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
6352        // named for it. Making `module_registration_grants_op` return false
6353        // unconditionally reddens five tests, and every one is named for something
6354        // else -- capability relay, probe/bind demultiplexing, supervision-only
6355        // probing. They exercise a successful advertisement check on the way to their
6356        // own subject.
6357        //
6358        // Real protection, fragile in a specific way: narrowing any of those tests to
6359        // focus on its stated subject would silently remove coverage nobody knows
6360        // they are carrying. Recorded here rather than as a sixth test, because the
6361        // useful fact is WHICH tests hold the guard up -- a new test would add
6362        // coverage without telling the next person what the existing ones quietly do.
6363        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6364        {
6365            return Ok(vec![control_error_frame(
6366                &frame,
6367                "health_not_advertised",
6368                format!("module_id '{module_id}' did not advertise health.check"),
6369            )?]);
6370        }
6371
6372        let deadline = Instant::now() + self.health_probe_timeout;
6373        let pending = match self.forwarding.begin_module_control_rpc_for(
6374            &module_id,
6375            MODULE_CONTROL_OP_HEALTH_CHECK,
6376            deadline,
6377        ) {
6378            Ok(pending) => pending,
6379            Err(err) => {
6380                return Ok(vec![control_error_frame(
6381                    &frame,
6382                    forwarding_error_code(&err),
6383                    err.to_string(),
6384                )?])
6385            }
6386        };
6387
6388        let PendingModuleControlRpc {
6389            endpoint,
6390            module_sink,
6391            negotiated_ver,
6392            corr: probe_corr,
6393            receiver,
6394        } = pending;
6395        let mut guard =
6396            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6397        let probe_body =
6398            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6399                RouterError::backend(
6400                    0,
6401                    frame.header.corr,
6402                    format!("failed to encode health.check request: {err}"),
6403                )
6404            })?;
6405        let probe_frame = Frame::build_with_version(
6406            negotiated_ver,
6407            FrameType::Request,
6408            control_flags(),
6409            0,
6410            0,
6411            probe_corr,
6412            probe_body,
6413        )
6414        .map_err(RouterError::FrameBuild)?;
6415
6416        // Queue admission and the reply share one deadline, so a module that
6417        // stops reading cannot park the requesting connection's reader forever.
6418        match timeout_at(deadline, module_sink.send(probe_frame)).await {
6419            Ok(Ok(())) => {}
6420            Ok(Err(err)) => {
6421                return Ok(vec![control_error_frame(
6422                    &frame,
6423                    "target_unavailable",
6424                    err.to_string(),
6425                )?]);
6426            }
6427            Err(_) => {
6428                return Ok(vec![control_error_frame(
6429                    &frame,
6430                    "module_timeout",
6431                    format!(
6432                        "module_id '{module_id}' did not answer health.check within {:?}",
6433                        self.health_probe_timeout
6434                    ),
6435                )?]);
6436            }
6437        }
6438
6439        match timeout_at(deadline, receiver).await {
6440            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6441                guard.disarm();
6442                let Some(report) = response.health_report() else {
6443                    return Ok(vec![control_error_frame(
6444                        &frame,
6445                        "invalid_control_body",
6446                        "health.check RPC returned a non-health response",
6447                    )?]);
6448                };
6449                // Metrics go out whole here. The supervisor's cached snapshot
6450                // caps this blob (see truncate_health_metrics), and this path
6451                // exists precisely to answer without that cap -- so applying it
6452                // here would leave no way to see what the cached view drops.
6453                let HealthReport {
6454                    status,
6455                    detail,
6456                    metrics,
6457                } = report;
6458                let capability_detail = self
6459                    .capability_evaluator
6460                    .required_problem_detail(&module_id);
6461                let response = ClientControlResponse::SupervisorHealthProbe {
6462                    module_id,
6463                    status,
6464                    detail: append_capability_problem_detail(detail, capability_detail),
6465                    metrics,
6466                };
6467                Ok(vec![control_response_body_frame(
6468                    &frame,
6469                    &response,
6470                    "ClientControlResponse::SupervisorHealthProbe",
6471                )?])
6472            }
6473            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6474                guard.disarm();
6475                Ok(vec![control_error_body_frame(&frame, body)?])
6476            }
6477            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6478                guard.disarm();
6479                Ok(vec![control_error_frame(
6480                    &frame,
6481                    "target_unavailable",
6482                    message,
6483                )?])
6484            }
6485            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6486                guard.disarm();
6487                Ok(vec![control_error_frame(
6488                    &frame,
6489                    "invalid_control_body",
6490                    message,
6491                )?])
6492            }
6493            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6494                guard.disarm();
6495                Ok(vec![control_error_frame(
6496                    &frame,
6497                    "invalid_control_body",
6498                    format!("expected module-control op '{expected}', got '{actual}'"),
6499                )?])
6500            }
6501            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6502                guard.disarm();
6503                Ok(vec![control_error_frame(
6504                    &frame,
6505                    "module_timeout",
6506                    format!(
6507                        "module_id '{module_id}' answered health.check after {:?}",
6508                        self.health_probe_timeout
6509                    ),
6510                )?])
6511            }
6512            Ok(Err(_)) => Ok(vec![control_error_frame(
6513                &frame,
6514                "target_unavailable",
6515                "health.check waiter was canceled before the module responded",
6516            )?]),
6517            Err(_) => Ok(vec![control_error_frame(
6518                &frame,
6519                "module_timeout",
6520                format!(
6521                    "module_id '{module_id}' did not answer health.check within {:?}",
6522                    self.health_probe_timeout
6523                ),
6524            )?]),
6525        }
6526    }
6527
6528    fn supervisor_status(
6529        &self,
6530        module_id: &str,
6531        corr: u64,
6532    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6533        self.supervisor
6534            .get(module_id)
6535            .map(|module| {
6536                let warming = module.is_warming_for_control("status").map_err(|err| {
6537                    RouterError::backend(
6538                        0,
6539                        corr,
6540                        format!(
6541                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
6542                        ),
6543                    )
6544                })?;
6545                module.status_for_control("status").map_err(|err| {
6546                    RouterError::backend(
6547                        0,
6548                        corr,
6549                        format!(
6550                            "failed to read supervisor status for module_id '{module_id}': {err}"
6551                        ),
6552                    )
6553                }).map(|status| (status, warming))
6554            })
6555            .transpose()
6556    }
6557
6558    fn guard_module_control_op(
6559        &self,
6560        frame: &Frame,
6561        module_id: &str,
6562        op: &str,
6563    ) -> Result<Option<Frame>, RouterError> {
6564        if self.module_grants_op(module_id, op, frame.header.corr)? {
6565            return Ok(None);
6566        }
6567
6568        Ok(Some(control_error_frame(
6569            frame,
6570            "op_not_allowed",
6571            format!("module_id '{module_id}' did not grant control op '{op}'"),
6572        )?))
6573    }
6574
6575    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6576        let Some(registration) = self
6577            .registry
6578            .get_module(module_id)
6579            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6580        else {
6581            return Ok(false);
6582        };
6583        Ok(module_registration_grants_op(&registration.control_ops, op))
6584    }
6585
6586    fn handle_status_update(
6587        &self,
6588        endpoint: ModuleEndpointId,
6589        frame: Frame,
6590    ) -> Result<Vec<Frame>, RouterError> {
6591        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6592            Ok(update) => update,
6593            Err(err) => {
6594                // Forward-compat: a newer module may push a channel-0 op this subc
6595                // version doesn't know. The control contract says unknown push ops
6596                // are IGNORED, never answered with an error. Only a malformed body
6597                // for an op we DO know is a real error worth surfacing.
6598                if is_known_module_push_op(&frame.body) {
6599                    return Ok(vec![control_error_frame(
6600                        &frame,
6601                        "invalid_control_body",
6602                        format!("malformed module control push body: {err}"),
6603                    )?]);
6604                }
6605                return Ok(Vec::new());
6606            }
6607        };
6608
6609        match update {
6610            ModuleControlPush::RouteStatus {
6611                route_channel,
6612                route_epoch,
6613                status,
6614            } => {
6615                self.forwarding
6616                    .cache_status(endpoint, route_channel, route_epoch, status)
6617                    .map_err(RouterError::Forwarding)?;
6618            }
6619        }
6620        Ok(Vec::new())
6621    }
6622
6623    fn handle_route_poll(
6624        &self,
6625        ctx: &RouteCtx,
6626        frame: Frame,
6627        route_channel: u16,
6628        route_epoch: u32,
6629        kind: PollKind,
6630    ) -> Result<Vec<Frame>, RouterError> {
6631        let snapshot = self
6632            .forwarding
6633            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6634            .map_err(RouterError::Forwarding)?;
6635        let response = match (kind, snapshot) {
6636            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6637                ClientControlResponse::RoutePoll {
6638                    route_channel,
6639                    route_epoch,
6640                    status,
6641                    live: None,
6642                }
6643            }
6644            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6645                route_channel,
6646                route_epoch,
6647                status: None,
6648                live: None,
6649            },
6650            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6651                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
6652                // is what makes reporting `true` correct rather than a
6653                // confident guess. `process_live` returns None only when the
6654                // module id has no supervisor snapshot at all -- an
6655                // externally-started module the daemon did not spawn -- and
6656                // for those the supervisor has no opinion to offer, ever. It
6657                // is never None for a supervised module in an unknown state:
6658                // a supervised module always has a snapshot, and the answer
6659                // comes from `state == Running && process_alive`.
6660                //
6661                // The route is Bound, so the module completed a HELLO on a
6662                // live connection; "the process this route points at is
6663                // running" is therefore attested by the binding rather than
6664                // assumed. Reporting `false` for an unsupervised module would
6665                // be the actual lie -- it would tell a client its healthy
6666                // route is dead because the daemon does not manage the
6667                // process.
6668                //
6669                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
6670                // module whose liveness is genuinely unknown, e.g. a snapshot
6671                // that has not been populated yet -- THIS DEFAULT BECOMES
6672                // WRONG and must split: unsupervised stays true, unknown
6673                // becomes null so the client can tell the two apart. The
6674                // response field is already `Option<bool>`, so the wire can
6675                // carry that distinction today.
6676                let live = self
6677                    .process_liveness
6678                    .as_ref()
6679                    .and_then(|source| source.process_live(&module_id))
6680                    .unwrap_or(true);
6681                ClientControlResponse::RoutePoll {
6682                    route_channel,
6683                    route_epoch,
6684                    status: None,
6685                    live: Some(live),
6686                }
6687            }
6688            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6689                route_channel,
6690                route_epoch,
6691                status: None,
6692                live: Some(false),
6693            },
6694        };
6695
6696        Ok(vec![control_response_body_frame(
6697            &frame,
6698            &response,
6699            "ClientControlResponse::RoutePoll",
6700        )?])
6701    }
6702
6703    pub(crate) fn observe_module_control_completion(
6704        &self,
6705        completion: ModuleControlRpcCompletion,
6706    ) -> bool {
6707        match completion {
6708            ModuleControlRpcCompletion::Unknown => false,
6709            ModuleControlRpcCompletion::Settled => true,
6710            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6711                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6712                info!(
6713                    module_id = %module_id,
6714                    latency_ms,
6715                    "late health.check answer proves the module is alive"
6716                );
6717                match self
6718                    .supervisor
6719                    .record_late_health_answer(&module_id, latency_ms)
6720                {
6721                    Ok(true) => {}
6722                    Ok(false) => debug!(
6723                        module_id = %module_id,
6724                        latency_ms,
6725                        "late health.check answer has no active supervisor snapshot"
6726                    ),
6727                    Err(err) => warn!(
6728                        module_id = %module_id,
6729                        latency_ms,
6730                        error = %err,
6731                        "failed to record late health.check answer"
6732                    ),
6733                }
6734                true
6735            }
6736        }
6737    }
6738
6739    /// Decide whether a failure while settling a relayed `route.bind` belongs to
6740    /// the module connection whose frame is being handled, or to the client that
6741    /// relay was opened for.
6742    ///
6743    /// This runs on the MODULE connection's frame handler, where returning `Err`
6744    /// ends that connection -- and a module connection carries every client's
6745    /// routes to that module, so ending it costs the whole fleet its tools.
6746    /// `ConnectionClosing` carries the id of the connection that is closing, and
6747    /// when that id is a CLIENT's, the condition is entirely about that one
6748    /// client's route.open. A client-scoped condition has no authority over a
6749    /// shared module connection, so it is logged and the single relay is dropped:
6750    /// the client is going away, and `complete_pending_relay` already removed the
6751    /// relay before failing, so there is nothing left to settle. Anything that
6752    /// relay still reserved is released by that client's own connection teardown,
6753    /// which is already under way -- that is what "closing" means.
6754    ///
6755    /// Every other failure is a statement about THIS connection and stays fatal:
6756    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
6757    /// id in `ConnectionClosing` all mean this connection cannot keep serving
6758    /// frames correctly.
6759    fn refuse_to_end_module_connection_for_a_client(
6760        &self,
6761        module_connection_id: ConnectionId,
6762        corr: u64,
6763        err: ForwardingError,
6764    ) -> Result<(), RouterError> {
6765        if let ForwardingError::ConnectionClosing { connection_id } = err {
6766            if connection_id != module_connection_id {
6767                warn!(
6768                    module_connection_id = module_connection_id.get(),
6769                    client_connection_id = connection_id.get(),
6770                    corr,
6771                    "dropping a route.bind response for a closing client; the module connection keeps serving"
6772                );
6773                return Ok(());
6774            }
6775        }
6776        Err(RouterError::Forwarding(err))
6777    }
6778
6779    fn handle_module_relay_response(
6780        &self,
6781        connection_id: ConnectionId,
6782        frame: Frame,
6783    ) -> Result<Vec<Frame>, RouterError> {
6784        let mut secondary_error = None;
6785        let outcome = match frame.header.ty {
6786            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6787                Ok(probe) if probe.op == "route.bind" => {
6788                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6789                        Ok(ModuleControlResponse::RouteBindAck {}) => {
6790                            RouteBindRelayOutcome::Accepted
6791                        }
6792                        Ok(other) => {
6793                            let message =
6794                                format!("route.bind response carried unexpected body: {other:?}");
6795                            secondary_error = Some(control_error_frame(
6796                                &frame,
6797                                "invalid_control_body",
6798                                message.clone(),
6799                            )?);
6800                            RouteBindRelayOutcome::ModuleGone(message)
6801                        }
6802                        Err(err) => {
6803                            let message = format!("malformed route.bind response body: {err}");
6804                            secondary_error = Some(control_error_frame(
6805                                &frame,
6806                                "invalid_control_body",
6807                                message.clone(),
6808                            )?);
6809                            RouteBindRelayOutcome::ModuleGone(message)
6810                        }
6811                    }
6812                }
6813                Ok(probe) => {
6814                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6815                    {
6816                        Ok(response) => ModuleControlRpcOutcome::Response(response),
6817                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6818                            "malformed {} response body: {err}",
6819                            probe.op
6820                        )),
6821                    };
6822                    let completion = self
6823                        .forwarding
6824                        .complete_module_control_rpc(
6825                            connection_id,
6826                            frame.header.corr,
6827                            Some(&probe.op),
6828                            outcome,
6829                        )
6830                        .map_err(RouterError::Forwarding)?;
6831                    if !self.observe_module_control_completion(completion) {
6832                        debug!(
6833                            connection_id = connection_id.get(),
6834                            corr = frame.header.corr,
6835                            op = %probe.op,
6836                            "dropping late or unknown module-control RPC response"
6837                        );
6838                    }
6839                    return Ok(Vec::new());
6840                }
6841                Err(err) => {
6842                    if let Some(expected_op) = self
6843                        .forwarding
6844                        .pending_module_control_op(connection_id, frame.header.corr)
6845                        .map_err(RouterError::Forwarding)?
6846                    {
6847                        let completion = self
6848                            .forwarding
6849                            .complete_module_control_rpc(
6850                                connection_id,
6851                                frame.header.corr,
6852                                None,
6853                                ModuleControlRpcOutcome::MalformedResponse(format!(
6854                                    "malformed {expected_op} response body: {err}"
6855                                )),
6856                            )
6857                            .map_err(RouterError::Forwarding)?;
6858                        if !self.observe_module_control_completion(completion) {
6859                            debug!(
6860                                connection_id = connection_id.get(),
6861                                corr = frame.header.corr,
6862                                "dropping late malformed module-control RPC response"
6863                            );
6864                        }
6865                        return Ok(Vec::new());
6866                    }
6867                    let message = format!("malformed route.bind response body: {err}");
6868                    secondary_error = Some(control_error_frame(
6869                        &frame,
6870                        "invalid_control_body",
6871                        message.clone(),
6872                    )?);
6873                    RouteBindRelayOutcome::ModuleGone(message)
6874                }
6875            },
6876            FrameType::Error => {
6877                if self
6878                    .forwarding
6879                    .pending_module_control_op(connection_id, frame.header.corr)
6880                    .map_err(RouterError::Forwarding)?
6881                    .is_some()
6882                {
6883                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6884                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6885                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6886                            "malformed module-control ERROR body: {err}"
6887                        )),
6888                    };
6889                    let completion = self
6890                        .forwarding
6891                        .complete_module_control_rpc(
6892                            connection_id,
6893                            frame.header.corr,
6894                            None,
6895                            outcome,
6896                        )
6897                        .map_err(RouterError::Forwarding)?;
6898                    if !self.observe_module_control_completion(completion) {
6899                        debug!(
6900                            connection_id = connection_id.get(),
6901                            corr = frame.header.corr,
6902                            "dropping late or unknown module-control RPC error"
6903                        );
6904                    }
6905                    return Ok(Vec::new());
6906                }
6907                match serde_json::from_slice::<ErrorBody>(&frame.body) {
6908                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
6909                    Err(err) => {
6910                        let message = format!("malformed route.bind ERROR body: {err}");
6911                        secondary_error = Some(control_error_frame(
6912                            &frame,
6913                            "invalid_control_body",
6914                            message.clone(),
6915                        )?);
6916                        RouteBindRelayOutcome::ModuleGone(message)
6917                    }
6918                }
6919            }
6920            ty => {
6921                return Ok(vec![control_error_frame(
6922                    &frame,
6923                    "unsupported_control_frame",
6924                    format!("unsupported module channel-0 frame {ty:?}"),
6925                )?])
6926            }
6927        };
6928
6929        let settled =
6930            self.forwarding
6931                .complete_pending_relay(connection_id, frame.header.corr, outcome);
6932        let completion = match settled {
6933            Ok(completion) => completion,
6934            Err(err) => {
6935                self.refuse_to_end_module_connection_for_a_client(
6936                    connection_id,
6937                    frame.header.corr,
6938                    err,
6939                )?;
6940                return Ok(secondary_error.into_iter().collect());
6941            }
6942        };
6943        if let Some(target) = completion.abandoned.as_ref() {
6944            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6945        }
6946        if !completion.settled {
6947            debug!(
6948                connection_id = connection_id.get(),
6949                corr = frame.header.corr,
6950                frame_type = ?frame.header.ty,
6951                "dropping late or unknown route.bind relay response"
6952            );
6953        }
6954        Ok(secondary_error.into_iter().collect())
6955    }
6956
6957    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6958        debug!(connection_id = connection_id.get(), "handling GOODBYE");
6959        // GOODBYE ends the connection's logical session even when its socket
6960        // stays open. Use disconnect teardown so verdicts, client notices and
6961        // scope authority are released at the same lifecycle boundary.
6962        self.cleanup_connection_with_end_reason(
6963            connection_id,
6964            RegistrationEndReason::ExplicitGoodbye,
6965        )
6966        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6967        Ok(Vec::new())
6968    }
6969}
6970
6971impl Default for ControlHandler {
6972    fn default() -> Self {
6973        Self::new(Arc::new(Registry::default()))
6974    }
6975}
6976
6977impl crate::supervise::SwapPromotionObserver for ControlHandler {
6978    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6979        self.apply_registration_capabilities(registration);
6980    }
6981}
6982
6983fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6984    CapabilityRequirementStatus {
6985        consumer: status.consumer,
6986        capability: status.capability,
6987        need: match status.need {
6988            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6989            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6990        },
6991        verdict: status.verdict.as_str().to_string(),
6992        episode_seq: status.episode_seq,
6993        config_satisfiable: status.config_satisfiable,
6994        runtime_available: status.runtime_available,
6995        detail: status.detail,
6996    }
6997}
6998
6999fn append_capability_problem_detail(
7000    detail: Option<String>,
7001    capability_detail: Option<String>,
7002) -> Option<String> {
7003    match (detail, capability_detail) {
7004        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7005        (Some(detail), None) => Some(detail),
7006        (None, Some(capability_detail)) => Some(capability_detail),
7007        (None, None) => None,
7008    }
7009}
7010
7011fn subc_ops() -> Vec<String> {
7012    SUBC_CONTROL_OPS
7013        .iter()
7014        .map(|op| (*op).to_string())
7015        .collect()
7016}
7017
7018fn module_subc_ops() -> Vec<String> {
7019    SUBC_CONTROL_OPS
7020        .iter()
7021        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7022        .map(|op| (*op).to_string())
7023        .collect()
7024}
7025
7026#[cfg(test)]
7027fn module_baseline_control_ops() -> Vec<String> {
7028    MODULE_BASELINE_CONTROL_OPS
7029        .iter()
7030        .map(|op| (*op).to_string())
7031        .collect()
7032}
7033
7034fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7035    let mut seen = HashSet::new();
7036    let mut effective = Vec::new();
7037    for op in MODULE_BASELINE_CONTROL_OPS {
7038        if seen.insert((*op).to_string()) {
7039            effective.push((*op).to_string());
7040        }
7041    }
7042    for op in declared.unwrap_or_default() {
7043        if seen.insert(op.clone()) {
7044            effective.push(op);
7045        }
7046    }
7047    effective
7048}
7049
7050fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7051    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7052}
7053
7054fn target_module_id(target: &RouteTarget) -> &str {
7055    match target {
7056        RouteTarget::ToolProvider { module_id }
7057        | RouteTarget::ManagementSurface { module_id }
7058        | RouteTarget::InternalService { module_id, .. } => module_id,
7059    }
7060}
7061
7062fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7063    roles.iter().any(|role| match (target, role) {
7064        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7065        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7066        (
7067            RouteTarget::InternalService { service_id, .. },
7068            ProviderRole::InternalService {
7069                service_id: provided,
7070                ..
7071            },
7072        ) => service_id == provided,
7073        _ => false,
7074    })
7075}
7076
7077fn is_routable_role(role: &ProviderRole) -> bool {
7078    matches!(
7079        role,
7080        ProviderRole::ToolProvider { .. }
7081            | ProviderRole::ManagementSurface { .. }
7082            | ProviderRole::InternalService { .. }
7083    )
7084}
7085
7086#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7087enum ControlRequestBodyError {
7088    UnknownOp,
7089    InvalidBody,
7090}
7091
7092#[derive(Debug, Deserialize)]
7093struct ControlOpProbe {
7094    op: String,
7095}
7096
7097/// Channel-0 push ops this subc version understands. A push whose `op` is not in
7098/// this set is treated as a forward-compat unknown and ignored rather than errored.
7099const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7100
7101fn is_known_module_push_op(body: &[u8]) -> bool {
7102    serde_json::from_slice::<ControlOpProbe>(body)
7103        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7104        .unwrap_or(false)
7105}
7106
7107fn is_known_module_request_op(body: &[u8]) -> bool {
7108    serde_json::from_slice::<ControlOpProbe>(body)
7109        .map(|probe| is_module_to_subc_op(&probe.op))
7110        .unwrap_or(false)
7111}
7112
7113fn is_module_to_subc_op(op: &str) -> bool {
7114    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7115}
7116
7117fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7118    debug!(
7119        op = %op,
7120        connection_id = connection_id.get(),
7121        corr,
7122        "control dispatch"
7123    );
7124}
7125
7126fn log_slow_control_dispatch(
7127    dispatch_started_at: Option<StdInstant>,
7128    op: &'static str,
7129    connection_id: ConnectionId,
7130    corr: u64,
7131    route_open_timings: Option<&RouteOpenTimings>,
7132) {
7133    let Some(dispatch_started_at) = dispatch_started_at else {
7134        return;
7135    };
7136    // Use the same monotonic clock as the phase timers and relay deadline.
7137    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7138    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7139        if let Some(timings) = route_open_timings {
7140            warn!(
7141                op = %op,
7142                connection_id = connection_id.get(),
7143                corr,
7144                elapsed_ms = elapsed.as_millis() as u64,
7145                module_id = ?timings.module_id,
7146                principal = %timings.principal,
7147                admission_ms = timings.admission.as_millis() as u64,
7148                project_root_ms = timings.project_root.as_millis() as u64,
7149                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7150                bind_send_ms = timings.bind_send.as_millis() as u64,
7151                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7152                "slow control dispatch"
7153            );
7154            return;
7155        }
7156        warn!(
7157            op = %op,
7158            connection_id = connection_id.get(),
7159            corr,
7160            elapsed_ms = elapsed.as_millis() as u64,
7161            "slow control dispatch"
7162        );
7163    }
7164}
7165
7166/// A test stand-in for the OS caller lookup, given the recorded endpoints.
7167#[cfg(test)]
7168type TestCallerLookup =
7169    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7170
7171/// Every new client operation needs an explicit privilege decision here.
7172fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7173    match request {
7174        ClientControlRequest::SupervisorRestart { module_id, .. } => {
7175            Some(PrivilegedOperation::Restart { module_id })
7176        }
7177        ClientControlRequest::SupervisorSwap { module_id, .. } => {
7178            Some(PrivilegedOperation::Swap { module_id })
7179        }
7180        ClientControlRequest::SupervisorReload { module_id } => {
7181            Some(PrivilegedOperation::Reload { module_id })
7182        }
7183        ClientControlRequest::SupervisorRescan { preview } => {
7184            if *preview {
7185                None
7186            } else {
7187                Some(PrivilegedOperation::Rescan)
7188            }
7189        }
7190        // Only the well-formed apply asks for approval. A preview changes
7191        // nothing, and any other combination is refused before it touches
7192        // anything, so prompting the operator for it would ask about a request
7193        // that cannot run.
7194        ClientControlRequest::SupervisorRescanChecked {
7195            preview,
7196            candidate,
7197            expect,
7198        } => (!*preview && candidate.is_none() && expect.is_some())
7199            .then_some(PrivilegedOperation::Rescan),
7200        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7201            Some(PrivilegedOperation::SetEnabled {
7202                module_id,
7203                enabled: *enabled,
7204            })
7205        }
7206        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7207            Some(PrivilegedOperation::ReleaseReserved { module_id })
7208        }
7209        ClientControlRequest::ServerDescribe {}
7210        | ClientControlRequest::CatalogList { .. }
7211        | ClientControlRequest::RouteOpen { .. }
7212        | ClientControlRequest::RoutePoll { .. }
7213        | ClientControlRequest::SupervisorList {}
7214        | ClientControlRequest::SupervisorHealth {}
7215        | ClientControlRequest::SupervisorHealthProbe { .. }
7216        | ClientControlRequest::SupervisorRoutes { .. }
7217        | ClientControlRequest::SupervisorProvenance { .. }
7218        | ClientControlRequest::SupervisorStderrTail { .. }
7219        | ClientControlRequest::SupervisorTerminals { .. }
7220        | ClientControlRequest::SupervisorSpawnSnapshot {}
7221        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7222    }
7223}
7224
7225fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7226    match request {
7227        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7228        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7229        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7230        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7231        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7232        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7233        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7234        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7235        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7236        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7237        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7238        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7239        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7240        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7241        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7242        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7243        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7244        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7245        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7246        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7247    }
7248}
7249
7250/// The `scope=` field of the `route.open accepted` line, from the stamp that
7251/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
7252/// so a ref holding control characters cannot forge a log line, and cut to 64
7253/// characters with a trailing `…` to bound the line. The bind proof and the
7254/// scope's attributes never appear.
7255fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7256    use subc_protocol::scope::ScopeKind;
7257    const MAX_REF_CHARS: usize = 64;
7258    let kind = match stamp.kind {
7259        ScopeKind::Head => "head",
7260        ScopeKind::Worker => "worker",
7261        ScopeKind::Ephemeral => "ephemeral",
7262    };
7263    let mut scope_ref: String = stamp.scope_ref.chars().take(MAX_REF_CHARS).collect();
7264    if stamp.scope_ref.chars().nth(MAX_REF_CHARS).is_some() {
7265        scope_ref.push('…');
7266    }
7267    format!(
7268        "{kind}/{}/{scope_ref:?}@{}",
7269        principal_label(&stamp.owner),
7270        stamp.scope_epoch
7271    )
7272}
7273
7274fn principal_label(principal: &Principal) -> String {
7275    match principal {
7276        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7277        Principal::Direct => "direct".to_string(),
7278        other => format!("{other:?}"),
7279    }
7280}
7281
7282fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7283    match request {
7284        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7285        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7286        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7287        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7288        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7289    }
7290}
7291
7292fn parse_client_control_request(
7293    body: &[u8],
7294) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7295    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7296        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7297            Ok(probe)
7298                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7299                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7300            {
7301                ControlRequestBodyError::InvalidBody
7302            }
7303            Ok(_) => ControlRequestBodyError::UnknownOp,
7304            Err(_) => ControlRequestBodyError::InvalidBody,
7305        };
7306        (err, classification)
7307    })
7308}
7309
7310fn parse_module_control_request_from_module(
7311    body: &[u8],
7312) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7313    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7314        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7315            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7316            Ok(_) => ControlRequestBodyError::UnknownOp,
7317            Err(_) => ControlRequestBodyError::InvalidBody,
7318        };
7319        (err, classification)
7320    })
7321}
7322
7323#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7324enum ProviderRoleKind {
7325    ToolProvider,
7326    PipelineStage,
7327    ManagementSurface,
7328    InternalService,
7329}
7330
7331fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7332    match role {
7333        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7334        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7335        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7336        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7337    }
7338}
7339
7340fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7341    roles.iter().map(provider_role_kind).collect()
7342}
7343
7344/// Most refused scope records named individually in the log per sync; the
7345/// `refused` count on the accepted line is always complete.
7346const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7347
7348/// Per-outcome counts of one accepted `scope.sync`, for its log line.
7349#[derive(Debug, Default, PartialEq, Eq)]
7350struct ScopeOutcomeCounts {
7351    created: usize,
7352    replaced: usize,
7353    updated: usize,
7354    unchanged: usize,
7355    refused: usize,
7356}
7357
7358impl ScopeOutcomeCounts {
7359    fn of(results: &[ScopeRecordResult]) -> Self {
7360        let mut counts = Self::default();
7361        for result in results {
7362            let slot = match result.outcome {
7363                ScopeRecordOutcome::Created => &mut counts.created,
7364                ScopeRecordOutcome::Replaced => &mut counts.replaced,
7365                ScopeRecordOutcome::Updated => &mut counts.updated,
7366                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7367                ScopeRecordOutcome::Refused => &mut counts.refused,
7368            };
7369            *slot += 1;
7370        }
7371        counts
7372    }
7373}
7374
7375#[cfg(test)]
7376mod scope_outcome_count_tests {
7377    use super::*;
7378
7379    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7380        ScopeRecordResult {
7381            scope_ref: "r".to_string(),
7382            scope_epoch: 1,
7383            outcome,
7384            code: None,
7385            message: None,
7386            version: None,
7387            parent_state: None,
7388        }
7389    }
7390
7391    /// Each outcome lands in its own count, so a refused record can never be
7392    /// hidden inside the total the log already printed.
7393    #[test]
7394    fn every_outcome_is_counted_in_its_own_field() {
7395        let results = [
7396            result(ScopeRecordOutcome::Created),
7397            result(ScopeRecordOutcome::Created),
7398            result(ScopeRecordOutcome::Replaced),
7399            result(ScopeRecordOutcome::Updated),
7400            result(ScopeRecordOutcome::Unchanged),
7401            result(ScopeRecordOutcome::Refused),
7402            result(ScopeRecordOutcome::Refused),
7403            result(ScopeRecordOutcome::Refused),
7404        ];
7405        assert_eq!(
7406            ScopeOutcomeCounts::of(&results),
7407            ScopeOutcomeCounts {
7408                created: 2,
7409                replaced: 1,
7410                updated: 1,
7411                unchanged: 1,
7412                refused: 3,
7413            }
7414        );
7415    }
7416}
7417
7418/// Return whether a catalog change can create a newly violating live route.
7419/// Removing an attested claim is intentionally excluded: it makes fewer routes
7420/// forbidden and therefore must leave the existing route census untouched.
7421fn capability_census_trigger(
7422    old: Option<&CapabilityDeclarations>,
7423    new: Option<&CapabilityDeclarations>,
7424) -> bool {
7425    let old_provides = old
7426        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7427        .unwrap_or_default();
7428    let old_denies = old
7429        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7430        .unwrap_or_default();
7431    let new = new.cloned().unwrap_or(CapabilityDeclarations {
7432        provides: Vec::new(),
7433        requires: Vec::new(),
7434        must_never_reach: Vec::new(),
7435    });
7436
7437    new.provides
7438        .iter()
7439        .any(|capability| !old_provides.contains(capability))
7440        || new
7441            .must_never_reach
7442            .iter()
7443            .any(|capability| !old_denies.contains(capability))
7444}
7445
7446/// Find the first capability an attested opener denies that an attested target
7447/// claims. Both manifests are live registry records, never cached or client data.
7448fn denied_capability<'a>(
7449    opening_manifest: &'a ModuleManifest,
7450    target_manifest: &ModuleManifest,
7451) -> Option<&'a str> {
7452    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7453    let target_capabilities = target_manifest.capabilities.as_ref()?;
7454    opening_capabilities
7455        .must_never_reach
7456        .iter()
7457        .find(|denied| {
7458            target_capabilities
7459                .provides
7460                .iter()
7461                .any(|provided| provided == *denied)
7462        })
7463        .map(String::as_str)
7464}
7465
7466fn catalog_update_frozen_field_message(
7467    registered: &ModuleManifest,
7468    provides: &[ProviderRole],
7469) -> Option<String> {
7470    let old_has_provides = !registered.provides.is_empty();
7471    let new_has_provides = !provides.is_empty();
7472    if old_has_provides != new_has_provides {
7473        return Some(format!(
7474            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7475            registered.module_id
7476        ));
7477    }
7478
7479    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
7480        return Some(format!(
7481            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7482            registered.module_id
7483        ));
7484    }
7485
7486    let registered_concurrency = manifest_concurrency(registered);
7487    let mut candidate = registered.clone();
7488    candidate.provides = provides.to_vec();
7489    let candidate_concurrency = manifest_concurrency(&candidate);
7490    if candidate_concurrency != registered_concurrency {
7491        return Some(format!(
7492            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7493            registered.module_id, registered_concurrency, candidate_concurrency
7494        ));
7495    }
7496
7497    // control_ops live beside the manifest in the HELLO body, not inside
7498    // ModuleManifest, so a provides-only catalog.update cannot change them.
7499    None
7500}
7501
7502fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7503    manifest.provides.iter().any(is_routable_role)
7504}
7505
7506/// Returns the routable-provider concurrency subc should enforce for this manifest.
7507///
7508/// ToolProvider and ManagementSurface store their delivery concurrency directly.
7509/// InternalService has no role-specific concurrency field, so it retains the
7510/// existing ModuleManaged default for backward compatibility.
7511fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7512    manifest
7513        .provides
7514        .iter()
7515        .find_map(|provider| match provider {
7516            ProviderRole::ToolProvider { concurrency, .. }
7517            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7518            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7519        })
7520        .unwrap_or(Concurrency::ModuleManaged)
7521}
7522
7523/// True when the manifest carries a ManagementSurface role whose concurrency
7524/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
7525/// bytes because the typed manifest deliberately erases that distinction: the
7526/// default exists for wire compatibility, and this probe exists so the default
7527/// stays observable. Any parse irregularity returns false -- the caller only
7528/// logs, and a malformed body already failed registration upstream.
7529fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7530    let has_management_surface = manifest
7531        .provides
7532        .iter()
7533        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7534    if !has_management_surface {
7535        return false;
7536    }
7537    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7538        return false;
7539    };
7540    let Some(provides) = raw
7541        .get("manifest")
7542        .and_then(|manifest| manifest.get("provides"))
7543        .and_then(serde_json::Value::as_array)
7544    else {
7545        return false;
7546    };
7547    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
7548    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
7549    // against the management_surface_manifest_without_concurrency golden, not
7550    // recalled (the externally-tagged guess was this function's first bug).
7551    provides.iter().any(|role| {
7552        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7553            && role.get("concurrency").is_none()
7554    })
7555}
7556
7557fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7558    if peer_version != PROTOCOL_VERSION {
7559        return Err(format!(
7560            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7561        ));
7562    }
7563    Ok(PROTOCOL_VERSION)
7564}
7565
7566fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7567    Frame::build_with_version(
7568        response_version(frame),
7569        FrameType::Pong,
7570        frame.header.flags,
7571        0,
7572        0,
7573        frame.header.corr,
7574        Vec::new(),
7575    )
7576    .map_err(RouterError::FrameBuild)
7577}
7578
7579pub(crate) fn control_error_frame(
7580    frame: &Frame,
7581    code: &'static str,
7582    message: impl Into<String>,
7583) -> Result<Frame, RouterError> {
7584    control_error_body_frame(
7585        frame,
7586        ErrorBody {
7587            code: code.to_string(),
7588            message: message.into(),
7589            detail: None,
7590        },
7591    )
7592}
7593
7594fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7595    let body = serde_json::to_vec(&error).map_err(|err| {
7596        RouterError::backend(
7597            0,
7598            frame.header.corr,
7599            format!("failed to encode control ERROR: {err}"),
7600        )
7601    })?;
7602
7603    Frame::build_with_version(
7604        response_version(frame),
7605        FrameType::Error,
7606        control_flags(),
7607        0,
7608        0,
7609        frame.header.corr,
7610        body,
7611    )
7612    .map_err(RouterError::FrameBuild)
7613}
7614
7615fn control_response_body_frame<T: Serialize>(
7616    frame: &Frame,
7617    reply: &T,
7618    label: &'static str,
7619) -> Result<Frame, RouterError> {
7620    let body = serde_json::to_vec(reply).map_err(|err| {
7621        RouterError::backend(
7622            0,
7623            frame.header.corr,
7624            format!("failed to encode {label}: {err}"),
7625        )
7626    })?;
7627
7628    Frame::build_with_version(
7629        response_version(frame),
7630        FrameType::Response,
7631        control_flags(),
7632        0,
7633        0,
7634        frame.header.corr,
7635        body,
7636    )
7637    .map_err(RouterError::FrameBuild)
7638}
7639
7640/// Map a forwarding failure to the wire code a client sees.
7641///
7642/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
7643/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
7644/// chosen here decides whether a caller retries or gives up.
7645///
7646/// That makes attribution the load-bearing property, not merely having a code. A
7647/// permanent fault published as a retryable one produces a fleet-wide retry storm
7648/// against something that can never recover; a transient fault published as
7649/// permanent gives up on work that would have succeeded. Both look correct in a
7650/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
7651/// the mapping per variant rather than merely asserting that some code exists.
7652///
7653/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
7654/// two codes on the same side of the boundary passes it. Measured rather than
7655/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
7656/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
7657/// a test named for something else that happens to assert the string.
7658///
7659/// That accidental coverage is deliberately left alone rather than promoted to a
7660/// named test, because it guards a property this function does not promise.
7661/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
7662/// specific code within a class, so identity is free to change and only the
7663/// partition is a contract. Splitting it out would assert a guarantee nothing
7664/// depends on — and a suite that promises more than the code does is the harder
7665/// thing to correct later, because the next reader cannot tell which assertions
7666/// are load-bearing.
7667///
7668/// Pin identity here the moment a consumer branches on a specific code.
7669fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7670    match err {
7671        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7672        ForwardingError::NoModuleConnection => "target_unavailable",
7673        ForwardingError::ModuleReloading { .. } => "module_reloading",
7674        ForwardingError::ClientRouteChannelExhausted { .. }
7675        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7676        ForwardingError::StaleModuleEndpoint
7677        | ForwardingError::UnknownReservation { .. }
7678        | ForwardingError::ConnectionClosing { .. }
7679        | ForwardingError::ClientEgressClosed { .. }
7680        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7681        // Only a swap candidate's registration can produce this, and it means
7682        // exactly what a second active HELLO for a live id means.
7683        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7684        ForwardingError::RelayCorrelationExhausted
7685        | ForwardingError::RouteOpenBuild(_)
7686        | ForwardingError::Poisoned => "forwarding_error",
7687    }
7688}
7689
7690fn response_version(frame: &Frame) -> u8 {
7691    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7692        frame.header.ver
7693    } else {
7694        PROTOCOL_VERSION
7695    }
7696}
7697
7698fn control_flags() -> Flags {
7699    Flags::new(false, Priority::Passive, false)
7700}
7701
7702/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
7703/// channel. The target is the module (a client never saw the route), so this
7704/// takes the module path: delivered late rather than dropped when the module's
7705/// queue is momentarily full, and never closing its connection.
7706fn send_goodbye_target_best_effort(
7707    counters: &DaemonCounters,
7708    target: &GoodbyeTarget,
7709    context: &'static str,
7710) {
7711    let Ok(frame) = Frame::build_with_version(
7712        target.negotiated_ver,
7713        FrameType::Goodbye,
7714        control_flags(),
7715        target.channel,
7716        target.epoch,
7717        0,
7718        Vec::new(),
7719    ) else {
7720        return;
7721    };
7722    crate::forwarding::send_module_route_goodbye(
7723        counters,
7724        &target.sink,
7725        frame,
7726        target.module_id.as_deref(),
7727        context,
7728    );
7729}
7730
7731pub(crate) fn send_route_control_pushes(
7732    forwarding: &ForwardingTable,
7733    routes: Vec<EndpointRoute>,
7734    push: ClientControlPush,
7735) {
7736    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7737    for route in routes {
7738        let target = route.goodbye_target;
7739        if let Some((existing, channels)) = targets
7740            .iter_mut()
7741            .find(|(existing, _)| existing.connection_id == target.connection_id)
7742        {
7743            debug_assert_eq!(
7744                existing.negotiated_ver, target.negotiated_ver,
7745                "one connection cannot negotiate multiple frame versions"
7746            );
7747            if !channels.contains(&target.channel) {
7748                channels.push(target.channel);
7749            }
7750            continue;
7751        }
7752        let channel = target.channel;
7753        targets.push((target, vec![channel]));
7754    }
7755    for (target, mut channels) in targets {
7756        channels.sort_unstable();
7757        let mut push = push.clone();
7758        match &mut push {
7759            ClientControlPush::RouteClosing {
7760                channels: covered, ..
7761            }
7762            | ClientControlPush::RouteClosed {
7763                channels: covered, ..
7764            } => *covered = channels,
7765        }
7766        let body = match serde_json::to_vec(&push) {
7767            Ok(body) => body,
7768            Err(err) => {
7769                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7770                continue;
7771            }
7772        };
7773        let frame = match Frame::build_with_version(
7774            target.negotiated_ver,
7775            FrameType::Push,
7776            control_flags(),
7777            0,
7778            0,
7779            0,
7780            body.clone(),
7781        ) {
7782            Ok(frame) => frame,
7783            Err(err) => {
7784                warn!(
7785                    route_channel = target.channel,
7786                    error = %err,
7787                    "failed to build route lifecycle control PUSH frame"
7788                );
7789                continue;
7790            }
7791        };
7792        if let Err(err) = target.sink.try_send(frame) {
7793            if target.close_on_delivery_failure() {
7794                warn!(
7795                    target_connection_id = target.connection_id.get(),
7796                    route_channel = target.channel,
7797                    error = %err,
7798                    "route lifecycle control PUSH was not delivered to client; closing target connection"
7799                );
7800                let _ = forwarding.escalate_client_delivery_failure(
7801                    target.connection_id,
7802                    target.channel,
7803                    target.epoch,
7804                    CloseReason::new(
7805                        "route_lifecycle_push_delivery_failed",
7806                        format!(
7807                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7808                            target.channel
7809                        ),
7810                    ),
7811                    crate::forwarding::UndeliveredFrame {
7812                        module_id: target.module_id.as_deref(),
7813                        sink: &target.sink,
7814                    },
7815                );
7816            }
7817        }
7818    }
7819}
7820
7821#[cfg(test)]
7822mod tests {
7823    mod activity_read {
7824        include!("activity_read_tests.rs");
7825    }
7826
7827    mod lifecycle_activity {
7828        include!("control_lifecycle_tests.rs");
7829    }
7830    #[cfg(unix)]
7831    #[tokio::test]
7832    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7833        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7834        let path = dir.join("subc.jsonc");
7835        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7836            "program":"/bin/sleep","args":["60"],"protocol":"none",
7837            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7838        }}}).to_string()).unwrap();
7839        let mut configured = crate::daemon_config::load(&path)
7840            .unwrap()
7841            .unwrap()
7842            .modules
7843            .pop()
7844            .unwrap();
7845        let registry = std::sync::Arc::new(crate::Registry::default());
7846        let handle = crate::SupervisorHandle::new();
7847        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7848            .with_handle(handle.clone());
7849        let module = supervisor
7850            .supervise_configured_with_health(
7851                configured.module_spec(),
7852                true,
7853                configured.health.clone(),
7854                None,
7855                configured.restart,
7856            )
7857            .unwrap();
7858        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7859        let before = module.status().unwrap().pid;
7860        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7861        configured.health.cadence = std::time::Duration::from_secs(3600);
7862        let health_only = handler
7863            .reconcile_supervised_modules(
7864                &supervisor,
7865                vec![configured.clone()],
7866                false,
7867                Principal::Direct,
7868            )
7869            .await
7870            .unwrap();
7871        assert!(
7872            health_only.0.changed_pending_reload.is_empty(),
7873            "health policy is already applied live"
7874        );
7875        assert_eq!(module.status().unwrap().pid, before);
7876        assert_eq!(
7877            module.configuration().unwrap().1.http,
7878            configured.health.http
7879        );
7880        configured.args = vec!["61".into()];
7881        let launch = handler
7882            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7883            .await
7884            .unwrap();
7885        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
7886        assert_eq!(
7887            module.status().unwrap().pid,
7888            before,
7889            "a launch edit is stored until reload"
7890        );
7891        module.drain().await.unwrap();
7892    }
7893    use cortexkit_test_support::ScratchDir;
7894    use std::{
7895        collections::BTreeMap,
7896        fmt,
7897        path::PathBuf,
7898        sync::{Arc, Mutex},
7899        time::Duration,
7900    };
7901
7902    use serde_json::{json, Value};
7903    use subc_protocol::{
7904        manifest::{
7905            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7906            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7907        },
7908        session::HealthStatus,
7909        FrameType,
7910    };
7911
7912    use super::*;
7913    use crate::{
7914        forwarding::{DataRoute, DataRouteState},
7915        registry::ChannelState,
7916        router::FrameSink,
7917        stderr_tail::DEFAULT_MAX_LINE_BYTES,
7918        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7919        RouteCtx, Router,
7920    };
7921    use tokio::{
7922        sync::mpsc,
7923        time::{sleep, Instant},
7924    };
7925    use tracing::{
7926        field::{Field, Visit},
7927        Event, Subscriber,
7928    };
7929    use tracing_subscriber::{layer::Context, prelude::*, Layer};
7930
7931    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
7932    ///
7933    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
7934    /// is only populated for `tests/*.rs` integration test binaries -- this file
7935    /// compiles as part of the library target, which gets neither. This test's
7936    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
7937    /// and the sibling binary lives two directories up at
7938    /// `<target-dir>/<profile>/fake-aft-stub`.
7939    ///
7940    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
7941    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
7942    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
7943    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
7944    /// `NotFound`, which reads as a broken test rather than an unbuilt
7945    /// dependency -- so state the cause and the remedy instead. Deliberately a
7946    /// panic and not a silent skip: a test that quietly passes when it could not
7947    /// run is worse than one that fails, because it reports health it never
7948    /// verified.
7949    fn fake_aft_stub_path() -> PathBuf {
7950        let mut path = std::env::current_exe().expect("current_exe available in tests");
7951        path.pop(); // .../deps/
7952        path.pop(); // .../<profile>/
7953        path.push(if cfg!(windows) {
7954            "fake-aft-stub.exe"
7955        } else {
7956            "fake-aft-stub"
7957        });
7958        assert!(
7959            path.exists(),
7960            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7961             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7962            path.display()
7963        );
7964        path
7965    }
7966
7967    /// Whether clients retry `code` in place: the predicate itself, never a copy
7968    /// of its set. A copied list breaks silently when a code is added to or
7969    /// removed from the real one, and a stale copy here would let exactly the
7970    /// failure this test exists to catch pass.
7971    fn client_retries(code: &str) -> bool {
7972        subc_protocol::error_codes::is_retryable_route_open(code)
7973    }
7974
7975    /// A code is not a label — clients branch on it, so publishing the wrong KIND
7976    /// of failure is worse than publishing none. A permanent fault dressed as
7977    /// retryable makes every client in the fleet retry forever against something
7978    /// that cannot recover; a transient fault dressed as permanent abandons work
7979    /// that would have succeeded.
7980    ///
7981    /// Asserting "a code exists" cannot catch either, because the string is free
7982    /// to say anything. This enumerates every variant and pins which side of the
7983    /// retry boundary it lands on, so a new variant must be classified here
7984    /// deliberately rather than inheriting whichever arm it was appended to.
7985    #[test]
7986    fn retryability_of_forwarding_codes_matches_the_failure() {
7987        // Transient by nature: the target is booting, reloading, or its endpoint
7988        // was swapped mid-flight. Retrying is how these resolve.
7989        let transient = [
7990            ForwardingError::NoModuleConnection,
7991            ForwardingError::ModuleReloading {
7992                module_id: "m".into(),
7993            },
7994            ForwardingError::StaleModuleEndpoint,
7995            ForwardingError::UnknownReservation {
7996                client_channel: 1,
7997                module_channel: 1,
7998            },
7999            ForwardingError::ConnectionClosing {
8000                connection_id: ConnectionId::new(1),
8001            },
8002            ForwardingError::ClientEgressClosed {
8003                connection_id: ConnectionId::new(1),
8004            },
8005            ForwardingError::ModuleEgressUnavailable {
8006                connection_id: ConnectionId::new(1),
8007            },
8008        ];
8009        for err in transient {
8010            let code = forwarding_error_code(&err);
8011            assert!(
8012                client_retries(code),
8013                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8014            );
8015        }
8016
8017        // Not fixed by retrying. Channel and correlation exhaustion need the
8018        // caller to close routes, and a poisoned lock is a daemon that cannot
8019        // recover at all — the worst thing to advertise as retryable, since every
8020        // client would storm a daemon that will never answer.
8021        let permanent = [
8022            ForwardingError::ConnectionRoleConflict {
8023                connection_id: ConnectionId::new(1),
8024            },
8025            ForwardingError::ClientRouteChannelExhausted {
8026                connection_id: ConnectionId::new(1),
8027            },
8028            ForwardingError::ModuleRouteChannelExhausted {
8029                endpoint: ModuleEndpointId {
8030                    connection_id: ConnectionId::new(1),
8031                    generation: 1,
8032                },
8033            },
8034            ForwardingError::RelayCorrelationExhausted,
8035            ForwardingError::RouteOpenBuild("x".into()),
8036            ForwardingError::Poisoned,
8037        ];
8038        for err in permanent {
8039            let code = forwarding_error_code(&err);
8040            assert!(
8041                !client_retries(code),
8042                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8043            );
8044        }
8045    }
8046
8047    /// The principal is the daemon's answer to "who is calling", and modules
8048    /// branch on it: aft gates bash on it, cerebellum gates browser control,
8049    /// plexus gates connector invocation. So a stamp is an authorization input in
8050    /// another process, not a label — and both possible answers SUCCEED, which is
8051    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
8052    /// first-party capability to something that never proved it; a supervised one
8053    /// stamped `Direct` silently strips a module of capability it is entitled to.
8054    ///
8055    /// Neither shows up in a test that only checks the bind succeeded. Before this
8056    /// test the only coverage was accidental —
8057    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
8058    /// stamped principal on its way past, so narrowing that wire-shape test to its
8059    /// stated subject would have deleted the last assertion on this value. It
8060    /// still asserts the stamp, which is now redundancy rather than the only
8061    /// guard: both fail under the same mutation, and this one names the reason.
8062    /// SCOPE: this handler's supervisor has spawned nothing, so
8063    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
8064    /// is unreachable here. Both assertions below are refusals, and a mutant that
8065    /// refuses everything would satisfy them.
8066    ///
8067    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
8068    /// spawns a supervised consumer, reads its live nonce, and asserts the module
8069    /// observed `principal.kind == "reserved"` carrying that module_id — verified
8070    /// at source rather than assumed, since a citation is a claim about another
8071    /// file and ages like one. Recorded because a harness that structurally
8072    /// cannot reach an arm reports "none" for that arm identically to one that
8073    /// covers it and found nothing.
8074    #[tokio::test]
8075    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8076        let handler = ControlHandler::default();
8077        let frame =
8078            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8079
8080        // Absent consumer_identity is the ordinary case: a human at a terminal, or
8081        // any process holding the connection file. Nothing was proved, so nothing
8082        // may be granted beyond the unattested floor.
8083        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8084        assert_eq!(
8085            stamped,
8086            Principal::Direct,
8087            "a caller that proved nothing must not be stamped as a supervised module"
8088        );
8089
8090        // A claimed module_id with a nonce no supervised child was given is a
8091        // forgery attempt, not a weaker caller: it must be REFUSED rather than
8092        // quietly demoted to Direct, or an impersonation attempt looks identical
8093        // to an ordinary unattested connection.
8094        let forged = handler
8095            .route_open_principal(
8096                &frame,
8097                Some(ConsumerIdentity {
8098                    module_id: "aft".to_string(),
8099                    launch_nonce: "not-a-real-nonce".to_string(),
8100                }),
8101            )
8102            .unwrap();
8103        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8104        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8105    }
8106
8107    /// The test above hands `route_open_principal` an identity it built itself,
8108    /// which proves the stamping rule and nothing about where the identity comes
8109    /// from. The real producer is a wire body, and the two are joined by a serde
8110    /// field name that nothing else asserts.
8111    ///
8112    /// That join fails quietly in one specific way: an unrecognised key is simply
8113    /// absent after parsing, so a renamed or misspelled `consumer_identity`
8114    /// yields `None` and every supervised module silently drops to `Direct`.
8115    /// Capability-wise that is the safe direction, but it surfaces far from its
8116    /// cause — as a module mysteriously refused bash — and it would pass every
8117    /// test that builds its own input.
8118    ///
8119    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
8120    /// would break every client the moment the daemon gains a field, trading a
8121    /// quiet demotion for a hard refusal on additive change. Asserting the join
8122    /// instead means a rename breaks a test here rather than the fleet.
8123    #[test]
8124    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8125        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8126        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8127        let ClientControlRequest::RouteOpen {
8128            consumer_identity, ..
8129        } = parsed
8130        else {
8131            panic!("route.open body must parse as RouteOpen");
8132        };
8133        assert_eq!(
8134            consumer_identity,
8135            Some(ConsumerIdentity {
8136                module_id: "aft".to_string(),
8137                launch_nonce: "n".to_string(),
8138            }),
8139            "the wire field name must reach the value route_open_principal reads"
8140        );
8141    }
8142
8143    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8144        ModuleManifest::builder(module_id, "0.1.0")
8145            .protocol_ver(protocol_ver)
8146            .provides(vec![ProviderRole::ToolProvider {
8147                tools: vec![Tool {
8148                    name: "read".to_string(),
8149                    description: None,
8150                    execution_mode: ExecutionMode::Pure,
8151                    schema: json!({"type": "object"}),
8152                }],
8153                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8154                concurrency: Concurrency::ModuleManaged,
8155                emits_push: true,
8156                sub_supervises: true,
8157            }])
8158            .build()
8159    }
8160
8161    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8162        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8163    }
8164
8165    fn hello_frame_with_control_ops(
8166        module_id: &str,
8167        protocol_ver: u8,
8168        corr: u64,
8169        control_ops: Option<Vec<String>>,
8170    ) -> Frame {
8171        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8172    }
8173
8174    fn hello_frame_with_nonce(
8175        module_id: &str,
8176        protocol_ver: u8,
8177        corr: u64,
8178        launch_nonce: Option<&str>,
8179    ) -> Frame {
8180        hello_frame_full(
8181            module_id,
8182            protocol_ver,
8183            corr,
8184            None,
8185            launch_nonce.map(ToOwned::to_owned),
8186        )
8187    }
8188
8189    fn hello_frame_full(
8190        module_id: &str,
8191        protocol_ver: u8,
8192        corr: u64,
8193        control_ops: Option<Vec<String>>,
8194        launch_nonce: Option<String>,
8195    ) -> Frame {
8196        let body = serde_json::to_vec(&ModuleHelloBody {
8197            manifest: manifest(module_id, protocol_ver),
8198            protocol_ver,
8199            control_ops,
8200            launch_nonce,
8201        })
8202        .unwrap();
8203        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8204    }
8205
8206    fn hello_frame_with_rich_ui(
8207        module_id: &str,
8208        protocol_ver: u8,
8209        corr: u64,
8210        launch_nonce: Option<&str>,
8211        rich_ui: Value,
8212    ) -> Frame {
8213        let mut body = serde_json::to_value(ModuleHelloBody {
8214            manifest: manifest(module_id, protocol_ver),
8215            protocol_ver,
8216            control_ops: None,
8217            launch_nonce: launch_nonce.map(str::to_string),
8218        })
8219        .unwrap();
8220        body["manifest"]["rich_ui"] = rich_ui;
8221        Frame::build(
8222            FrameType::Hello,
8223            control_flags(),
8224            0,
8225            0,
8226            corr,
8227            serde_json::to_vec(&body).unwrap(),
8228        )
8229        .unwrap()
8230    }
8231
8232    fn non_routable_hello_frame_with_control_ops(
8233        module_id: &str,
8234        corr: u64,
8235        control_ops: Option<Vec<String>>,
8236    ) -> Frame {
8237        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8238        manifest.provides.clear();
8239        let body = serde_json::to_vec(&ModuleHelloBody {
8240            manifest,
8241            protocol_ver: PROTOCOL_VERSION,
8242            control_ops,
8243            launch_nonce: None,
8244        })
8245        .unwrap();
8246        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8247    }
8248
8249    fn capability_grammar_hello_frame(
8250        capabilities: Value,
8251        runtime_computed: Option<Value>,
8252        corr: u64,
8253    ) -> Frame {
8254        let mut body = serde_json::to_value(ModuleHelloBody {
8255            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8256            protocol_ver: PROTOCOL_VERSION,
8257            control_ops: None,
8258            launch_nonce: None,
8259        })
8260        .expect("HELLO body serializes");
8261        body["manifest"]["capabilities"] = capabilities;
8262        if let Some(runtime_computed) = runtime_computed {
8263            body["runtime_computed"] = runtime_computed;
8264        }
8265        Frame::build(
8266            FrameType::Hello,
8267            control_flags(),
8268            0,
8269            0,
8270            corr,
8271            serde_json::to_vec(&body).expect("HELLO body reserializes"),
8272        )
8273        .expect("HELLO frame builds")
8274    }
8275
8276    fn channel_request(channel: u16, corr: u64) -> Frame {
8277        Frame::build(
8278            FrameType::Request,
8279            Flags::new(true, Priority::Interactive, false),
8280            channel,
8281            0,
8282            corr,
8283            b"opaque".to_vec(),
8284        )
8285        .unwrap()
8286    }
8287
8288    fn route_ctx(
8289        connection_id: ConnectionId,
8290    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8291        let (tx, rx) = mpsc::channel(8);
8292        (
8293            RouteCtx {
8294                connection_id,
8295                egress: FrameSink::new(tx),
8296            },
8297            rx,
8298        )
8299    }
8300
8301    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8302        serde_json::from_slice(&frame.body).unwrap()
8303    }
8304
8305    /// Register a module over a connection that has a sink and return the
8306    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
8307    /// module's own sink rather than returning it as a reply, so the ack is
8308    /// taken off `rx` here and whatever the test reads next is what followed it.
8309    async fn hello_via_sink(
8310        handler: &ControlHandler,
8311        ctx: &RouteCtx,
8312        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8313        hello: Frame,
8314    ) -> Frame {
8315        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8316        assert!(
8317            replies.is_empty(),
8318            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8319        );
8320        let ack = rx
8321            .try_recv()
8322            .expect("HELLO_ACK is queued on the module sink")
8323            .frame;
8324        assert_eq!(ack.header.ty, FrameType::HelloAck);
8325        ack
8326    }
8327
8328    fn parse_error(frame: &Frame) -> Value {
8329        serde_json::from_slice(&frame.body).unwrap()
8330    }
8331
8332    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8333        serde_json::from_slice(&frame.body).unwrap()
8334    }
8335
8336    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8337        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8338            route_channel,
8339            route_epoch: 0,
8340            kind,
8341        })
8342        .unwrap();
8343        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8344    }
8345
8346    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8347        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8348            module_id: module_id.to_string(),
8349        })
8350        .unwrap();
8351        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8352    }
8353
8354    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8355        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8356    }
8357
8358    fn route_open_frame_with_consumer_capabilities(
8359        corr: u64,
8360        module_id: &str,
8361        project_root: ScratchDir,
8362        consumer_capabilities: Option<Vec<String>>,
8363    ) -> Frame {
8364        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8365            target: RouteTarget::ToolProvider {
8366                module_id: module_id.to_string(),
8367            },
8368            identity: BindIdentity::new(
8369                project_root.path().to_path_buf(),
8370                "unit".to_string(),
8371                "session".to_string(),
8372            ),
8373            consumer_identity: None,
8374            consumer_capabilities,
8375            role_versions: None,
8376            admission_facts: None,
8377            scope: None,
8378        })
8379        .unwrap();
8380        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8381    }
8382
8383    fn route_open_frame_with_role_versions(
8384        corr: u64,
8385        module_id: &str,
8386        project_root: ScratchDir,
8387        role_versions: Option<BTreeMap<String, String>>,
8388    ) -> Frame {
8389        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8390            target: RouteTarget::ToolProvider {
8391                module_id: module_id.to_string(),
8392            },
8393            identity: BindIdentity::new(
8394                project_root.path().to_path_buf(),
8395                "unit".to_string(),
8396                format!("session-{corr}"),
8397            ),
8398            consumer_identity: None,
8399            consumer_capabilities: None,
8400            role_versions,
8401            admission_facts: None,
8402            scope: None,
8403        })
8404        .unwrap();
8405        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8406    }
8407
8408    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8409        entries
8410            .iter()
8411            .map(|(role, version)| (role.to_string(), version.to_string()))
8412            .collect()
8413    }
8414
8415    fn route_open_frame_with_admission_facts(
8416        corr: u64,
8417        module_id: &str,
8418        project_root: ScratchDir,
8419        consumer_identity: Option<subc_control::ConsumerIdentity>,
8420        facts: Option<Value>,
8421    ) -> Frame {
8422        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8423            target: RouteTarget::ToolProvider {
8424                module_id: module_id.to_string(),
8425            },
8426            identity: BindIdentity::new(
8427                project_root.path().to_path_buf(),
8428                "unit".to_string(),
8429                format!("session-{corr}"),
8430            ),
8431            consumer_identity,
8432            consumer_capabilities: None,
8433            role_versions: None,
8434            admission_facts: facts,
8435            scope: None,
8436        })
8437        .unwrap();
8438        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8439    }
8440
8441    #[derive(Clone, Default)]
8442    struct EventCapture {
8443        events: Arc<Mutex<Vec<CapturedEvent>>>,
8444    }
8445
8446    #[derive(Clone, Debug)]
8447    struct CapturedEvent {
8448        target: String,
8449        level: tracing::Level,
8450        fields: BTreeMap<String, String>,
8451    }
8452
8453    impl EventCapture {
8454        fn events(&self) -> Vec<CapturedEvent> {
8455            self.events.lock().unwrap().clone()
8456        }
8457    }
8458
8459    impl<S> Layer<S> for EventCapture
8460    where
8461        S: Subscriber,
8462    {
8463        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8464            let mut visitor = EventFieldVisitor::default();
8465            event.record(&mut visitor);
8466            self.events.lock().unwrap().push(CapturedEvent {
8467                target: event.metadata().target().to_string(),
8468                level: *event.metadata().level(),
8469                fields: visitor.fields,
8470            });
8471        }
8472    }
8473
8474    #[derive(Default)]
8475    struct EventFieldVisitor {
8476        fields: BTreeMap<String, String>,
8477    }
8478
8479    impl Visit for EventFieldVisitor {
8480        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8481            self.fields
8482                .insert(field.name().to_string(), format!("{value:?}"));
8483        }
8484    }
8485
8486    fn health_response(corr: u64, status: HealthStatus) -> Frame {
8487        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8488            status,
8489            detail: Some("warming".to_string()),
8490            metrics: Some(json!({"queue_depth": 3})),
8491        })
8492        .unwrap();
8493        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8494    }
8495
8496    fn route_bind_ack(corr: u64) -> Frame {
8497        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8498        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8499    }
8500
8501    fn unique_project_root(label: &str) -> ScratchDir {
8502        ScratchDir::new(label)
8503    }
8504
8505    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8506        match parse_route_poll(frame) {
8507            ClientControlResponse::RoutePoll {
8508                status: None,
8509                live: Some(live),
8510                ..
8511            } => assert_eq!(live, expected_live),
8512            other => panic!("unexpected route.poll response: {other:?}"),
8513        }
8514    }
8515
8516    fn bind_liveness_route(
8517        registry: &Registry,
8518        forwarding: &ForwardingTable,
8519        module_id: &str,
8520    ) -> (RouteCtx, u16, u32) {
8521        let module_connection = ConnectionId::new(101);
8522        let client_connection = ConnectionId::new(202);
8523        let registration = registry
8524            .register_with_control_ops(
8525                manifest(module_id, PROTOCOL_VERSION),
8526                PROTOCOL_VERSION,
8527                module_connection,
8528                module_baseline_control_ops(),
8529            )
8530            .unwrap();
8531        let (module_tx, _module_rx) = mpsc::channel(8);
8532        let endpoint = forwarding
8533            .register_module_connection(
8534                module_connection,
8535                module_id.to_string(),
8536                PROTOCOL_VERSION,
8537                manifest_concurrency(&registration.manifest),
8538                FrameSink::new(module_tx),
8539            )
8540            .unwrap();
8541        let (client_ctx, _client_rx) = route_ctx(client_connection);
8542        let pending = forwarding
8543            .begin_route_bind_relay_for_test(
8544                client_connection,
8545                client_ctx.egress.clone(),
8546                1,
8547                module_id,
8548            )
8549            .unwrap();
8550        assert_eq!(pending.endpoint, endpoint);
8551        let route_channel = pending.client_channel;
8552        let route_epoch = pending.client_epoch;
8553        forwarding
8554            .complete_pending_relay(
8555                module_connection,
8556                pending.corr,
8557                RouteBindRelayOutcome::Accepted,
8558            )
8559            .unwrap();
8560        (client_ctx, route_channel, route_epoch)
8561    }
8562
8563    struct FakeProcessLiveness {
8564        live: Option<bool>,
8565    }
8566
8567    impl ModuleProcessLiveness for FakeProcessLiveness {
8568        fn process_live(&self, _module_id: &str) -> Option<bool> {
8569            self.live
8570        }
8571    }
8572
8573    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8574    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8575    {
8576        let registry = Arc::new(Registry::default());
8577        let supervisor_handle = SupervisorHandle::new();
8578        let supervisor = Supervisor::new_for_test(
8579            Arc::clone(&registry),
8580            RestartPolicy::new(1, Duration::from_millis(10)),
8581        )
8582        .with_handle(supervisor_handle.clone());
8583        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8584        let module = supervisor
8585            .spawn(ModuleSpec {
8586                module_id: "stderr-tail-wire".to_string(),
8587                program: fake_aft_stub_path(),
8588                args: Vec::new(),
8589                env: vec![
8590                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8591                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8592                ],
8593                reserved: false,
8594                reserved_prefixes: Vec::new(),
8595                protocol: ModuleProtocol::Subc,
8596                overlap: Default::default(),
8597            })
8598            .unwrap();
8599
8600        let deadline = Instant::now() + Duration::from_secs(5);
8601        loop {
8602            let tail = module.stderr_tail(None, None);
8603            if tail
8604                .entries
8605                .iter()
8606                .any(|entry| matches!(entry, TailEntry::ProcessStart))
8607                && tail.entries.iter().any(|entry| {
8608                    matches!(
8609                        entry,
8610                        TailEntry::Line {
8611                            truncated: true,
8612                            ..
8613                        }
8614                    )
8615                })
8616            {
8617                break;
8618            }
8619            assert!(
8620                Instant::now() < deadline,
8621                "module did not produce a truncated line and restart boundary: {tail:?}"
8622            );
8623            sleep(Duration::from_millis(10)).await;
8624        }
8625
8626        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8627        let request = ClientControlRequest::SupervisorStderrTail {
8628            module_id: "stderr-tail-wire".to_string(),
8629            max_lines: None,
8630            max_bytes: None,
8631        };
8632        let frame = Frame::build(
8633            FrameType::Request,
8634            control_flags(),
8635            0,
8636            0,
8637            1,
8638            serde_json::to_vec(&request).unwrap(),
8639        )
8640        .unwrap();
8641        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8642        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8643        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8644            serde_json::from_slice(&responses[0].body).unwrap()
8645        else {
8646            panic!("expected supervisor.stderr_tail response");
8647        };
8648
8649        assert!(
8650            tail.entries
8651                .iter()
8652                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8653            "the control response lost the restart boundary"
8654        );
8655        let Some(StderrTailEntry::Line {
8656            text,
8657            truncated,
8658            at_ms,
8659        }) = tail.entries.iter().find(|entry| {
8660            matches!(
8661                entry,
8662                StderrTailEntry::Line {
8663                    truncated: true,
8664                    ..
8665                }
8666            )
8667        })
8668        else {
8669            panic!("the control response lost the truncated line");
8670        };
8671        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8672        assert!(*truncated);
8673        assert!(
8674            at_ms.is_some(),
8675            "the control response lost the line's capture time"
8676        );
8677    }
8678
8679    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
8680    /// read done on the worker thread would stall every other task until it
8681    /// finished; the read must run off the worker so this test's own task keeps
8682    /// running while the read is paused.
8683    #[tokio::test(flavor = "current_thread")]
8684    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8685        let dir = ScratchDir::new("terminals-off-worker");
8686        let journal_path = dir.join("terminals.jsonl");
8687        let registry = Arc::new(Registry::default());
8688        let supervisor_handle = SupervisorHandle::new();
8689        let supervisor =
8690            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8691                .with_handle(supervisor_handle.clone())
8692                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8693        let module = supervisor
8694            .spawn(ModuleSpec {
8695                module_id: "terminal-off-worker".to_string(),
8696                program: fake_aft_stub_path(),
8697                args: Vec::new(),
8698                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8699                reserved: false,
8700                reserved_prefixes: Vec::new(),
8701                protocol: ModuleProtocol::Subc,
8702                overlap: Default::default(),
8703            })
8704            .unwrap();
8705        let deadline = Instant::now() + Duration::from_secs(5);
8706        while module.terminal_history().entries.len() != 2 {
8707            assert!(Instant::now() < deadline, "module did not record two exits");
8708            sleep(Duration::from_millis(10)).await;
8709        }
8710
8711        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8712        let handler =
8713            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
8714        let frame = Frame::build(
8715            FrameType::Request,
8716            control_flags(),
8717            0,
8718            0,
8719            1,
8720            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8721                module_id: "terminal-off-worker".to_string(),
8722            })
8723            .unwrap(),
8724        )
8725        .unwrap();
8726        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8727        let spawned_at = std::time::Instant::now();
8728        let read = tokio::spawn({
8729            let handler = Arc::clone(&handler);
8730            async move { handler.handle_control_frame(&ctx, frame).await }
8731        });
8732        // Waiting for the pause from a blocking thread keeps this task pending,
8733        // so the runtime's single worker is free to run the read task.
8734        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8735            .await
8736            .unwrap()
8737            .expect("the history read reached its pause");
8738        let elapsed = spawned_at.elapsed();
8739        assert!(
8740            elapsed < Duration::from_secs(2) && !read.is_finished(),
8741            "this task could not run while the history read was paused \
8742             (resumed after {elapsed:?}, read finished: {})",
8743            read.is_finished()
8744        );
8745
8746        drop(release);
8747        let responses = read.await.unwrap().unwrap();
8748        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8749        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8750            panic!("expected supervisor.terminals response");
8751        };
8752        assert_eq!(terminals.entries.len(), 2);
8753        assert_eq!(terminals.journal_skipped_lines, 0);
8754        assert_eq!(terminals.journal_read_errors, 0);
8755    }
8756
8757    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8758    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8759        let registry = Arc::new(Registry::default());
8760        let supervisor_handle = SupervisorHandle::new();
8761        let supervisor =
8762            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8763                .with_handle(supervisor_handle.clone());
8764        let module = supervisor
8765            .spawn(ModuleSpec {
8766                module_id: "terminal-golden".to_string(),
8767                program: fake_aft_stub_path(),
8768                args: Vec::new(),
8769                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8770                reserved: false,
8771                reserved_prefixes: Vec::new(),
8772                protocol: ModuleProtocol::Subc,
8773                overlap: Default::default(),
8774            })
8775            .unwrap();
8776
8777        let deadline = Instant::now() + Duration::from_secs(5);
8778        while module.terminal_history().entries.len() != 2 {
8779            assert!(
8780                Instant::now() < deadline,
8781                "module did not retain two terminal exits: {:?}",
8782                module.terminal_history()
8783            );
8784            sleep(Duration::from_millis(10)).await;
8785        }
8786
8787        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8788        let request = ClientControlRequest::SupervisorTerminals {
8789            module_id: "terminal-golden".to_string(),
8790        };
8791        let frame = Frame::build(
8792            FrameType::Request,
8793            control_flags(),
8794            0,
8795            0,
8796            1,
8797            serde_json::to_vec(&request).unwrap(),
8798        )
8799        .unwrap();
8800        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8801        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8802        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8803        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8804            panic!("expected supervisor.terminals response");
8805        };
8806        assert_eq!(terminals.entries.len(), 2);
8807        assert_eq!(terminals.dropped, 0);
8808
8809        let mut rendered = serde_json::to_value(response).unwrap();
8810        // Wall-clock fields are the observation contract, but not stable fixture
8811        // bytes; normalize only them after the real handler has shaped the response.
8812        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8813        for (index, entry) in rendered["entries"]
8814            .as_array_mut()
8815            .expect("terminal response entries array")
8816            .iter_mut()
8817            .enumerate()
8818        {
8819            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8820        }
8821
8822        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8823            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8824        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8825        if std::env::var_os("UPDATE_GOLDEN").is_some() {
8826            std::fs::write(&golden_path, &serialized).unwrap();
8827        }
8828        let expected: Value =
8829            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8830        assert_eq!(rendered, expected);
8831    }
8832
8833    #[test]
8834    fn hello_registers_manifest_and_returns_ack() {
8835        let registry = Arc::new(Registry::default());
8836        let handler = ControlHandler::new(Arc::clone(&registry));
8837        let conn = ConnectionId::new(1);
8838
8839        let responses = handler
8840            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8841            .unwrap();
8842
8843        assert_eq!(responses.len(), 1);
8844        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8845        assert_eq!(responses[0].header.channel, 0);
8846        assert_eq!(responses[0].header.corr, 7);
8847        let ack = parse_ack(&responses[0]);
8848        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8849        assert!(ack
8850            .subc_capabilities
8851            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8852        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8853        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8854        assert!(ack
8855            .subc_ops
8856            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8857        assert!(ack
8858            .subc_ops
8859            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8860
8861        let registration = registry.get_module("aft").unwrap().unwrap();
8862        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8863        assert_eq!(registration.state, ChannelState::Active);
8864        assert_eq!(registration.connection_id, conn);
8865        assert_eq!(registration.control_ops, module_baseline_control_ops());
8866    }
8867
8868    #[test]
8869    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8870        let invalid_identifiers = [
8871            ("case_change", "credentials-Provider/v1"),
8872            ("leading_zero", "credentials-provider/v01"),
8873            ("trailing_hyphen", "credentials-provider-/v1"),
8874            ("consecutive_hyphens", "credentials--provider/v1"),
8875            ("uppercase", "Credentials-provider/v1"),
8876            ("missing_v", "credentials-provider/1"),
8877            ("whitespace", "credentials provider/v1"),
8878            ("zero_version", "credentials-provider/v0"),
8879            ("out_of_range_version", "credentials-provider/v4294967296"),
8880            (
8881                "overlength_name",
8882                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8883            ),
8884        ];
8885        let mut cases = invalid_identifiers
8886            .into_iter()
8887            .map(|(name, identifier)| {
8888                (
8889                    format!("identifier_{name}"),
8890                    "capabilities.provides[0]".to_string(),
8891                    identifier.to_string(),
8892                    json!({ "provides": [identifier] }),
8893                    None,
8894                )
8895            })
8896            .collect::<Vec<_>>();
8897        cases.extend([
8898            (
8899                "unknown_need".to_string(),
8900                "capabilities.requires[0].need".to_string(),
8901                "deferred".to_string(),
8902                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8903                None,
8904            ),
8905            (
8906                "duplicate_provides".to_string(),
8907                "capabilities.provides[1]".to_string(),
8908                "credentials-provider/v1".to_string(),
8909                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8910                None,
8911            ),
8912            (
8913                "duplicate_must_never_reach".to_string(),
8914                "capabilities.must_never_reach[1]".to_string(),
8915                "credentials-provider/v1".to_string(),
8916                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8917                None,
8918            ),
8919            (
8920                "duplicate_requires_same_need".to_string(),
8921                "capabilities.requires[1]".to_string(),
8922                "credentials-provider/v1".to_string(),
8923                json!({ "requires": [
8924                    { "capability": "credentials-provider/v1", "need": "required" },
8925                    { "capability": "credentials-provider/v1", "need": "required" }
8926                ] }),
8927                None,
8928            ),
8929            (
8930                "duplicate_requires_conflicting_need".to_string(),
8931                "capabilities.requires[1]".to_string(),
8932                "credentials-provider/v1".to_string(),
8933                json!({ "requires": [
8934                    { "capability": "credentials-provider/v1", "need": "required" },
8935                    { "capability": "credentials-provider/v1", "need": "optional" }
8936                ] }),
8937                None,
8938            ),
8939            (
8940                "capabilities_root_pointer".to_string(),
8941                "runtime_computed[0]".to_string(),
8942                "/capabilities".to_string(),
8943                json!({}),
8944                Some(json!(["/capabilities"])),
8945            ),
8946            (
8947                "capabilities_descendant_pointer".to_string(),
8948                "runtime_computed[0]".to_string(),
8949                "/capabilities/provides".to_string(),
8950                json!({}),
8951                Some(json!(["/capabilities/provides"])),
8952            ),
8953            (
8954                "malformed_pointer_without_leading_slash".to_string(),
8955                "runtime_computed[0]".to_string(),
8956                "capabilities".to_string(),
8957                json!({}),
8958                Some(json!(["capabilities"])),
8959            ),
8960            (
8961                "malformed_pointer_escape".to_string(),
8962                "runtime_computed[0]".to_string(),
8963                "/roles/~2/tools".to_string(),
8964                json!({}),
8965                Some(json!(["/roles/~2/tools"])),
8966            ),
8967            (
8968                "unknown_capabilities_field".to_string(),
8969                "capabilities.future".to_string(),
8970                "<array>".to_string(),
8971                json!({ "future": [] }),
8972                None,
8973            ),
8974        ]);
8975
8976        for (index, (name, field, value, capabilities, runtime_computed)) in
8977            cases.into_iter().enumerate()
8978        {
8979            let registry = Arc::new(Registry::default());
8980            let handler = ControlHandler::new(Arc::clone(&registry));
8981            let response = handler
8982                .handle_control(
8983                    ConnectionId::new((index + 1) as u64),
8984                    capability_grammar_hello_frame(
8985                        capabilities,
8986                        runtime_computed,
8987                        index as u64 + 1,
8988                    ),
8989                )
8990                .expect("invalid HELLO returns a refusal");
8991
8992            assert_eq!(response.len(), 1, "{name} must emit one refusal");
8993            let error = parse_error(&response[0]);
8994            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8995            let message = error["message"]
8996                .as_str()
8997                .expect("error message is a string");
8998            assert!(
8999                message.contains(&field),
9000                "{name}: field missing from {message}"
9001            );
9002            assert!(
9003                message.contains(&value),
9004                "{name}: value missing from {message}"
9005            );
9006            assert_eq!(
9007                registry
9008                    .active_registration_count()
9009                    .expect("registry reads"),
9010                0,
9011                "{name}: refused HELLO must not create a catalog entry"
9012            );
9013        }
9014    }
9015
9016    #[test]
9017    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9018        let registry = Arc::new(Registry::default());
9019        let handler = ControlHandler::new(Arc::clone(&registry));
9020        let capabilities = json!({
9021            "provides": ["credentials-provider/v1"],
9022            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9023            "must_never_reach": ["federation-transport/v1"]
9024        });
9025        let response = handler
9026            .handle_control(
9027                ConnectionId::new(99),
9028                capability_grammar_hello_frame(
9029                    capabilities.clone(),
9030                    Some(json!(["/roles/0/tools"])),
9031                    99,
9032                ),
9033            )
9034            .expect("valid HELLO registers");
9035        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9036
9037        let request = Frame::build(
9038            FrameType::Request,
9039            control_flags(),
9040            0,
9041            0,
9042            100,
9043            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9044                .expect("catalog request serializes"),
9045        )
9046        .expect("catalog request frame builds");
9047        let response = handler
9048            .handle_catalog_list(request, None)
9049            .expect("catalog list succeeds");
9050        let ClientControlResponse::CatalogList { modules, .. } =
9051            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9052        else {
9053            panic!("catalog request must return catalog.list");
9054        };
9055        assert_eq!(modules.len(), 1);
9056        assert_eq!(
9057            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9058            capabilities
9059        );
9060    }
9061
9062    #[test]
9063    fn catalog_list_mirrors_management_operation_description() {
9064        let registry = Arc::new(Registry::default());
9065        let handler = ControlHandler::new(Arc::clone(&registry));
9066        let description = "List managed records and return their identifiers and metadata.";
9067        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9068        manifest.provides = vec![ProviderRole::ManagementSurface {
9069            operations: vec![ManagementOperation {
9070                name: "records.list".to_string(),
9071                kind: ManagementOperationKind::Query,
9072                description: Some(description.to_string()),
9073            }],
9074            config_schema: json!({"type": "object"}),
9075            observability: vec![ObservabilitySurface {
9076                name: "records.stats".to_string(),
9077                kind: ObservabilityKind::Snapshot,
9078            }],
9079            identity_scope: vec![IdentityScope::Project],
9080            concurrency: Concurrency::ModuleManaged,
9081        }];
9082        registry
9083            .register_with_control_ops(
9084                manifest,
9085                PROTOCOL_VERSION,
9086                ConnectionId::new(99),
9087                Vec::new(),
9088            )
9089            .expect("described management manifest registers");
9090
9091        let request = Frame::build(
9092            FrameType::Request,
9093            control_flags(),
9094            0,
9095            0,
9096            100,
9097            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9098                .expect("catalog request serializes"),
9099        )
9100        .expect("catalog request frame builds");
9101        let response = handler
9102            .handle_catalog_list(request, None)
9103            .expect("catalog list succeeds");
9104        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9105        assert_eq!(
9106            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9107            "catalog.list must preserve the declared operation description verbatim"
9108        );
9109    }
9110
9111    #[test]
9112    fn catalog_list_mirrors_registered_operation_delivery() {
9113        let registry = Arc::new(Registry::default());
9114        let handler = ControlHandler::new(Arc::clone(&registry));
9115        let delivery =
9116            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9117        let mut body = serde_json::to_value(ModuleHelloBody {
9118            manifest: manifest("watch-management", PROTOCOL_VERSION),
9119            protocol_ver: PROTOCOL_VERSION,
9120            control_ops: None,
9121            launch_nonce: None,
9122        })
9123        .unwrap();
9124        body["manifest"]["provides"] = json!([{
9125            "role": "management_surface",
9126            "operations": [
9127                {"name": "status", "kind": "query"},
9128                {"name": "future", "kind": "query"},
9129                {"name": "reset", "kind": "mutate"}
9130            ],
9131            "config_schema": {}, "observability": [], "identity_scope": []
9132        }]);
9133        body["manifest"]["operation_delivery"] = delivery.clone();
9134        let hello = Frame::build(
9135            FrameType::Hello,
9136            control_flags(),
9137            0,
9138            0,
9139            99,
9140            serde_json::to_vec(&body).unwrap(),
9141        )
9142        .unwrap();
9143        let response = handler
9144            .handle_control(ConnectionId::new(99), hello)
9145            .unwrap();
9146        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9147
9148        let (_, registrations) = registry.list_modules().unwrap();
9149        let registered = &registrations[0].manifest;
9150        assert!(registered.is_watch_operation("status"));
9151        assert!(!registered.is_watch_operation("future"));
9152        assert!(!registered.is_watch_operation("reset"));
9153        assert!(!registered.is_watch_operation("missing"));
9154
9155        let request = Frame::build(
9156            FrameType::Request,
9157            control_flags(),
9158            0,
9159            0,
9160            100,
9161            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9162        )
9163        .unwrap();
9164        let response = handler.handle_catalog_list(request, None).unwrap();
9165        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9166        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9167        let ClientControlResponse::CatalogList { modules, .. } =
9168            serde_json::from_slice(&response[0].body).unwrap()
9169        else {
9170            panic!("expected catalog.list");
9171        };
9172        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9173    }
9174
9175    #[test]
9176    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9177        let registry = Arc::new(Registry::default());
9178        let supervisor = SupervisorHandle::new();
9179        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9180        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9181        let descriptor = json!({
9182            "schema": "ckios.richui.manifest/1",
9183            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9184            "bytes": 4096,
9185            "chunks": 4
9186        });
9187        let connection = ConnectionId::new(201);
9188        let hello = handler
9189            .handle_control(
9190                connection,
9191                hello_frame_with_rich_ui(
9192                    "vault",
9193                    PROTOCOL_VERSION,
9194                    201,
9195                    Some("the-real-nonce"),
9196                    descriptor.clone(),
9197                ),
9198            )
9199            .unwrap();
9200        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9201
9202        let request =
9203            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9204        let provides = registry
9205            .get_module_by_connection(connection)
9206            .unwrap()
9207            .unwrap()
9208            .manifest
9209            .provides;
9210        handler
9211            .handle_catalog_update(connection, request, provides, None, Some(false))
9212            .unwrap();
9213
9214        let request = Frame::build(
9215            FrameType::Request,
9216            control_flags(),
9217            0,
9218            0,
9219            203,
9220            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9221        )
9222        .unwrap();
9223        let response = handler.handle_catalog_list(request, None).unwrap();
9224        let ClientControlResponse::CatalogList { modules, .. } =
9225            serde_json::from_slice(&response[0].body).unwrap()
9226        else {
9227            panic!("expected catalog.list");
9228        };
9229        assert_eq!(
9230            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9231            descriptor
9232        );
9233        assert!(modules[0].nonce_verified);
9234    }
9235
9236    #[test]
9237    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9238        let registry = Arc::new(Registry::default());
9239        let handler = ControlHandler::new(registry.clone());
9240        let descriptor = json!({
9241            "schema": "ckios.richui.manifest/1",
9242            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9243            "bytes": 128,
9244            "chunks": 2
9245        });
9246        let malformed = [
9247            ("schema", json!("")),
9248            ("schema", json!("a".repeat(65))),
9249            ("schema", json!("Bad schema")),
9250            (
9251                "manifest_digest",
9252                json!(format!("sha256:{}", "A".repeat(64))),
9253            ),
9254            ("manifest_digest", json!("sha256:abc")),
9255            ("bytes", json!(0)),
9256            ("bytes", json!(16_777_217)),
9257            ("bytes", json!("128")),
9258            ("chunks", json!(0)),
9259            ("chunks", json!(1025)),
9260            ("chunks", json!("2")),
9261        ];
9262        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9263            let mut invalid = descriptor.clone();
9264            invalid[field] = invalid_value;
9265            let response = handler
9266                .handle_control(
9267                    ConnectionId::new(210 + index as u64),
9268                    hello_frame_with_rich_ui(
9269                        &format!("bad-rich-ui-{index}"),
9270                        PROTOCOL_VERSION,
9271                        210 + index as u64,
9272                        None,
9273                        invalid,
9274                    ),
9275                )
9276                .unwrap();
9277            assert_eq!(response[0].header.ty, FrameType::Error);
9278            let error = parse_error(&response[0]);
9279            assert_eq!(error["code"], "invalid_manifest");
9280            assert!(error["message"]
9281                .as_str()
9282                .unwrap()
9283                .contains(&format!("rich_ui.{field}")));
9284            assert!(registry
9285                .get_module(&format!("bad-rich-ui-{index}"))
9286                .unwrap()
9287                .is_none());
9288        }
9289    }
9290
9291    #[test]
9292    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9293        let registry = Arc::new(Registry::default());
9294        let handler = ControlHandler::new(Arc::clone(&registry));
9295        let connection = ConnectionId::new(219);
9296        registry
9297            .register_with_control_ops(
9298                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9299                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9300                        "bad schema",
9301                        format!("sha256:{}", "a".repeat(64)),
9302                        128,
9303                        2,
9304                    )))
9305                    .build(),
9306                PROTOCOL_VERSION,
9307                connection,
9308                Vec::new(),
9309            )
9310            .unwrap();
9311        let request =
9312            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9313        let response = handler
9314            .handle_catalog_update(connection, request, Vec::new(), None, None)
9315            .unwrap();
9316        assert_eq!(response[0].header.ty, FrameType::Error);
9317        let error = parse_error(&response[0]);
9318        assert_eq!(error["code"], "invalid_manifest");
9319        assert!(error["message"]
9320            .as_str()
9321            .unwrap()
9322            .contains("rich_ui.schema"));
9323    }
9324
9325    #[test]
9326    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9327        let registry = Arc::new(Registry::default());
9328        let supervisor = SupervisorHandle::new();
9329        supervisor.open_swap("provider", "candidate-nonce".to_string());
9330        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9331        let connection = ConnectionId::new(218);
9332        let hello = handler
9333            .handle_control(
9334                connection,
9335                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9336            )
9337            .unwrap();
9338        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9339        assert!(registry.list_modules().unwrap().1.is_empty());
9340
9341        registry.promote_candidate("provider").unwrap().unwrap();
9342        let request = Frame::build(
9343            FrameType::Request,
9344            control_flags(),
9345            0,
9346            0,
9347            219,
9348            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9349        )
9350        .unwrap();
9351        let response = handler.handle_catalog_list(request, None).unwrap();
9352        let ClientControlResponse::CatalogList { modules, .. } =
9353            serde_json::from_slice(&response[0].body).unwrap()
9354        else {
9355            panic!("expected catalog.list");
9356        };
9357        assert_eq!(modules.len(), 1);
9358        assert!(modules[0].nonce_verified);
9359    }
9360
9361    #[test]
9362    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9363        let registry = Arc::new(Registry::default());
9364        let handler = ControlHandler::new(Arc::clone(&registry));
9365        let hello = Frame::build(
9366            FrameType::Hello,
9367            control_flags(),
9368            0,
9369            0,
9370            220,
9371            serde_json::to_vec(&ModuleHelloBody {
9372                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9373                protocol_ver: PROTOCOL_VERSION,
9374                control_ops: None,
9375                launch_nonce: Some("unverified".to_string()),
9376            })
9377            .unwrap(),
9378        )
9379        .unwrap();
9380        assert_eq!(
9381            handler
9382                .handle_control(ConnectionId::new(220), hello)
9383                .unwrap()[0]
9384                .header
9385                .ty,
9386            FrameType::HelloAck
9387        );
9388        let request = Frame::build(
9389            FrameType::Request,
9390            control_flags(),
9391            0,
9392            0,
9393            221,
9394            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9395        )
9396        .unwrap();
9397        let response = handler.handle_catalog_list(request, None).unwrap();
9398        let ClientControlResponse::CatalogList { modules, .. } =
9399            serde_json::from_slice(&response[0].body).unwrap()
9400        else {
9401            panic!("expected catalog.list");
9402        };
9403        assert_eq!(
9404            serde_json::to_vec(&modules[0]).unwrap(),
9405            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9406        );
9407        assert!(!modules[0].nonce_verified);
9408    }
9409
9410    #[test]
9411    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9412        let registry = Arc::new(Registry::default());
9413        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9414            [("vault".to_string(), true), ("squatter".to_string(), true)],
9415            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9416        );
9417        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9418        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9419            provides: vec!["credentials-provider/v1".to_string()],
9420            requires: Vec::new(),
9421            must_never_reach: Vec::new(),
9422        });
9423        let frame = Frame::build(
9424            FrameType::Hello,
9425            control_flags(),
9426            0,
9427            0,
9428            77,
9429            serde_json::to_vec(&ModuleHelloBody {
9430                manifest: squatter,
9431                protocol_ver: PROTOCOL_VERSION,
9432                control_ops: None,
9433                launch_nonce: None,
9434            })
9435            .expect("HELLO serializes"),
9436        )
9437        .expect("HELLO frame builds");
9438        let response = handler
9439            .handle_control(ConnectionId::new(77), frame)
9440            .expect("reserved claim receives a typed refusal");
9441        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9442        assert_eq!(
9443            registry
9444                .active_registration_count()
9445                .expect("registry reads"),
9446            0,
9447            "a reserved capability refusal must not leave a catalog entry"
9448        );
9449    }
9450
9451    #[test]
9452    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9453        for settlement in ["timeout", "inconclusive", "drop"] {
9454            let breakers = RouteBindBreakers::default();
9455            let RouteBindAdmission::Admitted {
9456                guard: mut old,
9457                probe: false,
9458            } = breakers.admit("prov")
9459            else {
9460                panic!("ordinary relay admitted")
9461            };
9462            let RouteBindAdmission::Admitted {
9463                guard: mut opener, ..
9464            } = breakers.admit("prov")
9465            else {
9466                panic!("second relay admitted")
9467            };
9468            assert!(
9469                !opener
9470                    .record_timeout(1, Duration::ZERO)
9471                    .unwrap()
9472                    .reopened_after_probe
9473            );
9474            let RouteBindAdmission::Admitted {
9475                guard: mut probe,
9476                probe: true,
9477            } = breakers.admit("prov")
9478            else {
9479                panic!("one cooldown probe admitted")
9480            };
9481            match settlement {
9482                "timeout" => assert!(
9483                    !old.record_timeout(1, Duration::ZERO)
9484                        .unwrap()
9485                        .reopened_after_probe
9486                ),
9487                "inconclusive" => old.record_inconclusive(),
9488                "drop" => drop(old),
9489                _ => unreachable!(),
9490            }
9491            assert!(
9492                matches!(
9493                    breakers.admit("prov"),
9494                    RouteBindAdmission::Refused {
9495                        probe_in_flight: true,
9496                        ..
9497                    }
9498                ),
9499                "{settlement} of a pre-open relay cannot release the real probe"
9500            );
9501            assert!(
9502                probe
9503                    .record_timeout(1, Duration::ZERO)
9504                    .unwrap()
9505                    .reopened_after_probe
9506            );
9507            assert!(matches!(
9508                breakers.admit("prov"),
9509                RouteBindAdmission::Admitted { probe: true, .. }
9510            ));
9511        }
9512        let breakers = RouteBindBreakers::default();
9513        let admit = || match breakers.admit("prov") {
9514            RouteBindAdmission::Admitted { guard, .. } => guard,
9515            _ => panic!("relay admitted"),
9516        };
9517        admit().record_timeout(1, Duration::ZERO);
9518        let mut old_probe = admit();
9519        breakers.reset_for_new_module_connection("prov");
9520        admit().record_timeout(1, Duration::ZERO);
9521        let _new_probe = admit();
9522        old_probe.record_inconclusive();
9523        assert!(matches!(
9524            breakers.admit("prov"),
9525            RouteBindAdmission::Refused {
9526                probe_in_flight: true,
9527                ..
9528            }
9529        ));
9530    }
9531
9532    #[tokio::test]
9533    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9534        for candidate in [false, true] {
9535            let registry = Arc::new(Registry::default());
9536            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9537                [("vault".to_string(), true), ("squatter".to_string(), true)],
9538                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9539            );
9540            let conn = ConnectionId::new(77);
9541            let (ctx, mut rx) = route_ctx(conn);
9542            let initial = capability_manifest("squatter", &[], &[]);
9543            if candidate {
9544                registry
9545                    .register_candidate_with_control_ops(
9546                        initial.clone(),
9547                        PROTOCOL_VERSION,
9548                        conn,
9549                        module_baseline_control_ops(),
9550                    )
9551                    .unwrap();
9552                handler
9553                    .forwarding
9554                    .register_candidate_module_connection(
9555                        conn,
9556                        "squatter".to_string(),
9557                        PROTOCOL_VERSION,
9558                        manifest_concurrency(&initial),
9559                        ctx.egress.clone(),
9560                    )
9561                    .unwrap();
9562            } else {
9563                hello_via_sink(
9564                    &handler,
9565                    &ctx,
9566                    &mut rx,
9567                    hello_frame_with_manifest(initial.clone(), 1),
9568                )
9569                .await;
9570            }
9571            let response = handler
9572                .handle_control_frame(
9573                    &ctx,
9574                    catalog_update_with_capabilities_frame(
9575                        2,
9576                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
9577                            .capabilities
9578                            .unwrap(),
9579                    ),
9580                )
9581                .await
9582                .unwrap();
9583            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9584            assert_eq!(
9585                registry
9586                    .get_module_by_connection(conn)
9587                    .unwrap()
9588                    .unwrap()
9589                    .manifest,
9590                initial
9591            );
9592        }
9593    }
9594
9595    #[test]
9596    fn server_describe_surfaces_required_capability_verdict_fields() {
9597        let registry = Arc::new(Registry::default());
9598        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9599            [
9600                ("consumer".to_string(), true),
9601                ("provider".to_string(), false),
9602            ],
9603            BTreeMap::new(),
9604        );
9605        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9606        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9607            provides: Vec::new(),
9608            requires: vec![subc_protocol::manifest::CapabilityRequirement {
9609                capability: "credentials-provider/v1".to_string(),
9610                need: subc_protocol::manifest::CapabilityNeed::Required,
9611            }],
9612            must_never_reach: Vec::new(),
9613        });
9614        let hello = Frame::build(
9615            FrameType::Hello,
9616            control_flags(),
9617            0,
9618            0,
9619            78,
9620            serde_json::to_vec(&ModuleHelloBody {
9621                manifest: consumer,
9622                protocol_ver: PROTOCOL_VERSION,
9623                control_ops: None,
9624                launch_nonce: None,
9625            })
9626            .expect("HELLO serializes"),
9627        )
9628        .expect("HELLO frame builds");
9629        handler
9630            .handle_control(ConnectionId::new(78), hello)
9631            .expect("consumer registers");
9632        let describe = Frame::build(
9633            FrameType::Request,
9634            control_flags(),
9635            0,
9636            0,
9637            79,
9638            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9639                .expect("request serializes"),
9640        )
9641        .expect("describe frame builds");
9642        let response = handler
9643            .handle_server_describe(describe)
9644            .expect("server.describe succeeds");
9645        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9646        let requirement = &rendered["capability_requirements"][0];
9647        assert_eq!(requirement["consumer"], "consumer");
9648        assert_eq!(requirement["verdict"], "never_provided");
9649        assert_eq!(requirement["episode_seq"], 1);
9650        assert_eq!(requirement["config_satisfiable"], false);
9651        assert_eq!(requirement["runtime_available"], false);
9652        assert!(requirement["detail"]
9653            .as_str()
9654            .expect("detail string")
9655            .contains("credentials-provider/v1"));
9656    }
9657
9658    /// Register a consumer that requires `project-identity/v1` and the provider
9659    /// of it, optionally mark daemon shutdown as begun, then deregister the
9660    /// provider as its connection closes. Returns the `capability.requirement`
9661    /// events that deregistration logged and the verdict the evaluator then
9662    /// holds for the requirement (the one `route.open` refusals read).
9663    fn deregister_required_provider(
9664        daemon_shutdown_begun: bool,
9665    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
9666        const CAPABILITY: &str = "project-identity/v1";
9667        let registry = Arc::new(Registry::default());
9668        let daemon_shutdown = DaemonShutdownFlag::default();
9669        let handler = ControlHandler::new(Arc::clone(&registry))
9670            .with_capability_config(
9671                [
9672                    ("consumer".to_string(), true),
9673                    ("provider".to_string(), true),
9674                ],
9675                BTreeMap::new(),
9676            )
9677            .with_daemon_shutdown_flag(daemon_shutdown.clone());
9678        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9679        consumer.capabilities = Some(CapabilityDeclarations {
9680            provides: Vec::new(),
9681            requires: vec![subc_protocol::manifest::CapabilityRequirement {
9682                capability: CAPABILITY.to_string(),
9683                need: subc_protocol::manifest::CapabilityNeed::Required,
9684            }],
9685            must_never_reach: Vec::new(),
9686        });
9687        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
9688        let provider_conn = ConnectionId::new(182);
9689        for (conn, manifest, corr) in [
9690            (ConnectionId::new(181), consumer, 181),
9691            (provider_conn, provider, 182),
9692        ] {
9693            let hello = handler
9694                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
9695                .expect("module registers");
9696            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9697        }
9698        assert_eq!(
9699            handler.capability_evaluator.verdict("consumer", CAPABILITY),
9700            Some(CapabilityVerdict::Provided),
9701            "the registered provider must satisfy the requirement first"
9702        );
9703        if daemon_shutdown_begun {
9704            daemon_shutdown.set_for_test();
9705        }
9706
9707        let capture = EventCapture::default();
9708        let _subscriber =
9709            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9710        handler
9711            .cleanup_connection(provider_conn)
9712            .expect("provider deregisters");
9713        let events = capture
9714            .events()
9715            .into_iter()
9716            .filter(|event| {
9717                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
9718            })
9719            .collect();
9720        (
9721            events,
9722            handler.capability_evaluator.verdict("consumer", CAPABILITY),
9723        )
9724    }
9725
9726    #[test]
9727    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
9728        let (events, verdict) = deregister_required_provider(true);
9729        assert_eq!(
9730            verdict,
9731            Some(CapabilityVerdict::NeverProvided),
9732            "shutdown must not change the verdict route.open refusals read"
9733        );
9734        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9735        let event = &events[0];
9736        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
9737        assert_eq!(
9738            event.fields.get("cause").map(String::as_str),
9739            Some("\"daemon_shutdown\""),
9740            "{event:?}"
9741        );
9742        assert_eq!(
9743            event.fields.get("verdict").map(String::as_str),
9744            Some("\"never_provided\""),
9745            "{event:?}"
9746        );
9747    }
9748
9749    #[test]
9750    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
9751        let (events, verdict) = deregister_required_provider(false);
9752        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
9753        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9754        let event = &events[0];
9755        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
9756        assert_eq!(event.fields.get("cause"), None, "{event:?}");
9757        assert_eq!(
9758            event.fields.get("verdict").map(String::as_str),
9759            Some("\"never_provided\""),
9760            "{event:?}"
9761        );
9762    }
9763
9764    #[test]
9765    fn catalog_list_omits_capabilities_for_legacy_manifest() {
9766        let registry = Arc::new(Registry::default());
9767        let handler = ControlHandler::new(Arc::clone(&registry));
9768        let hello = handler
9769            .handle_control(
9770                ConnectionId::new(101),
9771                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9772            )
9773            .expect("legacy HELLO registers");
9774        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9775
9776        let request = Frame::build(
9777            FrameType::Request,
9778            control_flags(),
9779            0,
9780            0,
9781            102,
9782            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9783                .expect("catalog request serializes"),
9784        )
9785        .expect("catalog request frame builds");
9786        let response = handler
9787            .handle_catalog_list(request, None)
9788            .expect("catalog list succeeds");
9789        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9790        assert!(
9791            body["modules"][0].get("capabilities").is_none(),
9792            "legacy manifest must retain an absent capabilities field on catalog.list"
9793        );
9794    }
9795
9796    #[test]
9797    fn hello_ack_omits_storage_when_no_storage_config() {
9798        let registry = Arc::new(Registry::default());
9799        let handler = ControlHandler::new(Arc::clone(&registry));
9800        let responses = handler
9801            .handle_control(
9802                ConnectionId::new(1),
9803                hello_frame("aft", PROTOCOL_VERSION, 7),
9804            )
9805            .unwrap();
9806        let ack = parse_ack(&responses[0]);
9807        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9808        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9809    }
9810
9811    #[tokio::test]
9812    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9813        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9814        let registry = Arc::new(Registry::default());
9815        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
9816        let responses = handler
9817            .handle_control(
9818                ConnectionId::new(1),
9819                hello_frame("aft", PROTOCOL_VERSION, 7),
9820            )
9821            .unwrap();
9822        let ack = parse_ack(&responses[0]);
9823        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9824
9825        let described = handler
9826            .handle_control_frame(
9827                &route_ctx(ConnectionId::new(2)).0,
9828                Frame::build(
9829                    FrameType::Request,
9830                    control_flags(),
9831                    0,
9832                    0,
9833                    9,
9834                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9835                )
9836                .unwrap(),
9837            )
9838            .await
9839            .unwrap();
9840        let ClientControlResponse::ServerDescribe { machine_id, .. } =
9841            serde_json::from_slice(&described[0].body).unwrap()
9842        else {
9843            panic!("server.describe answered with another shape");
9844        };
9845        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9846    }
9847
9848    #[test]
9849    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9850        // With a central sqlite storage policy, each registering module gets its
9851        // own resolved descriptor in HELLO_ACK, keyed by its module id.
9852        let registry = Arc::new(Registry::default());
9853        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
9854            crate::daemon_config::StorageConfig::Sqlite {
9855                data_home: std::path::PathBuf::from("/data"),
9856            },
9857        ));
9858
9859        let responses = handler
9860            .handle_control(
9861                ConnectionId::new(1),
9862                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9863            )
9864            .unwrap();
9865        let ack = parse_ack(&responses[0]);
9866        assert_eq!(
9867            ack.storage,
9868            Some(serde_json::json!({
9869                "module_id": "alfonso-routing",
9870                "storage_namespace": "default",
9871                "isolation": { "kind": "module" },
9872                "backend": {
9873                    "backend": "sqlite",
9874                    "path": "/data/cortexkit/alfonso-routing/store.db"
9875                }
9876            })),
9877            "the delivered descriptor is the module's own sqlite store path"
9878        );
9879    }
9880
9881    #[test]
9882    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9883        let registry = Arc::new(Registry::default());
9884        let handler = ControlHandler::new(Arc::clone(&registry));
9885        let conn = ConnectionId::new(1);
9886        let responses = handler
9887            .handle_control(
9888                conn,
9889                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9890            )
9891            .unwrap();
9892        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9893        let registration = registry.get_module("aft").unwrap().unwrap();
9894        assert_eq!(registration.control_ops, module_baseline_control_ops());
9895
9896        let frame =
9897            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9898        assert!(handler
9899            .guard_module_control_op(&frame, "aft", "route.bind")
9900            .unwrap()
9901            .is_none());
9902        let error = handler
9903            .guard_module_control_op(&frame, "aft", "test.synthetic")
9904            .unwrap()
9905            .expect("synthetic ungranted op should be rejected");
9906        assert_eq!(error.header.ty, FrameType::Error);
9907        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9908    }
9909
9910    #[test]
9911    fn hello_control_ops_some_adds_optional_grants() {
9912        let registry = Arc::new(Registry::default());
9913        let handler = ControlHandler::new(Arc::clone(&registry));
9914        handler
9915            .handle_control(
9916                ConnectionId::new(1),
9917                hello_frame_with_control_ops(
9918                    "aft",
9919                    PROTOCOL_VERSION,
9920                    7,
9921                    Some(vec![
9922                        "future.synthetic".to_string(),
9923                        "route.bind".to_string(),
9924                    ]),
9925                ),
9926            )
9927            .unwrap();
9928        let registration = registry.get_module("aft").unwrap().unwrap();
9929        assert_eq!(
9930            registration.control_ops,
9931            vec![
9932                "route.bind".to_string(),
9933                "route.status".to_string(),
9934                "future.synthetic".to_string(),
9935            ]
9936        );
9937        let frame =
9938            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9939        assert!(handler
9940            .guard_module_control_op(&frame, "aft", "future.synthetic")
9941            .unwrap()
9942            .is_none());
9943    }
9944
9945    #[tokio::test]
9946    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9947        let registry = Arc::new(Registry::default());
9948        let forwarding = Arc::new(ForwardingTable::default());
9949        let handler =
9950            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9951        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9952        hello_via_sink(
9953            &handler,
9954            &module_ctx,
9955            &mut module_rx,
9956            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9957        )
9958        .await;
9959
9960        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9961        let responses = handler
9962            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9963            .await
9964            .unwrap();
9965        assert_eq!(responses.len(), 1);
9966        assert_eq!(responses[0].header.ty, FrameType::Error);
9967        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9968        assert!(module_rx.try_recv().is_err());
9969    }
9970
9971    #[tokio::test]
9972    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9973        let registry = Arc::new(Registry::default());
9974        let forwarding = Arc::new(ForwardingTable::default());
9975        let handler =
9976            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9977        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9978        hello_via_sink(
9979            &handler,
9980            &module_ctx,
9981            &mut module_rx,
9982            hello_frame_with_control_ops(
9983                "aft",
9984                PROTOCOL_VERSION,
9985                7,
9986                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9987            ),
9988        )
9989        .await;
9990
9991        let project_root = unique_project_root("demux");
9992        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9993        let route_handler = handler.clone();
9994        let route_task = tokio::spawn(async move {
9995            route_handler
9996                .handle_control_frame(
9997                    &route_client_ctx,
9998                    route_open_frame(100, "aft", project_root),
9999                )
10000                .await
10001                .unwrap()
10002        });
10003        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10004            .await
10005            .unwrap()
10006            .unwrap();
10007        assert!(matches!(
10008            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10009            ModuleControlRequest::RouteBind { .. }
10010        ));
10011
10012        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10013        let health_handler = handler.clone();
10014        let health_task = tokio::spawn(async move {
10015            health_handler
10016                .handle_control_frame(
10017                    &health_client_ctx,
10018                    supervisor_health_probe_frame(101, "aft"),
10019                )
10020                .await
10021                .unwrap()
10022        });
10023        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10024            .await
10025            .unwrap()
10026            .unwrap();
10027        assert_eq!(
10028            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10029            ModuleControlRequest::HealthCheck {}
10030        );
10031
10032        handler
10033            .handle_control_frame(
10034                &module_ctx,
10035                health_response(health_frame.header.corr, HealthStatus::Degraded),
10036            )
10037            .await
10038            .unwrap();
10039        let health_response = health_task.await.unwrap();
10040        assert_eq!(health_response.len(), 1);
10041        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10042            ClientControlResponse::SupervisorHealthProbe {
10043                module_id,
10044                status,
10045                detail,
10046                metrics,
10047            } => {
10048                assert_eq!(module_id, "aft");
10049                assert_eq!(status, HealthStatus::Degraded);
10050                assert_eq!(detail.as_deref(), Some("warming"));
10051                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10052            }
10053            other => panic!("unexpected health response: {other:?}"),
10054        }
10055
10056        handler
10057            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10058            .await
10059            .unwrap();
10060        let route_response = route_task.await.unwrap();
10061        assert!(route_response.is_empty());
10062        let published = route_client_rx.recv().await.unwrap();
10063        assert!(matches!(
10064            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10065            ClientControlResponse::RouteOpen { .. }
10066        ));
10067    }
10068
10069    /// Start one `route.open` on `client_connection` and return its still-running
10070    /// handler task together with the `route.bind` the module received for it.
10071    /// The handler blocks until the module answers, so it has to run as a task
10072    /// while the test drives the module side.
10073    async fn relay_route_open(
10074        handler: &ControlHandler,
10075        client_connection: ConnectionId,
10076        client_egress: &FrameSink,
10077        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10078        corr: u64,
10079        module_id: &str,
10080        project_root_label: &str,
10081    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10082        let ctx = RouteCtx {
10083            connection_id: client_connection,
10084            egress: client_egress.clone(),
10085        };
10086        let handler = handler.clone();
10087        let project_root = unique_project_root(project_root_label);
10088        let module_id = module_id.to_string();
10089        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10090        let task = tokio::spawn(async move {
10091            let _guard = tracing::dispatcher::set_default(&dispatch);
10092            handler
10093                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10094                .await
10095                .unwrap()
10096        });
10097        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10098            .await
10099            .expect("module receives the relayed route.bind")
10100            .expect("module egress is open");
10101        (task, bind.frame)
10102    }
10103
10104    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10105        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10106            ModuleControlRequest::RouteBind {
10107                route_channel,
10108                epoch,
10109                ..
10110            } => (route_channel, epoch),
10111            other => panic!("expected a route.bind request, got {other:?}"),
10112        }
10113    }
10114
10115    fn published_route(frame: &Frame) -> (u16, u32) {
10116        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10117            ClientControlResponse::RouteOpen {
10118                route_channel,
10119                route_epoch,
10120            } => (route_channel, route_epoch),
10121            other => panic!("expected a route.open response, got {other:?}"),
10122        }
10123    }
10124
10125    /// Reproduction of a production outage. A client had `route.open`s in
10126    /// flight to a module and was already marked closing -- its egress had refused a
10127    /// module frame, so the daemon asked its connection to end -- while its sink
10128    /// was still open. When the module acked those binds, the daemon refused to
10129    /// commit a route for a closing client, and that refusal was returned from
10130    /// the MODULE connection's frame handler, where a router error that has no
10131    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
10132    /// the supervisor correctly did not respawn a clean exit, and every seat lost
10133    /// its tools for hours -- one client's teardown took down a connection
10134    /// carrying ~170 other routes.
10135    ///
10136    /// The window is opened here by calling the production path that opens it
10137    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
10138    /// state that matters is "in `closing_connections`, sink still open, relay
10139    /// still pending", and it lasts only from the close request until the
10140    /// connection loop reacts to it; a socket-level test can flood a client into
10141    /// that escalation but cannot pin the module's ack inside the window. Closing
10142    /// the socket instead takes the other path entirely -- connection teardown
10143    /// removes the pending relay under the same lock, so the ack finds nothing.
10144    #[tokio::test]
10145    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10146        let registry = Arc::new(Registry::default());
10147        let forwarding = Arc::new(ForwardingTable::default());
10148        let handler =
10149            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10150
10151        let module_connection = ConnectionId::new(30);
10152        let (module_ctx, mut module_rx) = route_ctx(module_connection);
10153        hello_via_sink(
10154            &handler,
10155            &module_ctx,
10156            &mut module_rx,
10157            hello_frame("aft", PROTOCOL_VERSION, 7),
10158        )
10159        .await;
10160
10161        let dying_client = ConnectionId::new(31);
10162        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10163
10164        // A published route on the dying client. The escalation below only marks
10165        // a connection closing for a route it has already published.
10166        let (first_task, first_bind) = relay_route_open(
10167            &handler,
10168            dying_client,
10169            &dying_ctx.egress,
10170            &mut module_rx,
10171            100,
10172            "aft",
10173            "closing-first",
10174        )
10175        .await;
10176        handler
10177            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10178            .await
10179            .unwrap();
10180        assert!(first_task.await.unwrap().is_empty());
10181        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10182
10183        // A second route.open from the same client, relayed and awaiting its ack.
10184        let (second_task, second_bind) = relay_route_open(
10185            &handler,
10186            dying_client,
10187            &dying_ctx.egress,
10188            &mut module_rx,
10189            101,
10190            "aft",
10191            "closing-second",
10192        )
10193        .await;
10194        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10195
10196        // The window: the client is closing, its sink is still open, and its
10197        // second bind is still pending.
10198        assert!(forwarding
10199            .escalate_client_delivery_failure(
10200                dying_client,
10201                first_channel,
10202                first_epoch,
10203                CloseReason::new(
10204                    "module_to_client_delivery_failed",
10205                    "client egress refused a module frame",
10206                ),
10207                crate::forwarding::UndeliveredFrame {
10208                    module_id: None,
10209                    sink: &dying_ctx.egress,
10210                },
10211            )
10212            .unwrap());
10213        assert!(!dying_ctx.egress.is_closed());
10214
10215        // The frame that used to end the module connection.
10216        let ack = handler
10217            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10218            .await;
10219        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10220        if module_loop_error.is_some() {
10221            // What the server's connection loop does with a router error that has
10222            // no ERROR-frame translation: end the connection, which releases the
10223            // module's registration and every route on it.
10224            handler.cleanup_connection(module_connection).unwrap();
10225        }
10226        // Read the module's next frame before opening the co-tenant's route, so
10227        // the GOODBYE assertion below is about THIS ack and not about later
10228        // traffic. `None` means the module was told nothing.
10229        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10230            .await
10231            .ok()
10232            .flatten();
10233
10234        // 1. The module connection is still registered.
10235        assert!(
10236            registry
10237                .get_module_by_connection(module_connection)
10238                .unwrap()
10239                .is_some(),
10240            "one client's closing connection ended the shared module connection: \
10241             {module_loop_error:?}"
10242        );
10243        // ...and still serving: another client can open and use a route on it.
10244        let cotenant = ConnectionId::new(32);
10245        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10246        let (cotenant_task, cotenant_bind) = relay_route_open(
10247            &handler,
10248            cotenant,
10249            &cotenant_ctx.egress,
10250            &mut module_rx,
10251            102,
10252            "aft",
10253            "closing-cotenant",
10254        )
10255        .await;
10256        handler
10257            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10258            .await
10259            .unwrap();
10260        assert!(cotenant_task.await.unwrap().is_empty());
10261        let (cotenant_channel, cotenant_epoch) =
10262            published_route(&cotenant_rx.recv().await.unwrap());
10263        assert!(matches!(
10264            forwarding
10265                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10266                .unwrap(),
10267            DataRoute::Client(DataRouteState::Bound(_))
10268        ));
10269
10270        // 2. The module was told to drop the binding it created for the route
10271        //    that will never be published.
10272        let goodbye = post_ack_module_frame
10273            .expect("module receives a GOODBYE for the abandoned route channel");
10274        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10275        assert_eq!(goodbye.header.channel, abandoned_channel);
10276        assert_eq!(goodbye.header.epoch, abandoned_epoch);
10277
10278        // 3. The dying client received nothing: no route was ever published to
10279        //    it. Its route.open is answered as unavailable, which the connection
10280        //    loop would write to a socket that is already going away.
10281        assert!(dying_rx.try_recv().is_err());
10282        let second_response = second_task.await.unwrap();
10283        assert_eq!(second_response.len(), 1);
10284        assert_eq!(
10285            parse_error(&second_response[0])["code"],
10286            "target_unavailable"
10287        );
10288    }
10289
10290    /// One `catalog.list` as a client sees it: the generation and the listed
10291    /// module ids with their readiness.
10292    async fn catalog_snapshot(
10293        handler: &ControlHandler,
10294        client_ctx: &RouteCtx,
10295        corr: u64,
10296    ) -> (u64, Vec<(String, bool)>) {
10297        let body =
10298            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10299        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10300        let reply = handler
10301            .handle_control_frame(client_ctx, request)
10302            .await
10303            .unwrap()
10304            .pop()
10305            .unwrap();
10306        let ClientControlResponse::CatalogList {
10307            generation,
10308            modules,
10309            ..
10310        } = serde_json::from_slice(&reply.body).unwrap()
10311        else {
10312            panic!("not a catalog.list reply");
10313        };
10314        (
10315            generation,
10316            modules
10317                .into_iter()
10318                .map(|entry| (entry.module_id, entry.ready))
10319                .collect(),
10320        )
10321    }
10322
10323    /// A handler whose HELLO for module "aft" on connection 40 is parked
10324    /// between reserving its registry entry and installing its forwarding
10325    /// endpoint. Returns the handler, the two-party barrier (wait once to know
10326    /// the HELLO is parked, once more to release it), the HELLO's thread, and
10327    /// the module's context and receiver.
10328    fn hello_held_before_forwarding() -> (
10329        ControlHandler,
10330        Arc<std::sync::Barrier>,
10331        std::thread::JoinHandle<Vec<Frame>>,
10332        RouteCtx,
10333        mpsc::Receiver<crate::router::OutboundFrame>,
10334    ) {
10335        let hold = Arc::new(std::sync::Barrier::new(2));
10336        let handler = ControlHandler::with_forwarding(
10337            Arc::new(Registry::default()),
10338            Arc::new(ForwardingTable::default()),
10339        )
10340        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10341        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10342        let hello = {
10343            let handler = handler.clone();
10344            let module_ctx = module_ctx.clone();
10345            std::thread::spawn(move || {
10346                handler
10347                    .handle_hello(
10348                        module_ctx.connection_id,
10349                        Some(module_ctx.egress.clone()),
10350                        hello_frame("aft", PROTOCOL_VERSION, 7),
10351                    )
10352                    .unwrap()
10353            })
10354        };
10355        (handler, hold, hello, module_ctx, module_rx)
10356    }
10357
10358    /// While a HELLO has reserved its registry entry but its forwarding
10359    /// endpoint is not installed yet, `route.open` to the module can only be
10360    /// refused. `catalog.list` must not list the module in that window: a
10361    /// client that opens a route as soon as it sees a module listed and ready
10362    /// would be refused for a module the daemon had just advertised. The
10363    /// refusal in the window stays the retryable `target_unavailable`, never
10364    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
10365    /// Once the HELLO completes, the module is listed and a route to it opens.
10366    #[tokio::test]
10367    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10368        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10369        let client = ConnectionId::new(41);
10370        let (client_ctx, mut client_rx) = route_ctx(client);
10371
10372        hold.wait();
10373        // Parked: the registry holds "aft", forwarding has no endpoint for it.
10374        assert!(handler.registry.get_module("aft").unwrap().is_some());
10375        assert!(!handler
10376            .forwarding
10377            .has_live_module_connection("aft")
10378            .unwrap());
10379        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10380        assert!(
10381            listed.is_empty(),
10382            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10383        );
10384        let refused = handler
10385            .handle_control_frame(
10386                &client_ctx,
10387                route_open_frame(301, "aft", unique_project_root("listed-window")),
10388            )
10389            .await
10390            .unwrap();
10391        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10392        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10393        let duplicate = handler
10394            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10395            .await
10396            .unwrap();
10397        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10398
10399        hold.wait();
10400        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10401        let ack = module_rx.try_recv().unwrap().frame;
10402        assert_eq!(ack.header.ty, FrameType::HelloAck);
10403        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10404        assert_eq!(listed, vec![("aft".to_string(), true)]);
10405
10406        let (open_task, bind) = relay_route_open(
10407            &handler,
10408            client,
10409            &client_ctx.egress,
10410            &mut module_rx,
10411            303,
10412            "aft",
10413            "listed-after",
10414        )
10415        .await;
10416        assert!(handler
10417            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10418            .await
10419            .unwrap()
10420            .is_empty());
10421        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10422        published_route(&client_rx.try_recv().unwrap().frame);
10423    }
10424
10425    /// A catalog watcher keys on the generation: when it changes, it re-reads
10426    /// the listing. So the generation must advance exactly when the module
10427    /// appears. If it advanced while the module was still hidden, a watcher
10428    /// would read the new generation without the module, and the module would
10429    /// later appear under that same generation with nothing telling the
10430    /// watcher to look again.
10431    #[tokio::test]
10432    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10433        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10434        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10435        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10436
10437        hold.wait();
10438        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10439        assert!(listed.is_empty());
10440        assert_eq!(
10441            during, before,
10442            "the generation moved while the module was not listed"
10443        );
10444
10445        hold.wait();
10446        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10447        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10448        assert_eq!(listed, vec![("aft".to_string(), true)]);
10449        assert_eq!(
10450            after,
10451            before + 1,
10452            "the module appeared without exactly one generation bump"
10453        );
10454    }
10455
10456    /// A HELLO whose forwarding endpoint cannot be installed (here the
10457    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
10458    /// refused and leaves nothing behind: no registry entry, no catalog entry,
10459    /// and the catalog generation where it was.
10460    #[tokio::test]
10461    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10462        let registry = Arc::new(Registry::default());
10463        let handler = ControlHandler::with_forwarding(
10464            Arc::clone(&registry),
10465            Arc::new(ForwardingTable::default()),
10466        );
10467        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10468        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10469        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10470        drop(module_rx);
10471
10472        let replies = handler
10473            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10474            .await
10475            .unwrap();
10476        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10477        assert!(registry.get_module("aft").unwrap().is_none());
10478        assert!(registry
10479            .get_module_by_connection(module_ctx.connection_id)
10480            .unwrap()
10481            .is_none());
10482        assert_eq!(
10483            catalog_snapshot(&handler, &client_ctx, 301).await,
10484            (before, Vec::new())
10485        );
10486    }
10487
10488    /// A handler whose `route.open` parks after queueing the route.bind and
10489    /// before marking it enqueued, with module "aft" registered on connection
10490    /// 40. Returns the two-party barrier that parks and releases it.
10491    async fn route_open_held_after_bind_send() -> (
10492        ControlHandler,
10493        Arc<tokio::sync::Barrier>,
10494        RouteCtx,
10495        mpsc::Receiver<crate::router::OutboundFrame>,
10496    ) {
10497        let hold = Arc::new(tokio::sync::Barrier::new(2));
10498        let handler = ControlHandler::with_forwarding(
10499            Arc::new(Registry::default()),
10500            Arc::new(ForwardingTable::default()),
10501        )
10502        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10503        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10504        hello_via_sink(
10505            &handler,
10506            &module_ctx,
10507            &mut module_rx,
10508            hello_frame("aft", PROTOCOL_VERSION, 7),
10509        )
10510        .await;
10511        (handler, hold, module_ctx, module_rx)
10512    }
10513
10514    /// A fast module acks the route.bind, and the reader commits the route,
10515    /// before `route.open` has marked the relay enqueued. The daemon used to
10516    /// read the missing relay as an abandoned bind and send the module a
10517    /// GOODBYE for the route it had just committed: the client was told its
10518    /// route was open, the module unbound the channel, and the client's first
10519    /// request was dropped on the module side as unbound.
10520    #[tokio::test]
10521    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10522        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10523        let client = ConnectionId::new(41);
10524        let (client_ctx, mut client_rx) = route_ctx(client);
10525
10526        let (open_task, bind) = relay_route_open(
10527            &handler,
10528            client,
10529            &client_ctx.egress,
10530            &mut module_rx,
10531            200,
10532            "aft",
10533            "acked-before-mark",
10534        )
10535        .await;
10536        let (module_channel, module_epoch) = route_bind_channel(&bind);
10537        // route.open is now parked between queueing the bind and marking it.
10538        hold.wait().await;
10539
10540        // The module answers inside that window, and the route commits.
10541        assert!(handler
10542            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10543            .await
10544            .unwrap()
10545            .is_empty());
10546        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10547
10548        hold.wait().await;
10549        assert!(
10550            open_task.await.unwrap().is_empty(),
10551            "route.open answers accepted"
10552        );
10553
10554        // Every frame route.open could have sent is already queued, because it
10555        // has returned; the module must have been sent nothing.
10556        match module_rx.try_recv() {
10557            Err(mpsc::error::TryRecvError::Empty) => {}
10558            Ok(outbound) => panic!(
10559                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10560                outbound.frame.header.ty,
10561                outbound.frame.header.channel,
10562                outbound.frame.header.epoch
10563            ),
10564            Err(err) => panic!("module egress closed: {err}"),
10565        }
10566
10567        // The client's first request on the route is the next thing the module
10568        // reads, on the binding it just created.
10569        let router = Router::with_control_handler(Arc::new(handler.clone()));
10570        let request = Frame::build(
10571            FrameType::Request,
10572            Flags::new(true, Priority::Interactive, false),
10573            client_channel,
10574            client_epoch,
10575            201,
10576            b"first request".to_vec(),
10577        )
10578        .unwrap();
10579        router
10580            .route_for_connection(&client_ctx, request)
10581            .await
10582            .unwrap();
10583        let delivered = module_rx.try_recv().unwrap().frame;
10584        assert_eq!(delivered.header.ty, FrameType::Request);
10585        assert_eq!(
10586            (delivered.header.channel, delivered.header.epoch),
10587            (module_channel, module_epoch)
10588        );
10589        assert_eq!(delivered.body, b"first request");
10590    }
10591
10592    /// The client goes away while `route.open` is still queueing the bind, so
10593    /// the relay is dropped with no module reply. The teardown cannot send the
10594    /// GOODBYE itself (it could reach the module ahead of the bind), so
10595    /// `route.open` must send it once the bind is queued; otherwise the module
10596    /// keeps a binding the daemon will never commit.
10597    #[tokio::test]
10598    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10599        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10600        let client = ConnectionId::new(42);
10601        let (client_ctx, _client_rx) = route_ctx(client);
10602
10603        let (open_task, bind) = relay_route_open(
10604            &handler,
10605            client,
10606            &client_ctx.egress,
10607            &mut module_rx,
10608            300,
10609            "aft",
10610            "released-before-mark",
10611        )
10612        .await;
10613        let (module_channel, module_epoch) = route_bind_channel(&bind);
10614        hold.wait().await;
10615
10616        handler.cleanup_connection(client).unwrap();
10617        assert!(
10618            module_rx.try_recv().is_err(),
10619            "client teardown must not GOODBYE a bind route.open is still queueing"
10620        );
10621
10622        hold.wait().await;
10623        let response = open_task.await.unwrap();
10624        assert_eq!(response.len(), 1);
10625        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10626
10627        let goodbye = module_rx
10628            .try_recv()
10629            .expect("module is told to drop the binding the daemon abandoned")
10630            .frame;
10631        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10632        assert_eq!(
10633            (goodbye.header.channel, goodbye.header.epoch),
10634            (module_channel, module_epoch)
10635        );
10636        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10637    }
10638
10639    /// The module acks inside the same window, but the client's egress closed
10640    /// first, so the reader refuses to commit. The ack proves the module holds
10641    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
10642    /// send a second one.
10643    #[tokio::test]
10644    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
10645        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10646        let client = ConnectionId::new(43);
10647        let (client_ctx, client_rx) = route_ctx(client);
10648
10649        let (open_task, bind) = relay_route_open(
10650            &handler,
10651            client,
10652            &client_ctx.egress,
10653            &mut module_rx,
10654            400,
10655            "aft",
10656            "gone-client-before-mark",
10657        )
10658        .await;
10659        let (module_channel, module_epoch) = route_bind_channel(&bind);
10660        hold.wait().await;
10661
10662        drop(client_rx);
10663        assert!(client_ctx.egress.is_closed());
10664        handler
10665            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10666            .await
10667            .unwrap();
10668
10669        hold.wait().await;
10670        let response = open_task.await.unwrap();
10671        assert_eq!(response.len(), 1);
10672        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10673
10674        let goodbye = module_rx
10675            .try_recv()
10676            .expect("module is told to drop the binding it acked")
10677            .frame;
10678        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10679        assert_eq!(
10680            (goodbye.header.channel, goodbye.header.epoch),
10681            (module_channel, module_epoch)
10682        );
10683        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10684        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
10685    }
10686
10687    /// The fence at the module-loop boundary, stated as its own contract: which
10688    /// forwarding failures are allowed to end the module connection that is being
10689    /// served. A `ConnectionClosing` naming some client is about that client, and
10690    /// a module connection is shared; the same error naming the module's own
10691    /// connection is about this connection and must stay fatal, as must failures
10692    /// that are about the forwarding table itself.
10693    #[test]
10694    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
10695        let handler = ControlHandler::default();
10696        let module_connection = ConnectionId::new(30);
10697        let client_connection = ConnectionId::new(31);
10698
10699        handler
10700            .refuse_to_end_module_connection_for_a_client(
10701                module_connection,
10702                77,
10703                ForwardingError::ConnectionClosing {
10704                    connection_id: client_connection,
10705                },
10706            )
10707            .expect("a closing client must never end the module connection");
10708
10709        assert!(matches!(
10710            handler.refuse_to_end_module_connection_for_a_client(
10711                module_connection,
10712                78,
10713                ForwardingError::ConnectionClosing {
10714                    connection_id: module_connection,
10715                },
10716            ),
10717            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
10718                connection_id
10719            })) if connection_id == module_connection
10720        ));
10721        assert!(matches!(
10722            handler.refuse_to_end_module_connection_for_a_client(
10723                module_connection,
10724                79,
10725                ForwardingError::Poisoned,
10726            ),
10727            Err(RouterError::Forwarding(ForwardingError::Poisoned))
10728        ));
10729        assert!(matches!(
10730            handler.refuse_to_end_module_connection_for_a_client(
10731                module_connection,
10732                80,
10733                ForwardingError::StaleModuleEndpoint,
10734            ),
10735            Err(RouterError::Forwarding(
10736                ForwardingError::StaleModuleEndpoint
10737            ))
10738        ));
10739    }
10740
10741    /// The spawn-attestation guard is what stops a connected module from claiming
10742    /// another module's identity and being stamped `Reserved` for it. Every other
10743    /// test that supplies a consumer_identity supplies a CORRECT one, because a
10744    /// correct one is what the rest of the flow needs -- so the guard's rejection
10745    /// branch was never the subject of an assertion, only its acceptance branch.
10746    ///
10747    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
10748    /// whole subc-core library suite green; only the forwarding integration tests
10749    /// notice, and they notice for unrelated reasons. This test exists so the
10750    /// refusal itself is asserted where the guard lives: it fails if the identity
10751    /// check stops refusing, which is the direction that matters, since a guard
10752    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
10753    #[tokio::test]
10754    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
10755        let registry = Arc::new(Registry::default());
10756        let forwarding = Arc::new(ForwardingTable::default());
10757        let supervisor = SupervisorHandle::new();
10758        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10759        let handler =
10760            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10761                .with_supervisor(supervisor);
10762
10763        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10764        hello_via_sink(
10765            &handler,
10766            &target_ctx,
10767            &mut target_rx,
10768            hello_frame("target", PROTOCOL_VERSION, 1),
10769        )
10770        .await;
10771
10772        // A real supervised module id presenting the wrong nonce. This is the
10773        // impersonation case: the attacker knows a privileged module_id, which is
10774        // public, and guesses at the nonce, which is not.
10775        let wrong_nonce = handler
10776            .handle_control_frame(
10777                &route_ctx(ConnectionId::new(91)).0,
10778                route_open_frame_with_admission_facts(
10779                    20,
10780                    "target",
10781                    unique_project_root("admission-facts"),
10782                    Some(subc_control::ConsumerIdentity {
10783                        module_id: "fed".to_string(),
10784                        launch_nonce: "not-the-real-nonce".to_string(),
10785                    }),
10786                    None,
10787                ),
10788            )
10789            .await
10790            .unwrap();
10791        assert_eq!(
10792            parse_error(&wrong_nonce[0])["code"],
10793            "bad_consumer_identity",
10794            "a mismatched launch nonce must be refused, not stamped Reserved"
10795        );
10796
10797        // A module id the supervisor never spawned at all, so no nonce exists to
10798        // compare against. An implementation that treats "no record" as "nothing
10799        // to check" fails open here while passing the case above.
10800        let never_spawned = handler
10801            .handle_control_frame(
10802                &route_ctx(ConnectionId::new(92)).0,
10803                route_open_frame_with_admission_facts(
10804                    21,
10805                    "target",
10806                    unique_project_root("admission-facts"),
10807                    Some(subc_control::ConsumerIdentity {
10808                        module_id: "never-spawned".to_string(),
10809                        launch_nonce: "any-nonce".to_string(),
10810                    }),
10811                    None,
10812                ),
10813            )
10814            .await
10815            .unwrap();
10816        assert_eq!(
10817            parse_error(&never_spawned[0])["code"],
10818            "bad_consumer_identity",
10819            "an unspawned module_id must be refused rather than accepted for lack of a record"
10820        );
10821    }
10822
10823    /// The refusal test above proves the guard says NO. Nothing proved it can say
10824    /// YES, and the difference is not academic: replacing the whole authorization
10825    /// with `false` -- admitting no consumer identity at all, revoking Reserved
10826    /// standing for every supervised module in the fleet -- leaves 110 of the 111
10827    /// library tests GREEN. The one that notices does so by HANGING, because it
10828    /// waits for a bind that can no longer happen.
10829    ///
10830    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
10831    /// or flaky test, invites a RETRY rather than an investigation, and the retry
10832    /// hangs too and gets blamed on the runner. So a total revocation of the
10833    /// daemon's trust grant would have shipped behind a symptom nobody attributes
10834    /// to code.
10835    ///
10836    /// The bias is structural rather than accidental. A REFUSAL looks like a
10837    /// failure someone writes a test for; a GRANT looks like the happy path. Every
10838    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
10839    /// suite for that reason, and this one is the purest case in the daemon.
10840    ///
10841    /// This test asserts the EFFECT rather than the absence of an error: the module
10842    /// receives a RouteBind and it carries `Reserved` naming the attested module.
10843    /// A guard that admitted nobody would produce no bind at all; one that admitted
10844    /// everybody would stamp the wrong principal, which the refusal test catches.
10845    #[tokio::test]
10846    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10847        let registry = Arc::new(Registry::default());
10848        let forwarding = Arc::new(ForwardingTable::default());
10849        let supervisor = SupervisorHandle::new();
10850        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10851        let handler =
10852            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10853                .with_supervisor(supervisor);
10854
10855        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10856        hello_via_sink(
10857            &handler,
10858            &target_ctx,
10859            &mut target_rx,
10860            hello_frame("target", PROTOCOL_VERSION, 1),
10861        )
10862        .await;
10863
10864        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10865        let route_handler = handler.clone();
10866        let route_task = tokio::spawn(async move {
10867            route_handler
10868                .handle_control_frame(
10869                    &client_ctx,
10870                    route_open_frame_with_admission_facts(
10871                        30,
10872                        "target",
10873                        unique_project_root("admission-facts"),
10874                        Some(subc_control::ConsumerIdentity {
10875                            module_id: "fed".to_string(),
10876                            launch_nonce: "fed-nonce".to_string(),
10877                        }),
10878                        None,
10879                    ),
10880                )
10881                .await
10882                .unwrap()
10883        });
10884
10885        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
10886        // the very mutation it exists to catch -- a guard that admits nobody -- no
10887        // bind is ever sent, so it HUNG rather than failing. That reproduces the
10888        // exact defect being fixed: a total revocation detected only as a stalled
10889        // suite, which reads as flakiness and invites a retry. An acceptance test
10890        // that waits for an effect must bound the wait, or a red becomes a hang.
10891        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10892            .await
10893            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10894            .expect("module control channel closed before route.bind");
10895        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10896        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10897            panic!("expected route.bind")
10898        };
10899        assert_eq!(
10900            principal,
10901            Some(Principal::Reserved {
10902                module_id: "fed".to_string()
10903            }),
10904            "a correctly attested consumer must be stamped Reserved for its own id"
10905        );
10906
10907        handler
10908            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10909            .await
10910            .unwrap();
10911        assert!(route_task.await.unwrap().is_empty());
10912        assert!(
10913            matches!(
10914                serde_json::from_slice::<ClientControlResponse>(
10915                    &client_rx.recv().await.unwrap().body
10916                )
10917                .unwrap(),
10918                ClientControlResponse::RouteOpen { .. }
10919            ),
10920            "the route must actually open, not merely avoid an error"
10921        );
10922    }
10923
10924    #[tokio::test(start_paused = true)]
10925    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10926        let registry = Arc::new(Registry::default());
10927        let forwarding = Arc::new(ForwardingTable::default());
10928        let supervisor = SupervisorHandle::new();
10929        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10930        let handler =
10931            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10932                .with_supervisor(supervisor);
10933
10934        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10935        hello_via_sink(
10936            &handler,
10937            &target_ctx,
10938            &mut target_rx,
10939            hello_frame("target", PROTOCOL_VERSION, 1),
10940        )
10941        .await;
10942
10943        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10944        let direct_handler = handler.clone();
10945        let direct_open = tokio::spawn(async move {
10946            direct_handler
10947                .handle_control_frame(
10948                    &direct_ctx,
10949                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
10950                )
10951                .await
10952                .unwrap()
10953        });
10954        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10955            .await
10956            .expect("no direct route.bind within 5s")
10957            .expect("target control channel closed before direct route.bind");
10958        handler
10959            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10960            .await
10961            .unwrap();
10962        assert!(direct_open.await.unwrap().is_empty());
10963        let _ = direct_rx.recv().await.unwrap();
10964
10965        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10966        let reserved_handler = handler.clone();
10967        let reserved_open = tokio::spawn(async move {
10968            reserved_handler
10969                .handle_control_frame(
10970                    &reserved_ctx,
10971                    route_open_frame_with_admission_facts(
10972                        3,
10973                        "target",
10974                        unique_project_root("admission-facts"),
10975                        Some(ConsumerIdentity {
10976                            module_id: "fed".to_string(),
10977                            launch_nonce: "fed-nonce".to_string(),
10978                        }),
10979                        None,
10980                    ),
10981                )
10982                .await
10983                .unwrap()
10984        });
10985        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10986            .await
10987            .expect("no reserved route.bind within 5s")
10988            .expect("target control channel closed before reserved route.bind");
10989        handler
10990            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10991            .await
10992            .unwrap();
10993        assert!(reserved_open.await.unwrap().is_empty());
10994        let _ = reserved_rx.recv().await.unwrap();
10995
10996        forwarding
10997            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10998            .unwrap();
10999        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11000        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11001            module_id: Some("target".to_string()),
11002        })
11003        .unwrap();
11004        let census_frame =
11005            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11006        let response = handler
11007            .handle_control_frame(&census_ctx, census_frame)
11008            .await
11009            .unwrap()
11010            .pop()
11011            .unwrap();
11012        let actual: Value = serde_json::from_slice(&response.body).unwrap();
11013        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11014        assert!(matches!(
11015            decoded,
11016            ClientControlResponse::SupervisorRoutes { .. }
11017        ));
11018        let routes = actual["modules"][0]["routes"].as_array().unwrap();
11019        assert_eq!(routes.len(), 2);
11020        assert!(routes.iter().all(|route| route["draining"] == true));
11021        // The census carries WHY: the reason the drain was begun with, in the
11022        // route.closing vocabulary, on every draining route this drain marked.
11023        assert!(
11024            routes.iter().all(|route| route["drain_reason"] == "reload"),
11025            "draining routes must name the drain's reason: {routes:?}"
11026        );
11027        assert!(routes.iter().any(|route| {
11028            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11029        }));
11030        assert!(routes.iter().any(|route| {
11031            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11032        }));
11033
11034        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11035            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11036        if std::env::var_os("UPDATE_GOLDEN").is_some() {
11037            std::fs::write(
11038                &golden_path,
11039                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11040            )
11041            .unwrap();
11042        }
11043        let expected: Value =
11044            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11045        assert_eq!(actual, expected);
11046    }
11047
11048    async fn query_live_roots(
11049        handler: &ControlHandler,
11050        module_ctx: &RouteCtx,
11051    ) -> ModuleControlResponseToModule {
11052        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11053        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11054        let response = handler
11055            .handle_control_frame(module_ctx, frame)
11056            .await
11057            .unwrap()
11058            .pop()
11059            .unwrap();
11060        serde_json::from_slice(&response.body).unwrap()
11061    }
11062
11063    #[tokio::test(start_paused = true)]
11064    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11065        let registry = Arc::new(Registry::default());
11066        let forwarding = Arc::new(ForwardingTable::default());
11067        let handler = ControlHandler::with_forwarding(registry, forwarding);
11068        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11069        hello_via_sink(
11070            &handler,
11071            &target_ctx,
11072            &mut target_rx,
11073            hello_frame("target", PROTOCOL_VERSION, 1),
11074        )
11075        .await;
11076        let root = unique_project_root("live-roots-known");
11077        let path = ProjectRootId::from_path_allowing_missing(root.path())
11078            .unwrap()
11079            .as_path()
11080            .to_path_buf();
11081        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11082        let open_handler = handler.clone();
11083        let opened = tokio::spawn(async move {
11084            open_handler
11085                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11086                .await
11087                .unwrap()
11088        });
11089        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11090            .await
11091            .unwrap()
11092            .unwrap();
11093        handler
11094            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11095            .await
11096            .unwrap();
11097        assert!(opened.await.unwrap().is_empty());
11098        let _ = client_rx.recv().await.unwrap();
11099
11100        let root = unique_project_root("live-roots-pending");
11101        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11102            .unwrap()
11103            .as_path()
11104            .to_path_buf();
11105        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11106        let open_handler = handler.clone();
11107        let pending = tokio::spawn(async move {
11108            open_handler
11109                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11110                .await
11111                .unwrap()
11112        });
11113        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11114            .await
11115            .unwrap()
11116            .unwrap();
11117        let actual = query_live_roots(&handler, &target_ctx).await;
11118        let ModuleControlResponseToModule::LiveRoots {
11119            roots,
11120            unknown_root_bindings,
11121            total_bindings,
11122        } = actual
11123        else {
11124            panic!("expected live roots")
11125        };
11126        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11127        assert_eq!(unknown_root_bindings, 0);
11128        assert_eq!(
11129            roots.len(),
11130            2,
11131            "root-known arm must retain each canonical root"
11132        );
11133        assert_eq!(
11134            total_bindings,
11135            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11136        );
11137        let counts = roots
11138            .iter()
11139            .map(|root| (root.project_root.clone(), root.bound, root.pending))
11140            .collect::<Vec<_>>();
11141        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11142        expected.sort_by(|a, b| a.0.cmp(&b.0));
11143        assert_eq!(
11144            counts, expected,
11145            "roots must sort by path and count pending separately"
11146        );
11147        handler
11148            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11149            .await
11150            .unwrap();
11151        assert!(pending.await.unwrap().is_empty());
11152    }
11153
11154    #[tokio::test(start_paused = true)]
11155    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11156        let forwarding = Arc::new(ForwardingTable::default());
11157        let handler =
11158            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11159        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11160        hello_via_sink(
11161            &handler,
11162            &target_ctx,
11163            &mut target_rx,
11164            hello_frame("target", PROTOCOL_VERSION, 1),
11165        )
11166        .await;
11167        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11168        let pending = forwarding
11169            .begin_route_bind_relay_for_test(
11170                client_ctx.connection_id,
11171                client_ctx.egress.clone(),
11172                2,
11173                "target",
11174            )
11175            .unwrap();
11176        forwarding
11177            .complete_pending_relay(
11178                target_ctx.connection_id,
11179                pending.corr,
11180                RouteBindRelayOutcome::Accepted,
11181            )
11182            .unwrap();
11183        let actual = query_live_roots(&handler, &target_ctx).await;
11184        let ModuleControlResponseToModule::LiveRoots {
11185            roots,
11186            unknown_root_bindings,
11187            total_bindings,
11188        } = actual
11189        else {
11190            panic!("expected live roots")
11191        };
11192        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11193        assert_eq!(
11194            unknown_root_bindings, 1,
11195            "unknown-root arm must not read as no bindings"
11196        );
11197        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11198        assert_eq!(
11199            total_bindings,
11200            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11201        );
11202    }
11203
11204    /// A module reads HELLO_ACK as its first frame and exits on anything else,
11205    /// so the ack has to be on its outbound queue before the module is
11206    /// routable. The connection loop writes a handler's replies only after the
11207    /// handler returns; this test stops in exactly that gap, runs a real
11208    /// route.open from another connection, and only then writes whatever the
11209    /// HELLO handler returned, the way the loop would. If the ack were still a
11210    /// reply, the route.bind request would reach the module first.
11211    #[tokio::test(start_paused = true)]
11212    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11213        let forwarding = Arc::new(ForwardingTable::default());
11214        let handler =
11215            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11216        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11217        let replies = handler
11218            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11219            .await
11220            .unwrap();
11221        let queued_by_hello = module_rx.len();
11222
11223        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11224        let open_handler = handler.clone();
11225        let open = tokio::spawn(async move {
11226            open_handler
11227                .handle_control_frame(
11228                    &client_ctx,
11229                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11230                )
11231                .await
11232                .unwrap()
11233        });
11234        // Let the route.open run until its route.bind is on the module's queue.
11235        let mut spins = 0;
11236        while module_rx.len() == queued_by_hello {
11237            spins += 1;
11238            assert!(spins < 10_000, "route.open never queued a route.bind");
11239            tokio::task::yield_now().await;
11240        }
11241
11242        // Now the connection loop's half: write the HELLO handler's replies.
11243        for reply in replies {
11244            module_ctx.egress.send(reply).await.unwrap();
11245        }
11246
11247        let first = module_rx.recv().await.unwrap().frame;
11248        assert_eq!(
11249            first.header.ty,
11250            FrameType::HelloAck,
11251            "the first frame a registering module reads must be its HELLO_ACK"
11252        );
11253        assert_eq!(first.header.corr, 7);
11254        let second = module_rx.recv().await.unwrap().frame;
11255        assert_eq!(second.header.ty, FrameType::Request);
11256        assert!(
11257            matches!(
11258                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11259                ModuleControlRequest::RouteBind { .. }
11260            ),
11261            "the route.bind follows the ack"
11262        );
11263        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11264
11265        handler
11266            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11267            .await
11268            .unwrap();
11269        assert!(open.await.unwrap().is_empty());
11270        let _ = client_rx.recv().await.unwrap();
11271    }
11272
11273    #[tokio::test(start_paused = true)]
11274    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11275        let handler = ControlHandler::with_forwarding(
11276            Arc::new(Registry::default()),
11277            Arc::new(ForwardingTable::default()),
11278        );
11279        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11280        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11281        hello_via_sink(
11282            &handler,
11283            &first_ctx,
11284            &mut first_rx,
11285            hello_frame("first", PROTOCOL_VERSION, 1),
11286        )
11287        .await;
11288        hello_via_sink(
11289            &handler,
11290            &second_ctx,
11291            &mut second_rx,
11292            hello_frame("second", PROTOCOL_VERSION, 2),
11293        )
11294        .await;
11295        let root = unique_project_root("second-only");
11296        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11297        let cloned = handler.clone();
11298        let open = tokio::spawn(async move {
11299            cloned
11300                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11301                .await
11302                .unwrap()
11303        });
11304        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11305            .await
11306            .unwrap()
11307            .unwrap();
11308        let first = query_live_roots(&handler, &first_ctx).await;
11309        let second = query_live_roots(&handler, &second_ctx).await;
11310        assert!(
11311            matches!(
11312                first,
11313                ModuleControlResponseToModule::LiveRoots {
11314                    total_bindings: 0,
11315                    ..
11316                }
11317            ),
11318            "cross-module scope must not expose another module's roots"
11319        );
11320        assert!(
11321            matches!(
11322                second,
11323                ModuleControlResponseToModule::LiveRoots {
11324                    total_bindings: 1,
11325                    ..
11326                }
11327            ),
11328            "second module must see its pending route"
11329        );
11330        handler
11331            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11332            .await
11333            .unwrap();
11334        assert!(open.await.unwrap().is_empty());
11335    }
11336
11337    #[tokio::test(start_paused = true)]
11338    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11339        let handler = ControlHandler::with_forwarding(
11340            Arc::new(Registry::default()),
11341            Arc::new(ForwardingTable::default()),
11342        );
11343        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11344        hello_via_sink(
11345            &handler,
11346            &target_ctx,
11347            &mut target_rx,
11348            hello_frame("target", PROTOCOL_VERSION, 1),
11349        )
11350        .await;
11351        let actual = query_live_roots(&handler, &target_ctx).await;
11352        let ModuleControlResponseToModule::LiveRoots {
11353            roots,
11354            unknown_root_bindings,
11355            total_bindings,
11356        } = actual
11357        else {
11358            panic!("expected live roots")
11359        };
11360        assert!(roots.is_empty());
11361        assert_eq!(unknown_root_bindings, 0);
11362        assert_eq!(total_bindings, 0);
11363        assert_eq!(
11364            total_bindings,
11365            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11366        );
11367    }
11368
11369    /// Read the vendored fed corpus rather than hand-building a package.
11370    ///
11371    /// A hand-built object encodes what the test author believed the carrier
11372    /// emits. These vectors are what it actually emits, and one of them exists
11373    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
11374    /// ignores additive unknown fields at the traversal emit terminus."
11375    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11376        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11377            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11378        let text = std::fs::read_to_string(&path)
11379            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11380        let vectors: Vec<(String, Value)> = text
11381            .lines()
11382            .filter(|line| !line.trim().is_empty())
11383            .map(|line| {
11384                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11385                let id = entry["corpus_id"]
11386                    .as_str()
11387                    .expect("every vector carries a corpus_id")
11388                    .to_string();
11389                (id, entry["package"].clone())
11390            })
11391            .collect();
11392        // Pin the count: a corpus that silently shrinks would take its coverage
11393        // with it, and a suite reading N-1 vectors reports the same clean pass
11394        // as one reading N.
11395        assert_eq!(
11396            vectors.len(),
11397            3,
11398            "vendored fed corpus changed size; re-sync from subc-federation"
11399        );
11400
11401        // Pin what makes the corpus DISCRIMINATING, not just present.
11402        //
11403        // The relay test below takes its expected value from the corpus, so the
11404        // corpus supplies the test's power to detect a lossy relay rather than
11405        // its correctness. A relay that dropped unrecognised fields would still
11406        // be caught -- but only by a package carrying fields it does not know.
11407        // Shrink every package to the handful of keys any implementation would
11408        // recognise and the test keeps passing over an input that can no longer
11409        // fail, which is the same clean green as a corpus that shrank away.
11410        //
11411        // So assert the precondition rather than duplicating the packages here:
11412        // at least one vector must carry a field beyond the small common set.
11413        // That is one claim to maintain instead of nine, and it fails loudly if
11414        // a re-sync ever flattens the corpus.
11415        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11416        let richest = vectors
11417            .iter()
11418            .filter_map(|(_, package)| package.as_object())
11419            .map(|object| {
11420                object
11421                    .keys()
11422                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11423                    .count()
11424            })
11425            .max()
11426            .unwrap_or(0);
11427        assert!(
11428            richest >= 2,
11429            "vendored corpus no longer carries a package with unmodelled fields, \
11430             so the relay test can no longer distinguish a verbatim relay from a lossy one"
11431        );
11432
11433        vectors
11434    }
11435
11436    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
11437    ///
11438    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
11439    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
11440    /// three-key object, so a relay that quietly dropped fields it did not
11441    /// recognise would satisfy it. These vectors carry nine keys including ones
11442    /// this crate has no type for, so a typed relay fails here and only here.
11443    #[tokio::test]
11444    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11445        for (corpus_id, package) in fed_admission_facts_vectors() {
11446            let registry = Arc::new(Registry::default());
11447            let forwarding = Arc::new(ForwardingTable::default());
11448            let supervisor = SupervisorHandle::new();
11449            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11450            let handler =
11451                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11452                    .with_supervisor(supervisor)
11453                    .with_admission_facts_config(
11454                        Some("fed".to_string()),
11455                        Some(vec!["target".to_string()]),
11456                    );
11457
11458            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11459            hello_via_sink(
11460                &handler,
11461                &target_ctx,
11462                &mut target_rx,
11463                hello_frame("target", PROTOCOL_VERSION, 1),
11464            )
11465            .await;
11466
11467            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11468            let route_handler = handler.clone();
11469            let expected = package.clone();
11470            let route_task = tokio::spawn(async move {
11471                route_handler
11472                    .handle_control_frame(
11473                        &client_ctx,
11474                        route_open_frame_with_admission_facts(
11475                            20,
11476                            "target",
11477                            unique_project_root("admission-facts"),
11478                            Some(subc_control::ConsumerIdentity {
11479                                module_id: "fed".to_string(),
11480                                launch_nonce: "fed-nonce".to_string(),
11481                            }),
11482                            Some(package),
11483                        ),
11484                    )
11485                    .await
11486                    .unwrap()
11487            });
11488
11489            let bind_frame = target_rx.recv().await.unwrap();
11490            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11491            let ModuleControlRequest::RouteBind {
11492                admission_facts, ..
11493            } = bind
11494            else {
11495                panic!("{corpus_id}: expected route.bind")
11496            };
11497            assert_eq!(
11498                admission_facts,
11499                Some(expected),
11500                "{corpus_id}: relay must not add, drop or reshape any field"
11501            );
11502
11503            handler
11504                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11505                .await
11506                .unwrap();
11507            route_task.await.unwrap();
11508        }
11509    }
11510
11511    #[tokio::test]
11512    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11513        let registry = Arc::new(Registry::default());
11514        let forwarding = Arc::new(ForwardingTable::default());
11515        let supervisor = SupervisorHandle::new();
11516        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11517        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11518        let handler =
11519            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11520                .with_supervisor(supervisor)
11521                .with_admission_facts_config(
11522                    Some("fed".to_string()),
11523                    Some(vec!["target".to_string()]),
11524                );
11525
11526        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11527        hello_via_sink(
11528            &handler,
11529            &target_ctx,
11530            &mut target_rx,
11531            hello_frame("target", PROTOCOL_VERSION, 1),
11532        )
11533        .await;
11534        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11535        hello_via_sink(
11536            &handler,
11537            &other_ctx,
11538            &mut other_rx,
11539            hello_frame("other", PROTOCOL_VERSION, 2),
11540        )
11541        .await;
11542
11543        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11544        let expected_facts = facts.clone();
11545        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11546        let route_handler = handler.clone();
11547        let route_task = tokio::spawn(async move {
11548            route_handler
11549                .handle_control_frame(
11550                    &client_ctx,
11551                    route_open_frame_with_admission_facts(
11552                        10,
11553                        "target",
11554                        unique_project_root("admission-facts"),
11555                        Some(subc_control::ConsumerIdentity {
11556                            module_id: "fed".to_string(),
11557                            launch_nonce: "fed-nonce".to_string(),
11558                        }),
11559                        Some(facts.clone()),
11560                    ),
11561                )
11562                .await
11563                .unwrap()
11564        });
11565        let bind_frame = target_rx.recv().await.unwrap();
11566        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11567        let ModuleControlRequest::RouteBind {
11568            admission_facts, ..
11569        } = bind
11570        else {
11571            panic!("expected route.bind")
11572        };
11573        assert_eq!(admission_facts, Some(expected_facts));
11574        handler
11575            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11576            .await
11577            .unwrap();
11578        assert!(route_task.await.unwrap().is_empty());
11579        assert!(matches!(
11580            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11581                .unwrap(),
11582            ClientControlResponse::RouteOpen { .. }
11583        ));
11584
11585        let direct = handler
11586            .handle_control_frame(
11587                &route_ctx(ConnectionId::new(73)).0,
11588                route_open_frame_with_admission_facts(
11589                    11,
11590                    "target",
11591                    unique_project_root("admission-facts"),
11592                    None,
11593                    Some(json!({"x": 1})),
11594                ),
11595            )
11596            .await
11597            .unwrap();
11598        assert_eq!(
11599            parse_error(&direct[0])["code"],
11600            "admission_facts_not_permitted"
11601        );
11602
11603        let different_reserved = handler
11604            .handle_control_frame(
11605                &route_ctx(ConnectionId::new(77)).0,
11606                route_open_frame_with_admission_facts(
11607                    15,
11608                    "target",
11609                    unique_project_root("admission-facts"),
11610                    Some(subc_control::ConsumerIdentity {
11611                        module_id: "other".to_string(),
11612                        launch_nonce: "other-nonce".to_string(),
11613                    }),
11614                    Some(json!({"x": 1})),
11615                ),
11616            )
11617            .await
11618            .unwrap();
11619        assert_eq!(
11620            parse_error(&different_reserved[0])["code"],
11621            "admission_facts_not_permitted"
11622        );
11623
11624        let other_target = handler
11625            .handle_control_frame(
11626                &route_ctx(ConnectionId::new(74)).0,
11627                route_open_frame_with_admission_facts(
11628                    12,
11629                    "other",
11630                    unique_project_root("admission-facts"),
11631                    Some(subc_control::ConsumerIdentity {
11632                        module_id: "fed".to_string(),
11633                        launch_nonce: "fed-nonce".to_string(),
11634                    }),
11635                    Some(json!({"x": 1})),
11636                ),
11637            )
11638            .await
11639            .unwrap();
11640        assert_eq!(
11641            parse_error(&other_target[0])["code"],
11642            "admission_facts_target_not_allowed"
11643        );
11644
11645        let nonexistent = handler
11646            .handle_control_frame(
11647                &route_ctx(ConnectionId::new(75)).0,
11648                route_open_frame_with_admission_facts(
11649                    13,
11650                    "missing",
11651                    unique_project_root("admission-facts"),
11652                    None,
11653                    Some(json!({"x": 1})),
11654                ),
11655            )
11656            .await
11657            .unwrap();
11658        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
11659
11660        let described = handler
11661            .handle_control_frame(
11662                &route_ctx(ConnectionId::new(76)).0,
11663                Frame::build(
11664                    FrameType::Request,
11665                    control_flags(),
11666                    0,
11667                    0,
11668                    14,
11669                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11670                )
11671                .unwrap(),
11672            )
11673            .await
11674            .unwrap();
11675        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11676            serde_json::from_slice(&described[0].body).unwrap()
11677        else {
11678            panic!("expected server.describe response")
11679        };
11680        assert!(capabilities
11681            .iter()
11682            .any(|cap| cap == "admission_facts_relay_v1"));
11683    }
11684
11685    #[tokio::test]
11686    async fn admission_facts_without_configured_carrier_are_rejected() {
11687        let registry = Arc::new(Registry::default());
11688        let forwarding = Arc::new(ForwardingTable::default());
11689        let handler = ControlHandler::with_forwarding(registry, forwarding);
11690        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
11691        hello_via_sink(
11692            &handler,
11693            &target_ctx,
11694            &mut target_rx,
11695            hello_frame("target", PROTOCOL_VERSION, 1),
11696        )
11697        .await;
11698
11699        let responses = handler
11700            .handle_control_frame(
11701                &route_ctx(ConnectionId::new(79)).0,
11702                route_open_frame_with_admission_facts(
11703                    16,
11704                    "target",
11705                    unique_project_root("admission-facts"),
11706                    None,
11707                    Some(json!({"x": 1})),
11708                ),
11709            )
11710            .await
11711            .unwrap();
11712        assert_eq!(
11713            parse_error(&responses[0])["code"],
11714            "admission_facts_not_permitted"
11715        );
11716    }
11717
11718    #[tokio::test]
11719    async fn route_open_relays_consumer_capabilities_verbatim() {
11720        let registry = Arc::new(Registry::default());
11721        let forwarding = Arc::new(ForwardingTable::default());
11722        let handler =
11723            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11724        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
11725        hello_via_sink(
11726            &handler,
11727            &module_ctx,
11728            &mut module_rx,
11729            hello_frame("aft", PROTOCOL_VERSION, 7),
11730        )
11731        .await;
11732
11733        let expected = vec!["elicitation".to_string(), "roots".to_string()];
11734        let expected_for_request = expected.clone();
11735        let project_root = unique_project_root("consumer-capabilities-present");
11736        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
11737        let route_handler = handler.clone();
11738        let route_task = tokio::spawn(async move {
11739            route_handler
11740                .handle_control_frame(
11741                    &client_ctx,
11742                    route_open_frame_with_consumer_capabilities(
11743                        401,
11744                        "aft",
11745                        project_root,
11746                        Some(expected_for_request),
11747                    ),
11748                )
11749                .await
11750                .unwrap()
11751        });
11752        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11753            .await
11754            .unwrap()
11755            .unwrap();
11756        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11757        let ModuleControlRequest::RouteBind {
11758            consumer_capabilities,
11759            ..
11760        } = bind
11761        else {
11762            panic!("expected route.bind request, got {bind:?}");
11763        };
11764        assert_eq!(consumer_capabilities, Some(expected.clone()));
11765
11766        handler
11767            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11768            .await
11769            .unwrap();
11770        let route_response = route_task.await.unwrap();
11771        assert!(route_response.is_empty());
11772        let published = client_rx.recv().await.unwrap();
11773        assert!(matches!(
11774            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11775            ClientControlResponse::RouteOpen { .. }
11776        ));
11777    }
11778
11779    #[tokio::test]
11780    async fn route_open_without_consumer_capabilities_relays_none() {
11781        let registry = Arc::new(Registry::default());
11782        let forwarding = Arc::new(ForwardingTable::default());
11783        let handler =
11784            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11785        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11786        hello_via_sink(
11787            &handler,
11788            &module_ctx,
11789            &mut module_rx,
11790            hello_frame("aft", PROTOCOL_VERSION, 7),
11791        )
11792        .await;
11793
11794        let project_root = unique_project_root("consumer-capabilities-absent");
11795        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11796        let route_handler = handler.clone();
11797        let route_task = tokio::spawn(async move {
11798            route_handler
11799                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11800                .await
11801                .unwrap()
11802        });
11803        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11804            .await
11805            .unwrap()
11806            .unwrap();
11807        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11808        let ModuleControlRequest::RouteBind {
11809            consumer_capabilities,
11810            ..
11811        } = bind
11812        else {
11813            panic!("expected route.bind request, got {bind:?}");
11814        };
11815        assert_eq!(consumer_capabilities, None);
11816
11817        handler
11818            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11819            .await
11820            .unwrap();
11821        let route_response = route_task.await.unwrap();
11822        assert!(route_response.is_empty());
11823        let published = client_rx.recv().await.unwrap();
11824        assert!(matches!(
11825            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11826            ClientControlResponse::RouteOpen { .. }
11827        ));
11828    }
11829
11830    /// Opens a route to a freshly registered `aft` with `sent` as the
11831    /// route.open's role_versions, acks the bind, and returns the role_versions
11832    /// the module's bind carried.
11833    async fn bind_role_versions_for(
11834        sent: Option<BTreeMap<String, String>>,
11835        connection: u64,
11836    ) -> Option<BTreeMap<String, String>> {
11837        let registry = Arc::new(Registry::default());
11838        let forwarding = Arc::new(ForwardingTable::default());
11839        let handler =
11840            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11841        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11842        hello_via_sink(
11843            &handler,
11844            &module_ctx,
11845            &mut module_rx,
11846            hello_frame("aft", PROTOCOL_VERSION, 7),
11847        )
11848        .await;
11849        let project_root = unique_project_root("role-versions");
11850        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11851        let route_handler = handler.clone();
11852        let route_task = tokio::spawn(async move {
11853            route_handler
11854                .handle_control_frame(
11855                    &client_ctx,
11856                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
11857                )
11858                .await
11859                .unwrap()
11860        });
11861        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11862            .await
11863            .expect("a well-formed route.open reaches the module as a bind")
11864            .unwrap();
11865        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11866        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11867            panic!("expected route.bind request, got {bind:?}");
11868        };
11869        handler
11870            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11871            .await
11872            .unwrap();
11873        assert!(route_task.await.unwrap().is_empty());
11874        let published = client_rx.recv().await.unwrap();
11875        assert!(matches!(
11876            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11877            ClientControlResponse::RouteOpen { .. }
11878        ));
11879        role_versions
11880    }
11881
11882    #[tokio::test]
11883    async fn route_open_relays_role_versions_verbatim() {
11884        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11885        assert_eq!(
11886            bind_role_versions_for(Some(sent.clone()), 141).await,
11887            Some(sent)
11888        );
11889    }
11890
11891    /// An empty map declares nothing, so the provider sees no field rather
11892    /// than an empty object it would have to treat as a second "none".
11893    #[tokio::test]
11894    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11895        assert_eq!(bind_role_versions_for(None, 143).await, None);
11896        assert_eq!(
11897            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11898            None
11899        );
11900    }
11901
11902    #[tokio::test]
11903    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11904        let registry = Arc::new(Registry::default());
11905        let forwarding = Arc::new(ForwardingTable::default());
11906        let handler =
11907            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11908        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11909        hello_via_sink(
11910            &handler,
11911            &module_ctx,
11912            &mut module_rx,
11913            hello_frame("aft", PROTOCOL_VERSION, 7),
11914        )
11915        .await;
11916
11917        let nine: BTreeMap<String, String> = (0..9)
11918            .map(|index| (format!("role-{index}"), "v1".to_string()))
11919            .collect();
11920        for (label, malformed) in [
11921            (
11922                "invalid role name",
11923                role_versions(&[("Tool_Provider", "v1")]),
11924            ),
11925            ("invalid version", role_versions(&[("tool-provider", "v0")])),
11926            ("nine entries", nine),
11927        ] {
11928            // A refused open answers at once; one that reached the module
11929            // would wait for its bind ack and trip this timeout.
11930            let responses = tokio::time::timeout(
11931                Duration::from_secs(1),
11932                handler.handle_control_frame(
11933                    &route_ctx(ConnectionId::new(148)).0,
11934                    route_open_frame_with_role_versions(
11935                        404,
11936                        "aft",
11937                        unique_project_root("role-versions-malformed"),
11938                        Some(malformed),
11939                    ),
11940                ),
11941            )
11942            .await
11943            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11944            .unwrap();
11945            assert_eq!(responses.len(), 1, "{label}");
11946            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11947            let error = parse_error(&responses[0]);
11948            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11949            assert_eq!(
11950                error["detail"]["field"], "role_versions",
11951                "{label}: {error}"
11952            );
11953            assert!(
11954                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11955                "{label}: a malformed declaration is terminal"
11956            );
11957            assert!(
11958                module_rx.try_recv().is_err(),
11959                "{label}: the module must never see a bind"
11960            );
11961        }
11962    }
11963
11964    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
11965    /// consumer can tell this daemon forwards the field from one that would
11966    /// drop it.
11967    #[tokio::test]
11968    async fn route_role_versions_capability_is_advertised() {
11969        let handler = ControlHandler::new(Arc::new(Registry::default()));
11970        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11971        let ack = hello_via_sink(
11972            &handler,
11973            &ctx,
11974            &mut rx,
11975            hello_frame("m", PROTOCOL_VERSION, 1),
11976        )
11977        .await;
11978        let ack = parse_ack(&ack);
11979        assert!(
11980            ack.subc_capabilities
11981                .iter()
11982                .any(|c| c == "route-role-versions/v1"),
11983            "{:?}",
11984            ack.subc_capabilities
11985        );
11986
11987        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11988        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11989        let reply = handler
11990            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11991            .await
11992            .unwrap()
11993            .pop()
11994            .unwrap();
11995        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11996            serde_json::from_slice(&reply.body).unwrap()
11997        else {
11998            panic!("not a server.describe reply");
11999        };
12000        assert!(
12001            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12002            "{capabilities:?}"
12003        );
12004    }
12005
12006    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
12007    /// catalog entry without `events` (the module declares none) from a daemon
12008    /// that never reports declared events.
12009    #[tokio::test]
12010    async fn catalog_events_capability_is_advertised() {
12011        let handler = ControlHandler::new(Arc::new(Registry::default()));
12012        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12013        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12014        let reply = handler
12015            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12016            .await
12017            .unwrap()
12018            .pop()
12019            .unwrap();
12020        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12021            serde_json::from_slice(&reply.body).unwrap()
12022        else {
12023            panic!("not a server.describe reply");
12024        };
12025        assert!(
12026            capabilities
12027                .iter()
12028                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12029            "{capabilities:?}"
12030        );
12031    }
12032
12033    #[tokio::test]
12034    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12035        let handler = ControlHandler::new(Arc::new(Registry::default()));
12036        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12037        let ack = hello_via_sink(
12038            &handler,
12039            &module,
12040            &mut module_rx,
12041            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12042        )
12043        .await;
12044        let ack = parse_ack(&ack);
12045        assert!(
12046            ack.subc_capabilities
12047                .iter()
12048                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12049            "{:?}",
12050            ack.subc_capabilities
12051        );
12052        assert!(ack
12053            .subc_capabilities
12054            .iter()
12055            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12056
12057        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12058        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12059        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12060        let reply = handler
12061            .handle_control_frame(&client, frame)
12062            .await
12063            .unwrap()
12064            .pop()
12065            .unwrap();
12066        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12067            serde_json::from_slice(&reply.body).unwrap()
12068        else {
12069            panic!("not a server.describe reply");
12070        };
12071        assert!(
12072            capabilities
12073                .iter()
12074                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12075            "{capabilities:?}"
12076        );
12077        assert!(capabilities
12078            .iter()
12079            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12080    }
12081
12082    #[tokio::test]
12083    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12084        let handler = ControlHandler::new(Arc::new(Registry::default()));
12085        let (module, mut rx) = route_ctx(ConnectionId::new(156));
12086        let ack = hello_via_sink(
12087            &handler,
12088            &module,
12089            &mut rx,
12090            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12091        )
12092        .await;
12093        assert!(parse_ack(&ack)
12094            .subc_capabilities
12095            .iter()
12096            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12097        assert!(parse_ack(&ack)
12098            .subc_capabilities
12099            .iter()
12100            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12101        let (client, _rx) = route_ctx(ConnectionId::new(157));
12102        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12103        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12104        let reply = handler
12105            .handle_control_frame(&client, frame)
12106            .await
12107            .unwrap()
12108            .pop()
12109            .unwrap();
12110        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12111            serde_json::from_slice(&reply.body).unwrap()
12112        else {
12113            panic!("server.describe reply")
12114        };
12115        assert!(capabilities
12116            .iter()
12117            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12118        assert!(capabilities
12119            .iter()
12120            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12121    }
12122
12123    #[tokio::test]
12124    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12125        let handler = ControlHandler::new(Arc::new(Registry::default()));
12126        let (module, mut rx) = route_ctx(ConnectionId::new(158));
12127        let ack = hello_via_sink(
12128            &handler,
12129            &module,
12130            &mut rx,
12131            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12132        )
12133        .await;
12134        assert!(parse_ack(&ack)
12135            .subc_capabilities
12136            .iter()
12137            .any(|cap| cap == "scope-workdir/v1"));
12138        let (client, _rx) = route_ctx(ConnectionId::new(159));
12139        let frame = Frame::build(
12140            FrameType::Request,
12141            control_flags(),
12142            0,
12143            0,
12144            5,
12145            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12146        )
12147        .unwrap();
12148        let reply = handler
12149            .handle_control_frame(&client, frame)
12150            .await
12151            .unwrap()
12152            .remove(0);
12153        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12154            serde_json::from_slice(&reply.body).unwrap()
12155        else {
12156            panic!("server.describe reply")
12157        };
12158        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12159    }
12160
12161    #[tokio::test]
12162    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12163        let handler = ControlHandler::new(Arc::new(Registry::default()));
12164        let (module, mut rx) = route_ctx(ConnectionId::new(154));
12165        let ack = hello_via_sink(
12166            &handler,
12167            &module,
12168            &mut rx,
12169            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12170        )
12171        .await;
12172        assert!(parse_ack(&ack)
12173            .subc_capabilities
12174            .iter()
12175            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12176        let (client, _rx) = route_ctx(ConnectionId::new(155));
12177        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12178        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12179        let reply = handler
12180            .handle_control_frame(&client, frame)
12181            .await
12182            .unwrap()
12183            .pop()
12184            .unwrap();
12185        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12186            serde_json::from_slice(&reply.body).unwrap()
12187        else {
12188            panic!("not a server.describe reply")
12189        };
12190        assert!(capabilities
12191            .iter()
12192            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12193    }
12194
12195    #[tokio::test]
12196    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12197        let registry = Arc::new(Registry::default());
12198        let forwarding = Arc::new(ForwardingTable::default());
12199        let handler =
12200            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12201                .with_health_probe_timeout(Duration::from_secs(5));
12202        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12203        hello_via_sink(
12204            &handler,
12205            &module_ctx,
12206            &mut module_rx,
12207            non_routable_hello_frame_with_control_ops(
12208                "mcp",
12209                300,
12210                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12211            ),
12212        )
12213        .await;
12214        assert!(registry
12215            .get_module("mcp")
12216            .unwrap()
12217            .unwrap()
12218            .manifest
12219            .provides
12220            .is_empty());
12221
12222        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12223        let route_response = handler
12224            .handle_control_frame(
12225                &route_client_ctx,
12226                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12227            )
12228            .await
12229            .unwrap();
12230        assert_eq!(route_response[0].header.ty, FrameType::Error);
12231        assert_eq!(
12232            parse_error(&route_response[0])["code"],
12233            "target_unavailable"
12234        );
12235        assert!(parse_error(&route_response[0])["message"]
12236            .as_str()
12237            .unwrap()
12238            .contains("does not provide the requested target"));
12239        assert!(module_rx.try_recv().is_err());
12240
12241        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12242        let health_handler = handler.clone();
12243        let health_task = tokio::spawn(async move {
12244            health_handler
12245                .handle_control_frame(
12246                    &health_client_ctx,
12247                    supervisor_health_probe_frame(302, "mcp"),
12248                )
12249                .await
12250                .unwrap()
12251        });
12252        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12253            .await
12254            .unwrap()
12255            .unwrap();
12256        assert_eq!(
12257            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12258            ModuleControlRequest::HealthCheck {}
12259        );
12260        handler
12261            .handle_control_frame(
12262                &module_ctx,
12263                health_response(health_frame.header.corr, HealthStatus::Ok),
12264            )
12265            .await
12266            .unwrap();
12267        let health_response = health_task.await.unwrap();
12268        assert_eq!(health_response[0].header.ty, FrameType::Response);
12269        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12270            ClientControlResponse::SupervisorHealthProbe {
12271                module_id, status, ..
12272            } => {
12273                assert_eq!(module_id, "mcp");
12274                assert_eq!(status, HealthStatus::Ok);
12275            }
12276            other => panic!("unexpected health response: {other:?}"),
12277        }
12278
12279        // Exercise the forwarding cleanup path directly while leaving the registry
12280        // advertisement in place. If cleanup leaves a stale control sink behind,
12281        // the next probe will enqueue onto it and wait for the long probe timeout
12282        // instead of returning an immediate no-connection error.
12283        forwarding
12284            .cleanup_connection(module_ctx.connection_id)
12285            .unwrap();
12286        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12287        let cleanup_response = tokio::time::timeout(
12288            Duration::from_millis(200),
12289            handler.handle_control_frame(
12290                &cleanup_probe_ctx,
12291                supervisor_health_probe_frame(303, "mcp"),
12292            ),
12293        )
12294        .await
12295        .expect("probe should fail immediately when the control lane is gone")
12296        .unwrap();
12297        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12298        assert_eq!(
12299            parse_error(&cleanup_response[0])["code"],
12300            "target_unavailable"
12301        );
12302        assert!(parse_error(&cleanup_response[0])["message"]
12303            .as_str()
12304            .unwrap()
12305            .contains("no module connection"));
12306
12307        handler
12308            .cleanup_connection(module_ctx.connection_id)
12309            .unwrap();
12310    }
12311
12312    #[tokio::test]
12313    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12314        let registry = Arc::new(Registry::default());
12315        let supervisor_handle = SupervisorHandle::new();
12316        let supervisor =
12317            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12318                .with_handle(supervisor_handle.clone())
12319                .with_connection_file_path(
12320                    std::env::temp_dir()
12321                        .join(format!("subc-route-open-warming-{}", std::process::id())),
12322                );
12323        let module = supervisor
12324            .supervise_configured(
12325                ModuleSpec {
12326                    module_id: "warming".to_string(),
12327                    program: fake_aft_stub_path(),
12328                    args: Vec::new(),
12329                    env: Vec::new(),
12330                    reserved: false,
12331                    reserved_prefixes: Vec::new(),
12332                    protocol: ModuleProtocol::Subc,
12333                    overlap: Default::default(),
12334                },
12335                true,
12336            )
12337            .unwrap();
12338        assert_eq!(module.state().unwrap(), ModuleState::Running);
12339
12340        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12341        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12342        let response = handler
12343            .handle_control_frame(
12344                &ctx,
12345                route_open_frame(304, "warming", unique_project_root("warming")),
12346            )
12347            .await
12348            .unwrap();
12349        module.stop().await.unwrap();
12350
12351        assert_eq!(response[0].header.ty, FrameType::Error);
12352        let error = parse_error(&response[0]);
12353        assert_eq!(error["code"], "module_warming");
12354        assert!(error["message"]
12355            .as_str()
12356            .unwrap()
12357            .contains("state=running, enabled=true, live=false"));
12358    }
12359
12360    #[test]
12361    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12362        let handler = ControlHandler::new(Arc::new(Registry::default()));
12363        let capture = EventCapture::default();
12364        let _subscriber =
12365            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12366        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12367        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12368        let pending = (0..limit).collect::<Vec<_>>();
12369        let response = handler
12370            .route_open_capacity_refusal(
12371                &ctx,
12372                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12373                "busy",
12374                pending.len(),
12375                limit,
12376            )
12377            .unwrap();
12378        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12379        let event = capture
12380            .events()
12381            .into_iter()
12382            .find(|event| {
12383                event.target == "control"
12384                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12385            })
12386            .expect("connection admission refusal event");
12387        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12388        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12389    }
12390
12391    #[test]
12392    fn route_open_target_cap_logs_admission_reason_and_capacity() {
12393        let handler = ControlHandler::new(Arc::new(Registry::default()));
12394        let capture = EventCapture::default();
12395        let _subscriber =
12396            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12397        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12398        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12399        let guards = (0..limit)
12400            .map(|_| {
12401                handler
12402                    .route_bind_concurrency
12403                    .try_admit("busy", limit)
12404                    .unwrap()
12405            })
12406            .collect::<Vec<_>>();
12407        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12408            Err(in_flight) => in_flight,
12409            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12410        };
12411        let response = handler
12412            .route_open_target_capacity_refusal(
12413                &ctx,
12414                &route_open_frame(397, "busy", unique_project_root("target-cap")),
12415                "busy",
12416                in_flight,
12417            )
12418            .unwrap();
12419        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12420        let event = capture
12421            .events()
12422            .into_iter()
12423            .find(|event| {
12424                event.target == "control"
12425                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12426            })
12427            .expect("target admission refusal event");
12428        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12429        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12430        drop(guards);
12431    }
12432
12433    #[tokio::test]
12434    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
12435        let registry = Arc::new(Registry::default());
12436        let handle = SupervisorHandle::new();
12437        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
12438            .with_handle(handle.clone());
12439        supervisor
12440            .supervise_configured(
12441                crate::ModuleSpec {
12442                    module_id: "drift".into(),
12443                    program: PathBuf::from("disabled-module"),
12444                    args: Vec::new(),
12445                    env: Vec::new(),
12446                    reserved: false,
12447                    reserved_prefixes: Vec::new(),
12448                    protocol: ModuleProtocol::Subc,
12449                    overlap: Default::default(),
12450                },
12451                false,
12452            )
12453            .unwrap();
12454        let handler = ControlHandler::new(registry).with_supervisor(handle);
12455        for configured in [None, Some(false), Some(true)] {
12456            handler.capability_evaluator.configure(
12457                configured.map(|enabled| ("drift".to_string(), enabled)),
12458                BTreeMap::new(),
12459            );
12460            let frame =
12461                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12462            let replies = handler
12463                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
12464                .await
12465                .unwrap();
12466            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12467            assert!(body.get("configured_enabled").is_none());
12468            assert_eq!(body["op"], "supervisor.ack");
12469            let frame =
12470                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12471            let replies = handler.handle_supervisor_list(frame).await.unwrap();
12472            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12473            assert_eq!(
12474                body["modules"][0].get("configured_enabled"),
12475                configured.map(Value::Bool).as_ref()
12476            );
12477        }
12478    }
12479
12480    #[tokio::test(flavor = "current_thread")]
12481    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12482        let handler = ControlHandler::new(Arc::new(Registry::default()));
12483        let capture = EventCapture::default();
12484        let _subscriber =
12485            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12486        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12487        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12488
12489        let _ = handler
12490            .handle_client_control_request(
12491                &ctx,
12492                frame,
12493                ClientControlRequest::SupervisorSetEnabled {
12494                    module_id: "broca".to_string(),
12495                    enabled: false,
12496                },
12497                &mut None,
12498            )
12499            .await
12500            .unwrap();
12501
12502        let events = capture
12503            .events()
12504            .into_iter()
12505            .filter(|event| {
12506                event.target == "control"
12507                    && event.fields.get("message").map(String::as_str)
12508                        == Some("supervisor request received")
12509            })
12510            .collect::<Vec<_>>();
12511        assert_eq!(events.len(), 1, "one supervisor request log line");
12512        let event = &events[0];
12513        assert_eq!(event.level, tracing::Level::INFO);
12514        assert_eq!(
12515            event.fields.get("op"),
12516            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12517        );
12518        assert_eq!(
12519            event.fields.get("module_id"),
12520            Some(&"\"broca\"".to_string())
12521        );
12522        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12523        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12524        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12525    }
12526
12527    #[tokio::test(flavor = "current_thread")]
12528    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12529        let registry = Arc::new(Registry::default());
12530        let module_connection = ConnectionId::new(303);
12531        registry
12532            .register_with_control_ops(
12533                manifest("aft", PROTOCOL_VERSION),
12534                PROTOCOL_VERSION,
12535                module_connection,
12536                module_baseline_control_ops(),
12537            )
12538            .unwrap();
12539        let handler = ControlHandler::new(registry);
12540        let capture = EventCapture::default();
12541        let _subscriber =
12542            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12543        let (ctx, _rx) = route_ctx(module_connection);
12544        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12545
12546        let _ = handler
12547            .handle_client_control_request(
12548                &ctx,
12549                frame,
12550                ClientControlRequest::SupervisorSetEnabled {
12551                    module_id: "broca".to_string(),
12552                    enabled: false,
12553                },
12554                &mut None,
12555            )
12556            .await
12557            .unwrap();
12558
12559        let callers = capture
12560            .events()
12561            .into_iter()
12562            .filter(|event| {
12563                event.fields.get("message").map(String::as_str)
12564                    == Some("supervisor request received")
12565            })
12566            .map(|event| event.fields.get("caller").cloned())
12567            .collect::<Vec<_>>();
12568        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12569    }
12570
12571    #[tokio::test(flavor = "current_thread")]
12572    async fn supervisor_list_does_not_log_request_received() {
12573        let handler = ControlHandler::new(Arc::new(Registry::default()));
12574        let capture = EventCapture::default();
12575        let _subscriber =
12576            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12577        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12578        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12579
12580        handler
12581            .handle_client_control_request(
12582                &ctx,
12583                frame,
12584                ClientControlRequest::SupervisorList {},
12585                &mut None,
12586            )
12587            .await
12588            .unwrap();
12589
12590        assert!(capture.events().into_iter().all(|event| {
12591            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12592        }));
12593    }
12594
12595    #[tokio::test(flavor = "current_thread")]
12596    async fn supervisor_rescan_preview_does_not_log_request_received() {
12597        let handler = ControlHandler::new(Arc::new(Registry::default()));
12598        let capture = EventCapture::default();
12599        let _subscriber =
12600            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12601        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
12602        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12603
12604        handler
12605            .handle_client_control_request(
12606                &ctx,
12607                frame,
12608                ClientControlRequest::SupervisorRescan { preview: true },
12609                &mut None,
12610            )
12611            .await
12612            .unwrap();
12613
12614        assert!(capture.events().into_iter().all(|event| {
12615            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12616        }));
12617    }
12618
12619    struct ApproveOperator;
12620    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
12621        fn prompt(
12622            &self,
12623            _: &str,
12624            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
12625        ) -> crate::operator_confirm::ProviderResult {
12626            crate::operator_confirm::ProviderResult::Approved
12627        }
12628    }
12629
12630    #[tokio::test]
12631    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
12632        assert_eq!(
12633            ControlHandler::default().privileged_operator_prompt,
12634            PrivilegedOperatorPrompt::Log
12635        );
12636        for (initial, changed, running) in [
12637            ("log", "enforce", PrivilegedOperatorPrompt::Log),
12638            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
12639        ] {
12640            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
12641            let path = dir.join("subc.jsonc");
12642            std::fs::write(
12643                &path,
12644                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
12645            )
12646            .unwrap();
12647            let config = crate::daemon_config::load(&path).unwrap().unwrap();
12648            let registry = Arc::new(Registry::default());
12649            let handle = SupervisorHandle::new();
12650            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
12651                .with_handle(handle.clone());
12652            let handler = ControlHandler::new(registry)
12653                .with_supervisor(handle)
12654                .with_privileged_operator_prompt(config.privileged_operator_prompt)
12655                .with_supervisor_rescan(supervisor, &path, config.port);
12656            handler.forwarding.operator_confirms().configure(
12657                Arc::new(ApproveOperator),
12658                crate::operator_confirm::OperatorLimits::default(),
12659            );
12660            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
12661
12662            for (disk_mode, needs_restart) in [
12663                (Some(initial), false),
12664                (Some(changed), true),
12665                (Some(changed), true),
12666                (None, running == PrivilegedOperatorPrompt::Enforce),
12667                (Some(initial), false),
12668            ] {
12669                let mut doc = json!({"version":1});
12670                if let Some(mode) = disk_mode {
12671                    doc["privileged_operator_prompt"] = json!(mode);
12672                }
12673                std::fs::write(&path, doc.to_string()).unwrap();
12674                for preview in [true, false] {
12675                    let request = ClientControlRequest::SupervisorRescan { preview };
12676                    let frame = Frame::build(
12677                        FrameType::Request,
12678                        control_flags(),
12679                        0,
12680                        0,
12681                        1,
12682                        serde_json::to_vec(&request).unwrap(),
12683                    )
12684                    .unwrap();
12685                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12686                    assert_eq!(frames.len(), 1);
12687                    let ClientControlResponse::SupervisorRescan { result } =
12688                        serde_json::from_slice(&frames[0].body).unwrap()
12689                    else {
12690                        panic!("expected a rescan response");
12691                    };
12692                    let expected = if needs_restart {
12693                        vec!["privileged_operator_prompt"]
12694                    } else {
12695                        Vec::new()
12696                    };
12697                    assert_eq!(result.restart_required, expected);
12698                    assert_eq!(handler.privileged_operator_prompt, running);
12699                    assert_eq!(
12700                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
12701                        running
12702                    );
12703                }
12704            }
12705        }
12706    }
12707
12708    #[tokio::test]
12709    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
12710        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
12711        let path = dir.join("subc.jsonc");
12712        let registry = Arc::new(Registry::default());
12713        let handle = SupervisorHandle::new();
12714        let supervisor =
12715            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
12716        let handler = ControlHandler::new(registry)
12717            .with_supervisor(handle)
12718            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
12719            .with_supervisor_rescan(supervisor, &path, None);
12720        handler.forwarding.operator_confirms().configure(
12721            Arc::new(ApproveOperator),
12722            crate::operator_confirm::OperatorLimits::default(),
12723        );
12724        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
12725        for preview in [true, false] {
12726            std::fs::write(
12727                &path,
12728                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
12729            )
12730            .unwrap();
12731            let request = ClientControlRequest::SupervisorRescan { preview };
12732            let frame = Frame::build(
12733                FrameType::Request,
12734                control_flags(),
12735                0,
12736                0,
12737                1,
12738                serde_json::to_vec(&request).unwrap(),
12739            )
12740            .unwrap();
12741            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12742            assert_eq!(frames.len(), 1);
12743            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
12744            assert_eq!(
12745                handler.privileged_operator_prompt,
12746                PrivilegedOperatorPrompt::Enforce
12747            );
12748        }
12749    }
12750
12751    /// One wire code has several senders, so the refusal line names the check
12752    /// that refused. This drives the shared refusal path for ordinary refusals
12753    /// with an unregistered
12754    /// target and requires the branch label on the event.
12755    #[tokio::test]
12756    async fn route_open_refusal_names_the_check_that_refused() {
12757        let handler = ControlHandler::new(Arc::new(Registry::default()));
12758        let capture = EventCapture::default();
12759        let _subscriber =
12760            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12761        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12762        let response = handler
12763            .handle_control_frame(
12764                &ctx,
12765                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
12766            )
12767            .await
12768            .unwrap();
12769
12770        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12771        let event = capture
12772            .events()
12773            .into_iter()
12774            .find(|event| {
12775                event.target == "control"
12776                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12777            })
12778            .expect("route.open refusal event");
12779        assert_eq!(
12780            event.fields.get("reason"),
12781            Some(&"\"not_registered\"".to_string())
12782        );
12783    }
12784
12785    #[tokio::test]
12786    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
12787        let registry = Arc::new(Registry::default());
12788        let supervisor_handle = SupervisorHandle::new();
12789        let supervisor =
12790            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12791                .with_handle(supervisor_handle.clone())
12792                .with_connection_file_path(std::env::temp_dir().join(format!(
12793                    "subc-route-open-refusal-info-{}",
12794                    std::process::id()
12795                )));
12796        let module = supervisor
12797            .supervise_configured(
12798                ModuleSpec {
12799                    module_id: "warming".to_string(),
12800                    program: fake_aft_stub_path(),
12801                    args: Vec::new(),
12802                    env: Vec::new(),
12803                    reserved: false,
12804                    reserved_prefixes: Vec::new(),
12805                    protocol: ModuleProtocol::Subc,
12806                    overlap: Default::default(),
12807                },
12808                true,
12809            )
12810            .unwrap();
12811        assert_eq!(module.state().unwrap(), ModuleState::Running);
12812
12813        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12814        assert!(handler
12815            .counters()
12816            .snapshot()
12817            .get("route_open_refused_by_code")
12818            .is_none());
12819        let capture = EventCapture::default();
12820        let _subscriber =
12821            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12822        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
12823        let response = handler
12824            .handle_control_frame(
12825                &ctx,
12826                route_open_frame(394, "warming", unique_project_root("refusal-info")),
12827            )
12828            .await
12829            .unwrap();
12830        module.stop().await.unwrap();
12831
12832        assert_eq!(parse_error(&response[0])["code"], "module_warming");
12833        let event = capture
12834            .events()
12835            .into_iter()
12836            .find(|event| {
12837                event.target == "control"
12838                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
12839            })
12840            .expect("route.open refusal event");
12841        assert_eq!(
12842            event.fields.get("module_id"),
12843            Some(&"\"warming\"".to_string())
12844        );
12845        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
12846        assert_eq!(
12847            event.fields.get("reason"),
12848            Some(&"\"supervised_not_registered\"".to_string())
12849        );
12850        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
12851        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
12852        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
12853        assert_eq!(
12854            handler.counters().snapshot()["route_open_refused_by_code"],
12855            json!({ "module_warming": 1 })
12856        );
12857    }
12858
12859    const OUTAGE_START: &str = "route.open refusing module: not serving";
12860    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
12861
12862    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
12863        capture
12864            .events()
12865            .into_iter()
12866            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
12867            .collect()
12868    }
12869
12870    fn supervise_stub(
12871        registry: &Arc<Registry>,
12872        module_id: &str,
12873        enabled: bool,
12874    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
12875        let supervisor_handle = SupervisorHandle::new();
12876        let supervisor =
12877            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
12878                .with_handle(supervisor_handle.clone())
12879                .with_connection_file_path(std::env::temp_dir().join(format!(
12880                    "subc-route-outage-{module_id}-{}",
12881                    std::process::id()
12882                )));
12883        let module = supervisor
12884            .supervise_configured(
12885                ModuleSpec {
12886                    module_id: module_id.to_string(),
12887                    program: fake_aft_stub_path(),
12888                    args: Vec::new(),
12889                    env: Vec::new(),
12890                    reserved: false,
12891                    reserved_prefixes: Vec::new(),
12892                    protocol: ModuleProtocol::Subc,
12893                    overlap: Default::default(),
12894                },
12895                enabled,
12896            )
12897            .unwrap();
12898        (supervisor_handle, module)
12899    }
12900
12901    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
12902        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
12903            module_id: module_id.to_string(),
12904            drain_timeout_ms: Some(50),
12905        })
12906        .unwrap();
12907        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
12908    }
12909
12910    /// Two handlers built over one forwarding table must share one outage
12911    /// tracker; separate trackers would each log their own opening line for
12912    /// the same outage.
12913    #[test]
12914    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
12915        let registry = Arc::new(Registry::default());
12916        let forwarding = Arc::new(ForwardingTable::default());
12917        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12918        let second = ControlHandler::with_forwarding(registry, forwarding);
12919        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
12920    }
12921
12922    /// A client can name any module id it likes. Refusing an unknown one,
12923    /// however often, must not create outage state or outage lines, or the
12924    /// tracker would be a memory sink any client could fill.
12925    #[tokio::test(flavor = "current_thread")]
12926    async fn route_open_unknown_module_refusals_add_no_outage_state() {
12927        let handler = ControlHandler::new(Arc::new(Registry::default()));
12928        let capture = EventCapture::default();
12929        let _subscriber =
12930            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12931        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12932        for corr in 0..8 {
12933            let response = handler
12934                .handle_control_frame(
12935                    &ctx,
12936                    route_open_frame(
12937                        380 + corr,
12938                        &format!("nobody-{corr}"),
12939                        unique_project_root("outage-unknown"),
12940                    ),
12941                )
12942                .await
12943                .unwrap();
12944            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12945        }
12946
12947        assert_eq!(handler.route_outages.tracked_module_count(), 0);
12948        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12949        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12950    }
12951
12952    /// Drives the refusal path end to end: a supervised module that served
12953    /// before and stopped being registered with no instruction to stop is a
12954    /// WARN, and the same module refused after an operator `supervisor.restart`
12955    /// is an INFO.
12956    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12957    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12958        let registry = Arc::new(Registry::default());
12959        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
12960        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12961        let capture = EventCapture::default();
12962        let _subscriber =
12963            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12964        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12965        // The stub never registers, so pretend it served once: otherwise every
12966        // refusal would fall in its startup window.
12967        handler.route_outages.record_accepted("outage-restart");
12968
12969        let response = handler
12970            .handle_control_frame(
12971                &ctx,
12972                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12973            )
12974            .await
12975            .unwrap();
12976        assert_eq!(response[0].header.ty, FrameType::Error);
12977        let starts = outage_lines(&capture, OUTAGE_START);
12978        assert_eq!(starts.len(), 1, "{starts:?}");
12979        assert_eq!(starts[0].level, tracing::Level::WARN);
12980        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12981        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12982        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12983        handler.route_outages.record_accepted("outage-restart");
12984        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12985
12986        let restart = handler
12987            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12988            .await
12989            .unwrap();
12990        assert_eq!(
12991            restart[0].header.ty,
12992            FrameType::Response,
12993            "{:?}",
12994            parse_error(&restart[0])
12995        );
12996        handler
12997            .handle_control_frame(
12998                &ctx,
12999                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13000            )
13001            .await
13002            .unwrap();
13003        module.stop().await.unwrap();
13004
13005        let starts = outage_lines(&capture, OUTAGE_START);
13006        assert_eq!(starts.len(), 2, "{starts:?}");
13007        assert_eq!(starts[1].level, tracing::Level::INFO);
13008        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13009    }
13010
13011    /// A restart refused before it touched the module (here: the module is
13012    /// disabled) must clear its operator mark, so the next real outage is
13013    /// still reported as a warning.
13014    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13015    async fn failed_operator_restart_leaves_no_operator_mark() {
13016        let registry = Arc::new(Registry::default());
13017        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
13018        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13019        let capture = EventCapture::default();
13020        let _subscriber =
13021            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13022        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13023        handler.route_outages.record_accepted("outage-disabled");
13024
13025        let restart = handler
13026            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13027            .await
13028            .unwrap();
13029        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13030        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13031
13032        handler
13033            .handle_control_frame(
13034                &ctx,
13035                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13036            )
13037            .await
13038            .unwrap();
13039        let starts = outage_lines(&capture, OUTAGE_START);
13040        assert_eq!(starts.len(), 1, "{starts:?}");
13041        assert_eq!(starts[0].level, tracing::Level::WARN);
13042    }
13043
13044    #[tokio::test(flavor = "current_thread")]
13045    async fn route_open_unknown_module_escapes_target_module_id() {
13046        let handler = ControlHandler::new(Arc::new(Registry::default()));
13047        let capture = EventCapture::default();
13048        let _subscriber =
13049            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13050        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13051        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13052        let response = handler
13053            .handle_control_frame(
13054                &ctx,
13055                route_open_frame(
13056                    395,
13057                    hostile_module_id,
13058                    unique_project_root("hostile-target-module-id"),
13059                ),
13060            )
13061            .await
13062            .unwrap();
13063
13064        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13065        let event = capture
13066            .events()
13067            .into_iter()
13068            .find(|event| {
13069                event.target == "control"
13070                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13071            })
13072            .expect("route.open unknown-module refusal event");
13073        let logged = event.fields.get("module_id").expect("module_id field");
13074        assert!(!logged.bytes().any(|byte| byte < 0x20));
13075        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13076    }
13077
13078    #[tokio::test(flavor = "current_thread")]
13079    async fn route_open_module_rejection_uses_daemon_counter_key() {
13080        let registry = Arc::new(Registry::default());
13081        let forwarding = Arc::new(ForwardingTable::default());
13082        let handler =
13083            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13084        let module_connection = ConnectionId::new(95);
13085        let (module_ctx, mut module_rx) = route_ctx(module_connection);
13086        hello_via_sink(
13087            &handler,
13088            &module_ctx,
13089            &mut module_rx,
13090            hello_frame("aft", PROTOCOL_VERSION, 395),
13091        )
13092        .await;
13093
13094        let client_connection = ConnectionId::new(96);
13095        let (client_ctx, _client_rx) = route_ctx(client_connection);
13096        let capture = EventCapture::default();
13097        let _subscriber =
13098            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13099        let (route_task, bind) = relay_route_open(
13100            &handler,
13101            client_connection,
13102            &client_ctx.egress,
13103            &mut module_rx,
13104            396,
13105            "aft",
13106            "hostile-module-code",
13107        )
13108        .await;
13109        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13110        let rejection = Frame::build(
13111            FrameType::Error,
13112            control_flags(),
13113            0,
13114            0,
13115            bind.header.corr,
13116            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13117        )
13118        .unwrap();
13119        handler
13120            .handle_control_frame(&module_ctx, rejection)
13121            .await
13122            .unwrap();
13123
13124        let response = route_task.await.unwrap();
13125        assert_eq!(parse_error(&response[0])["code"], hostile_code);
13126        let counters = handler.counters().snapshot();
13127        assert_eq!(
13128            counters["route_open_refused_by_code"],
13129            json!({ "module_rejected": 1 })
13130        );
13131        assert!(counters["route_open_refused_by_code"]
13132            .get(hostile_code)
13133            .is_none());
13134
13135        let event = capture
13136            .events()
13137            .into_iter()
13138            .find(|event| {
13139                event.target == "control"
13140                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13141            })
13142            .expect("route.open module-rejection refusal event");
13143        let logged = event.fields.get("module_code").expect("module_code field");
13144        assert!(!logged.bytes().any(|byte| byte < 0x20));
13145        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13146    }
13147
13148    #[tokio::test]
13149    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13150        let registry = Arc::new(Registry::default());
13151        let supervisor_handle = SupervisorHandle::new();
13152        let missing_program = std::env::temp_dir().join(format!(
13153            "subc-route-open-missing-program-{}",
13154            std::process::id()
13155        ));
13156        let supervisor =
13157            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13158                .with_handle(supervisor_handle.clone());
13159        let module = supervisor
13160            .supervise_configured(
13161                ModuleSpec {
13162                    module_id: "failed".to_string(),
13163                    program: missing_program,
13164                    args: Vec::new(),
13165                    env: Vec::new(),
13166                    reserved: false,
13167                    reserved_prefixes: Vec::new(),
13168                    protocol: ModuleProtocol::Subc,
13169                    overlap: Default::default(),
13170                },
13171                true,
13172            )
13173            .unwrap();
13174        assert_eq!(module.state().unwrap(), ModuleState::Failed);
13175
13176        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13177        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13178        let response = handler
13179            .handle_control_frame(
13180                &ctx,
13181                route_open_frame(305, "failed", unique_project_root("failed")),
13182            )
13183            .await
13184            .unwrap();
13185
13186        assert_eq!(response[0].header.ty, FrameType::Error);
13187        let error = parse_error(&response[0]);
13188        assert_eq!(error["code"], "target_unavailable");
13189        assert!(error["message"]
13190            .as_str()
13191            .unwrap()
13192            .contains("state=failed, enabled=true, live=false"));
13193    }
13194
13195    #[tokio::test]
13196    async fn route_open_role_mismatch_remains_target_unavailable() {
13197        let registry = Arc::new(Registry::default());
13198        let handler = ControlHandler::new(Arc::clone(&registry));
13199        handler
13200            .handle_control(
13201                ConnectionId::new(41),
13202                non_routable_hello_frame_with_control_ops("health-only", 306, None),
13203            )
13204            .unwrap();
13205
13206        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13207        let response = handler
13208            .handle_control_frame(
13209                &ctx,
13210                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13211            )
13212            .await
13213            .unwrap();
13214
13215        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13216        assert!(parse_error(&response[0])["message"]
13217            .as_str()
13218            .unwrap()
13219            .contains("does not provide the requested target"));
13220    }
13221
13222    #[tokio::test]
13223    async fn route_open_inactive_registration_remains_target_unavailable() {
13224        let registry = Arc::new(Registry::default());
13225        let handler = ControlHandler::new(Arc::clone(&registry));
13226        handler
13227            .handle_control(
13228                ConnectionId::new(43),
13229                hello_frame("inactive", PROTOCOL_VERSION, 308),
13230            )
13231            .unwrap();
13232        assert!(registry
13233            .set_module_state_for_test("inactive", ChannelState::Closed)
13234            .unwrap());
13235
13236        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13237        let response = handler
13238            .handle_control_frame(
13239                &ctx,
13240                route_open_frame(309, "inactive", unique_project_root("inactive")),
13241            )
13242            .await
13243            .unwrap();
13244
13245        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13246        assert!(parse_error(&response[0])["message"]
13247            .as_str()
13248            .unwrap()
13249            .contains("is not active"));
13250    }
13251
13252    #[tokio::test]
13253    async fn late_health_reply_is_recorded_through_the_module_response_path() {
13254        let registry = Arc::new(Registry::default());
13255        let forwarding = Arc::new(ForwardingTable::default());
13256        let supervisor_handle = SupervisorHandle::new();
13257        let supervisor =
13258            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
13259                .with_forwarding(Arc::clone(&forwarding))
13260                .with_handle(supervisor_handle.clone());
13261        let module = supervisor
13262            .supervise_configured(
13263                crate::ModuleSpec {
13264                    module_id: "late-health-response".to_string(),
13265                    program: PathBuf::from("disabled-module"),
13266                    args: Vec::new(),
13267                    env: Vec::new(),
13268                    reserved: false,
13269                    reserved_prefixes: Vec::new(),
13270                    protocol: ModuleProtocol::Subc,
13271                    overlap: Default::default(),
13272                },
13273                false,
13274            )
13275            .unwrap();
13276        let handler =
13277            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13278                .with_supervisor(supervisor_handle);
13279        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13280        handler
13281            .handle_control_frame(
13282                &module_ctx,
13283                hello_frame_with_control_ops(
13284                    "late-health-response",
13285                    PROTOCOL_VERSION,
13286                    7,
13287                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13288                ),
13289            )
13290            .await
13291            .unwrap();
13292        let probe_started_at = Instant::now() - Duration::from_millis(80);
13293        let pending = forwarding
13294            .begin_health_probe_rpc_for(
13295                "late-health-response",
13296                MODULE_CONTROL_OP_HEALTH_CHECK,
13297                probe_started_at,
13298                Instant::now() - Duration::from_millis(1),
13299            )
13300            .unwrap();
13301        assert!(forwarding
13302            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13303            .unwrap());
13304
13305        let responses = handler
13306            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13307            .await
13308            .unwrap();
13309
13310        assert!(responses.is_empty());
13311        let health = module.status().unwrap().health;
13312        assert_eq!(health.late_answer_count, 1);
13313        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13314    }
13315
13316    #[tokio::test]
13317    async fn health_probe_timeout_and_module_death_are_typed() {
13318        let registry = Arc::new(Registry::default());
13319        let forwarding = Arc::new(ForwardingTable::default());
13320        let handler =
13321            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13322                .with_health_probe_timeout(Duration::from_millis(50));
13323        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13324        hello_via_sink(
13325            &handler,
13326            &module_ctx,
13327            &mut module_rx,
13328            hello_frame_with_control_ops(
13329                "aft",
13330                PROTOCOL_VERSION,
13331                7,
13332                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13333            ),
13334        )
13335        .await;
13336
13337        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13338        let responses = handler
13339            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
13340            .await
13341            .unwrap();
13342        assert_eq!(responses[0].header.ty, FrameType::Error);
13343        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
13344        let _ = module_rx.try_recv();
13345
13346        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
13347        let health_handler = handler.clone();
13348        let death_task = tokio::spawn(async move {
13349            health_handler
13350                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
13351                .await
13352                .unwrap()
13353        });
13354        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13355            .await
13356            .unwrap()
13357            .unwrap();
13358        handler
13359            .cleanup_connection(module_ctx.connection_id)
13360            .unwrap();
13361        let responses = death_task.await.unwrap();
13362        assert_eq!(responses[0].header.ty, FrameType::Error);
13363        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
13364    }
13365
13366    #[test]
13367    fn hello_requires_exact_protocol_version() {
13368        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
13369            let registry = Arc::new(Registry::default());
13370            let handler = ControlHandler::new(Arc::clone(&registry));
13371            let responses = handler
13372                .handle_control(
13373                    ConnectionId::new(connection),
13374                    hello_frame("aft", offered, 9),
13375                )
13376                .unwrap();
13377
13378            assert_eq!(responses.len(), 1);
13379            assert_eq!(responses[0].header.ty, FrameType::Error);
13380            let error = parse_error(&responses[0]);
13381            assert_eq!(error["code"], "version_unsupported");
13382            assert!(registry.get_module("aft").unwrap().is_none());
13383            assert_eq!(registry.active_registration_count().unwrap(), 0);
13384        }
13385    }
13386
13387    #[test]
13388    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
13389        let registry = Arc::new(Registry::default());
13390        let forwarding = Arc::new(ForwardingTable::default());
13391        let handler =
13392            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13393        let module_connection = ConnectionId::new(301);
13394        let registration = registry
13395            .register_with_control_ops(
13396                manifest("aft-push", PROTOCOL_VERSION),
13397                PROTOCOL_VERSION,
13398                module_connection,
13399                module_baseline_control_ops(),
13400            )
13401            .unwrap();
13402        let (module_tx, _module_rx) = mpsc::channel(8);
13403        let endpoint = forwarding
13404            .register_module_connection(
13405                module_connection,
13406                "aft-push".to_string(),
13407                PROTOCOL_VERSION,
13408                manifest_concurrency(&registration.manifest),
13409                FrameSink::new(module_tx),
13410            )
13411            .unwrap();
13412
13413        // A push op this version does not know is ignored (forward-compat), not errored.
13414        let unknown = Frame::build(
13415            FrameType::Push,
13416            control_flags(),
13417            0,
13418            0,
13419            5,
13420            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
13421        )
13422        .unwrap();
13423        let out = handler.handle_status_update(endpoint, unknown).unwrap();
13424        assert!(
13425            out.is_empty(),
13426            "unknown push op must be ignored, got {out:?}"
13427        );
13428
13429        // A malformed body for a KNOWN op is a real error worth surfacing.
13430        let malformed = Frame::build(
13431            FrameType::Push,
13432            control_flags(),
13433            0,
13434            0,
13435            6,
13436            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13437        )
13438        .unwrap();
13439        let out = handler.handle_status_update(endpoint, malformed).unwrap();
13440        assert_eq!(out.len(), 1);
13441        assert_eq!(out[0].header.ty, FrameType::Error);
13442        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13443    }
13444
13445    #[test]
13446    fn hello_rejected_when_connection_already_owns_client_routes() {
13447        let registry = Arc::new(Registry::default());
13448        let forwarding = Arc::new(ForwardingTable::default());
13449        let handler =
13450            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13451        // Commits a client route on connection 202 (bound to a module on conn 101).
13452        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
13453        let client_connection = ConnectionId::new(202);
13454
13455        // That same connection now tries to register as a module: rejected, so one
13456        // connection never holds both client-route and module-endpoint state.
13457        let responses = handler
13458            .handle_control(
13459                client_connection,
13460                hello_frame("aft-second", PROTOCOL_VERSION, 9),
13461            )
13462            .unwrap();
13463        assert_eq!(responses[0].header.ty, FrameType::Error);
13464        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13465        assert!(registry.get_module("aft-second").unwrap().is_none());
13466    }
13467
13468    #[tokio::test]
13469    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13470        let registry = Arc::new(Registry::default());
13471        let forwarding = Arc::new(ForwardingTable::default());
13472        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13473        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13474        hello_via_sink(
13475            &handler,
13476            &module_ctx,
13477            &mut module_rx,
13478            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13479        )
13480        .await;
13481        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13482        let pending = forwarding
13483            .begin_route_bind_relay_for_test(
13484                client_ctx.connection_id,
13485                client_ctx.egress.clone(),
13486                2,
13487                "alpha",
13488            )
13489            .unwrap();
13490        forwarding
13491            .complete_pending_relay(
13492                module_ctx.connection_id,
13493                pending.corr,
13494                RouteBindRelayOutcome::Accepted,
13495            )
13496            .unwrap();
13497        client_rx.try_recv().unwrap();
13498        for module_id in ["beta", "alpha"] {
13499            let replies = handler
13500                .handle_control_frame(
13501                    &module_ctx,
13502                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13503                )
13504                .await
13505                .unwrap();
13506            assert_eq!(replies.len(), 1, "second HELLO must be refused");
13507            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13508        }
13509        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13510        assert!(registry.get_module("beta").unwrap().is_none());
13511        assert!(matches!(
13512            forwarding
13513                .lookup_data_route(
13514                    client_ctx.connection_id,
13515                    pending.client_channel,
13516                    pending.client_epoch,
13517                )
13518                .unwrap(),
13519            DataRoute::Client(DataRouteState::Bound(_))
13520        ));
13521        assert!(handler
13522            .hello_launch_nonces
13523            .lock()
13524            .unwrap()
13525            .presented(module_ctx.connection_id, Some("alpha-nonce")));
13526        assert!(module_rx.try_recv().is_err());
13527    }
13528
13529    #[test]
13530    fn reserved_module_hello_requires_matching_launch_nonce() {
13531        let registry = Arc::new(Registry::default());
13532        let supervisor = SupervisorHandle::new();
13533        // The supervisor recorded the nonce it injected when it spawned the reserved
13534        // module; the HELLO verifier checks against the same shared handle.
13535        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13536        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13537
13538        // A HELLO with NO nonce is rejected.
13539        let no_nonce = handler
13540            .handle_control(
13541                ConnectionId::new(1),
13542                hello_frame("vault", PROTOCOL_VERSION, 1),
13543            )
13544            .unwrap();
13545        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13546        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13547        assert!(registry.get_module("vault").unwrap().is_none());
13548
13549        // A HELLO with the WRONG nonce is rejected.
13550        let wrong = handler
13551            .handle_control(
13552                ConnectionId::new(2),
13553                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13554            )
13555            .unwrap();
13556        assert_eq!(wrong[0].header.ty, FrameType::Error);
13557        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13558        assert!(registry.get_module("vault").unwrap().is_none());
13559
13560        // A HELLO with the CORRECT nonce registers.
13561        let ok = handler
13562            .handle_control(
13563                ConnectionId::new(3),
13564                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13565            )
13566            .unwrap();
13567        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13568        assert!(registry.get_module("vault").unwrap().is_some());
13569    }
13570
13571    #[test]
13572    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13573        let registry = Arc::new(Registry::default());
13574        let supervisor = SupervisorHandle::new();
13575        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13576        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13577        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13578
13579        let squat = handler
13580            .handle_control(
13581                ConnectionId::new(1),
13582                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
13583            )
13584            .unwrap();
13585        assert_eq!(squat[0].header.ty, FrameType::Error);
13586        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
13587        assert!(parse_error(&squat[0])["message"]
13588            .as_str()
13589            .unwrap()
13590            .contains("fed:"));
13591
13592        let accepted_peer = handler
13593            .handle_control(
13594                ConnectionId::new(2),
13595                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
13596            )
13597            .unwrap();
13598        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
13599
13600        let accepted_short = handler
13601            .handle_control(
13602                ConnectionId::new(3),
13603                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
13604            )
13605            .unwrap();
13606        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
13607
13608        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
13609            let response = handler
13610                .handle_control(
13611                    ConnectionId::new(conn),
13612                    hello_frame(module_id, PROTOCOL_VERSION, conn),
13613                )
13614                .unwrap();
13615            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
13616        }
13617    }
13618
13619    #[test]
13620    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
13621        let registry = Arc::new(Registry::default());
13622        let supervisor = SupervisorHandle::new();
13623        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13624        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13625        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
13626        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13627
13628        let owner_nonce = handler
13629            .handle_control(
13630                ConnectionId::new(1),
13631                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
13632            )
13633            .unwrap();
13634        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
13635        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
13636        assert!(registry.get_module("fed:special").unwrap().is_none());
13637
13638        let exact_nonce = handler
13639            .handle_control(
13640                ConnectionId::new(2),
13641                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
13642            )
13643            .unwrap();
13644        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
13645        assert!(registry.get_module("fed:special").unwrap().is_some());
13646    }
13647
13648    #[test]
13649    fn non_reserved_module_ignores_launch_nonce() {
13650        let registry = Arc::new(Registry::default());
13651        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
13652        // registration succeeds whether a spawned process echoes a nonce or not.
13653        let handler = ControlHandler::new(Arc::clone(&registry));
13654        let no_nonce = handler
13655            .handle_control(
13656                ConnectionId::new(1),
13657                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
13658            )
13659            .unwrap();
13660        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
13661        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
13662
13663        let echoed_nonce = handler
13664            .handle_control(
13665                ConnectionId::new(2),
13666                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
13667            )
13668            .unwrap();
13669        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
13670        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
13671    }
13672
13673    #[test]
13674    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
13675        let handler = ControlHandler::default();
13676        let conn = ConnectionId::new(1);
13677        let malformed = Frame::build(
13678            FrameType::Hello,
13679            control_flags(),
13680            0,
13681            0,
13682            3,
13683            b"{not json".to_vec(),
13684        )
13685        .unwrap();
13686
13687        let error = handler.handle_control(conn, malformed).unwrap();
13688        assert_eq!(error[0].header.ty, FrameType::Error);
13689        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
13690
13691        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13692        let pong = handler.handle_control(conn, ping).unwrap();
13693        assert_eq!(pong[0].header.ty, FrameType::Pong);
13694        assert_eq!(pong[0].header.corr, 4);
13695    }
13696
13697    #[test]
13698    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
13699        let registry = Arc::new(Registry::default());
13700        let handler = ControlHandler::new(Arc::clone(&registry));
13701
13702        handler
13703            .handle_control(
13704                ConnectionId::new(1),
13705                hello_frame("aft", PROTOCOL_VERSION, 1),
13706            )
13707            .unwrap();
13708        let duplicate = handler
13709            .handle_control(
13710                ConnectionId::new(2),
13711                hello_frame("aft", PROTOCOL_VERSION, 2),
13712            )
13713            .unwrap();
13714
13715        assert_eq!(duplicate[0].header.ty, FrameType::Error);
13716        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13717        let registration = registry.get_module("aft").unwrap().unwrap();
13718        assert_eq!(registration.connection_id, ConnectionId::new(1));
13719    }
13720
13721    #[test]
13722    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
13723        let registry = Arc::new(Registry::default());
13724        let forwarding = Arc::new(ForwardingTable::default());
13725        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
13726        let handler =
13727            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13728                .with_process_liveness(process_liveness);
13729        let (ctx, route_channel, route_epoch) =
13730            bind_liveness_route(&registry, &forwarding, "aft-dead");
13731        let responses = handler
13732            .handle_route_poll(
13733                &ctx,
13734                route_poll_frame(41, PollKind::Liveness, route_channel),
13735                route_channel,
13736                route_epoch,
13737                PollKind::Liveness,
13738            )
13739            .unwrap();
13740
13741        assert_eq!(responses.len(), 1);
13742        assert_eq!(responses[0].header.ty, FrameType::Response);
13743        assert_route_poll_liveness(&responses[0], false);
13744    }
13745
13746    #[test]
13747    fn liveness_poll_without_process_source_uses_bound_route() {
13748        let registry = Arc::new(Registry::default());
13749        let forwarding = Arc::new(ForwardingTable::default());
13750        let handler =
13751            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13752        let (ctx, route_channel, route_epoch) =
13753            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
13754        let responses = handler
13755            .handle_route_poll(
13756                &ctx,
13757                route_poll_frame(42, PollKind::Liveness, route_channel),
13758                route_channel,
13759                route_epoch,
13760                PollKind::Liveness,
13761            )
13762            .unwrap();
13763
13764        assert_route_poll_liveness(&responses[0], true);
13765    }
13766
13767    #[test]
13768    fn liveness_poll_untracked_process_source_uses_bound_route() {
13769        let registry = Arc::new(Registry::default());
13770        let forwarding = Arc::new(ForwardingTable::default());
13771        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
13772        let handler =
13773            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13774                .with_process_liveness(process_liveness);
13775        let (ctx, route_channel, route_epoch) =
13776            bind_liveness_route(&registry, &forwarding, "aft-untracked");
13777        let responses = handler
13778            .handle_route_poll(
13779                &ctx,
13780                route_poll_frame(43, PollKind::Liveness, route_channel),
13781                route_channel,
13782                route_epoch,
13783                PollKind::Liveness,
13784            )
13785            .unwrap();
13786
13787        assert_route_poll_liveness(&responses[0], true);
13788    }
13789
13790    #[tokio::test]
13791    async fn unknown_op_returns_unknown_control_op() {
13792        let handler = ControlHandler::default();
13793        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
13794        let request = Frame::build(
13795            FrameType::Request,
13796            control_flags(),
13797            0,
13798            0,
13799            55,
13800            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
13801        )
13802        .unwrap();
13803
13804        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13805
13806        assert_eq!(response.len(), 1);
13807        assert_eq!(response[0].header.ty, FrameType::Error);
13808        assert_eq!(response[0].header.corr, 55);
13809        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
13810    }
13811
13812    #[tokio::test]
13813    async fn supervisor_provenance_rejects_unknown_exact_module() {
13814        let handler = ControlHandler::default();
13815        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
13816        let request = Frame::build(
13817            FrameType::Request,
13818            control_flags(),
13819            0,
13820            0,
13821            57,
13822            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
13823        )
13824        .unwrap();
13825
13826        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13827
13828        assert_eq!(response.len(), 1);
13829        assert_eq!(response[0].header.ty, FrameType::Error);
13830        assert_eq!(response[0].header.corr, 57);
13831        let error = parse_error(&response[0]);
13832        assert_eq!(error["code"], "unknown_module");
13833        assert_eq!(error["message"], "module_id 'missing' is not supervised");
13834    }
13835
13836    #[test]
13837    fn provenance_probe_override_keeps_handler_tests_deterministic() {
13838        let expected = subc_control::RunningImageAgreement::Unavailable {
13839            reason: subc_control::RunningImageUnavailableReason::HashFailed,
13840        };
13841        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
13842        assert_eq!(handler.provenance_probe_override, Some(expected));
13843    }
13844
13845    #[test]
13846    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
13847        let verdict = reload_verdict(
13848            std::path::Path::new("/bin/new"),
13849            Some(std::path::Path::new("/bin/old")),
13850            subc_control::RunningImageAgreement::Unavailable {
13851                reason: subc_control::RunningImageUnavailableReason::HashFailed,
13852            },
13853        );
13854        assert!(matches!(
13855            verdict.path,
13856            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
13857                if configured == std::path::Path::new("/bin/new")
13858                    && spawned_from == std::path::Path::new("/bin/old")
13859        ));
13860    }
13861
13862    #[test]
13863    fn launch_spec_without_running_digest_is_unavailable_not_match() {
13864        use subc_control::{
13865            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
13866        };
13867        let configured = crate::supervise::ModuleSpec {
13868            module_id: "legacy".into(),
13869            program: "/module".into(),
13870            args: Vec::new(),
13871            env: Vec::new(),
13872            reserved: false,
13873            reserved_prefixes: Vec::new(),
13874            protocol: subc_control::ModuleProtocol::Subc,
13875            overlap: Default::default(),
13876        };
13877        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
13878        assert_eq!(
13879            launch_spec_agreement(&configured, false, Some(&digest)),
13880            Agreement::Unavailable {
13881                reason: Reason::NotRunning
13882            }
13883        );
13884        assert_eq!(
13885            launch_spec_agreement(&configured, true, None),
13886            Agreement::Unavailable {
13887                reason: Reason::SpawnedSpecUnavailable
13888            }
13889        );
13890        assert_eq!(
13891            launch_spec_agreement(&configured, true, Some(&digest)),
13892            Agreement::Match
13893        );
13894    }
13895
13896    #[cfg(windows)]
13897    #[test]
13898    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
13899        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
13900        let configured = dir.join("module.exe");
13901        std::fs::write(&configured, b"executable path fixture").unwrap();
13902        let spawned = std::fs::canonicalize(&configured).unwrap();
13903        assert_ne!(
13904            configured, spawned,
13905            "fixture must exercise different path spellings"
13906        );
13907        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
13908        let lower_case = configured.to_string_lossy().to_lowercase();
13909        for spelling in [
13910            &configured,
13911            Path::new(&forward_slashes),
13912            Path::new(&lower_case),
13913        ] {
13914            let verdict = reload_verdict(
13915                spelling,
13916                Some(&spawned),
13917                subc_control::RunningImageAgreement::Unavailable {
13918                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13919                },
13920            );
13921            assert_eq!(
13922                verdict.path,
13923                ReloadPathAgreement::Match,
13924                "configured {spelling:?}, spawned {spawned:?}"
13925            );
13926        }
13927    }
13928
13929    #[cfg(windows)]
13930    #[test]
13931    fn windows_reload_path_keeps_different_existing_program_pending() {
13932        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
13933        let old = dir.join("old.exe");
13934        let new = dir.join("new.exe");
13935        std::fs::write(&old, b"same executable content").unwrap();
13936        std::fs::copy(&old, &new).unwrap();
13937        let spawned = std::fs::canonicalize(&old).unwrap();
13938        let image = subc_control::RunningImageAgreement::Unavailable {
13939            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13940        };
13941        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
13942        assert!(
13943            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
13944            if configured == new && spawned_from == spawned)
13945        );
13946        let missing = dir.join("missing.exe");
13947        assert!(matches!(
13948            reload_verdict(&missing, Some(&spawned), image).path,
13949            ReloadPathAgreement::Mismatch { .. }
13950        ));
13951    }
13952
13953    #[test]
13954    fn reload_verdict_detects_replaced_image_at_same_path() {
13955        let image = subc_control::RunningImageAgreement::Mismatch {
13956            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13957                digest: "old".into(),
13958            },
13959            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13960                digest: "new".into(),
13961            },
13962        };
13963        let verdict = reload_verdict(
13964            std::path::Path::new("/bin/same"),
13965            Some(std::path::Path::new("/bin/same")),
13966            image.clone(),
13967        );
13968        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13969        assert_eq!(verdict.image, image);
13970    }
13971
13972    #[test]
13973    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13974        let image = subc_control::RunningImageAgreement::Unavailable {
13975            reason: subc_control::RunningImageUnavailableReason::NotRunning,
13976        };
13977        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13978        assert_eq!(
13979            verdict.path,
13980            subc_control::ReloadPathAgreement::Unavailable {
13981                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13982            }
13983        );
13984        assert_eq!(verdict.image, image);
13985
13986        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13987            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13988        };
13989        let verdict = reload_verdict(
13990            std::path::Path::new("/bin/same"),
13991            Some(std::path::Path::new("/bin/same")),
13992            unconfirmed.clone(),
13993        );
13994        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13995        assert_eq!(verdict.image, unconfirmed);
13996    }
13997
13998    #[test]
13999    fn reload_verdict_preserves_each_image_unavailability_reason() {
14000        use subc_control::RunningImageUnavailableReason as Reason;
14001
14002        for reason in [
14003            Reason::NotRunning,
14004            Reason::UnsupportedPlatform,
14005            Reason::RunningExecutableUnreadable,
14006            Reason::SpawnedPathUnreadable,
14007            Reason::HashFailed,
14008            Reason::ProcessIdentityUnconfirmed,
14009            Reason::Unknown("future_probe_reason".to_string()),
14010        ] {
14011            let image = subc_control::RunningImageAgreement::Unavailable {
14012                reason: reason.clone(),
14013            };
14014            let verdict = reload_verdict(
14015                std::path::Path::new("/bin/same"),
14016                Some(std::path::Path::new("/bin/same")),
14017                image.clone(),
14018            );
14019            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14020            assert_eq!(verdict.image, image, "{reason:?}");
14021        }
14022    }
14023
14024    #[tokio::test]
14025    async fn malformed_control_bodies_return_invalid_control_body() {
14026        let handler = ControlHandler::default();
14027        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14028
14029        for (corr, body) in [
14030            (56, br#"{"route_channel":1}"#.as_slice()),
14031            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14032            (
14033                58,
14034                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14035            ),
14036        ] {
14037            let request = Frame::build(
14038                FrameType::Request,
14039                control_flags(),
14040                0,
14041                0,
14042                corr,
14043                body.to_vec(),
14044            )
14045            .unwrap();
14046            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14047
14048            assert_eq!(response.len(), 1);
14049            assert_eq!(response[0].header.ty, FrameType::Error);
14050            assert_eq!(response[0].header.corr, corr);
14051            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14052        }
14053    }
14054
14055    #[tokio::test]
14056    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14057        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14058        let registry = Arc::new(Registry::default());
14059        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14060        let router = Router::with_control_handler(Arc::clone(&control));
14061        let connection = router.begin_connection();
14062        let (ctx, mut rx) = route_ctx(connection.id());
14063
14064        router
14065            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14066            .await
14067            .unwrap();
14068        let response = rx.recv().await.unwrap();
14069        let ack = parse_ack(&response);
14070        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14071        let channel = 1;
14072
14073        let goodbye =
14074            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14075        router.route_for_connection(&ctx, goodbye).await.unwrap();
14076        assert!(rx.try_recv().is_err());
14077        assert!(registry.get_module("aft").unwrap().is_none());
14078
14079        router
14080            .route_for_connection(&ctx, channel_request(channel, 13))
14081            .await
14082            .unwrap();
14083        let error_frame = rx.recv().await.unwrap();
14084        assert_eq!(error_frame.header.ty, FrameType::Error);
14085        assert_eq!(error_frame.header.channel, channel);
14086        let captured = crate::router::test_log::captured_logs(&logs);
14087        assert_eq!(
14088            captured
14089                .lines()
14090                .filter(|line| {
14091                    line.contains("module_id=aft")
14092                        && line.contains("reason=explicit_goodbye")
14093                        && line.contains("module registration ended")
14094                })
14095                .count(),
14096            1,
14097            "unexpected GOODBYE registry log: {captured}"
14098        );
14099    }
14100
14101    #[tokio::test]
14102    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14103        let registry = Arc::new(Registry::default());
14104        let handler = ControlHandler::new(registry).with_capability_config(
14105            [("prov".to_string(), true), ("cons".to_string(), true)],
14106            BTreeMap::new(),
14107        );
14108        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14109        register_capability_manifest(
14110            &handler,
14111            &provider_ctx,
14112            &mut provider_rx,
14113            capability_manifest("prov", &["thing/v1"], &[]),
14114            1,
14115        )
14116        .await;
14117        let mut consumer = capability_manifest("cons", &[], &[]);
14118        consumer.capabilities.as_mut().unwrap().requires.push(
14119            subc_protocol::manifest::CapabilityRequirement {
14120                capability: "thing/v1".to_string(),
14121                need: subc_protocol::manifest::CapabilityNeed::Required,
14122            },
14123        );
14124        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14125        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14126        assert_eq!(
14127            handler.capability_evaluator.verdict("cons", "thing/v1"),
14128            Some(CapabilityVerdict::Provided)
14129        );
14130        let (mut client_rx, _) = open_route_for_capability_test(
14131            &handler,
14132            &provider_ctx,
14133            &mut provider_rx,
14134            703,
14135            3,
14136            "prov",
14137            None,
14138        )
14139        .await;
14140        let goodbye =
14141            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14142        handler
14143            .handle_control_frame(&provider_ctx, goodbye)
14144            .await
14145            .unwrap();
14146        assert_eq!(
14147            handler.capability_evaluator.verdict("cons", "thing/v1"),
14148            Some(CapabilityVerdict::NeverProvided)
14149        );
14150        let closed = client_rx
14151            .try_recv()
14152            .expect("GOODBYE pushes route.closed before route GOODBYE");
14153        assert!(
14154            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14155            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14156        );
14157        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14158        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14159    }
14160
14161    struct RouteClosedFixture {
14162        handler: ControlHandler,
14163        #[cfg(unix)]
14164        supervisor: Supervisor,
14165        module_ctx: RouteCtx,
14166        #[cfg(unix)]
14167        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14168        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14169        route: Arc<crate::forwarding::RouteBinding>,
14170    }
14171
14172    async fn route_closed_fixture() -> RouteClosedFixture {
14173        let registry = Arc::new(Registry::default());
14174        let handler = ControlHandler::new(Arc::clone(&registry));
14175        #[cfg(unix)]
14176        let supervisor = Supervisor::new(registry, RestartPolicy::default())
14177            .with_forwarding(Arc::clone(&handler.forwarding));
14178        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14179        register_capability_manifest(
14180            &handler,
14181            &module_ctx,
14182            &mut module_rx,
14183            capability_manifest("prov", &[], &[]),
14184            1,
14185        )
14186        .await;
14187        let (client_rx, bind) = open_route_for_capability_test(
14188            &handler,
14189            &module_ctx,
14190            &mut module_rx,
14191            702,
14192            2,
14193            "prov",
14194            None,
14195        )
14196        .await;
14197        let ModuleControlRequest::RouteBind {
14198            route_channel: channel,
14199            epoch: route_epoch,
14200            ..
14201        } = bind
14202        else {
14203            panic!("expected route.bind");
14204        };
14205        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14206            handler
14207                .forwarding
14208                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14209                .unwrap()
14210        else {
14211            panic!("the acknowledged route must be bound");
14212        };
14213        RouteClosedFixture {
14214            handler,
14215            #[cfg(unix)]
14216            supervisor,
14217            module_ctx,
14218            #[cfg(unix)]
14219            module_rx,
14220            client_rx,
14221            route,
14222        }
14223    }
14224
14225    impl RouteClosedFixture {
14226        #[cfg(unix)]
14227        async fn cut(&mut self) {
14228            let notices = async {
14229                let mut draining = self.module_rx.recv().await.unwrap();
14230                assert!(matches!(
14231                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14232                        &draining.body
14233                    )
14234                    .unwrap(),
14235                    subc_protocol::session::ModuleControlCommand::Draining {
14236                        reason: RouteCloseReason::Restart,
14237                        ..
14238                    }
14239                ));
14240                draining.flushed.take().unwrap().send(()).unwrap();
14241                let mut closing = self.client_rx.recv().await.unwrap();
14242                assert!(matches!(
14243                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14244                    ClientControlPush::RouteClosing {
14245                        reason: RouteCloseReason::Restart,
14246                        ..
14247                    }
14248                ));
14249                closing.flushed.take().unwrap().send(()).unwrap();
14250            };
14251            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14252            result.unwrap();
14253        }
14254
14255        fn disconnect(&mut self) -> ClientControlPush {
14256            self.handler
14257                .cleanup_connection(self.module_ctx.connection_id)
14258                .unwrap();
14259            let closed = self.client_rx.try_recv().unwrap();
14260            assert_eq!(closed.header.ty, FrameType::Push);
14261            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14262            assert!(matches!(
14263                &push,
14264                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14265                    if channels == &[self.route.client_channel]
14266            ));
14267            assert_eq!(
14268                self.client_rx.try_recv().unwrap().header.ty,
14269                FrameType::Goodbye
14270            );
14271            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14272            push
14273        }
14274    }
14275
14276    #[cfg(unix)]
14277    #[tokio::test(start_paused = true)]
14278    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14279        let mut fixture = route_closed_fixture().await;
14280        fixture.cut().await;
14281        assert!(matches!(
14282            fixture.disconnect(),
14283            ClientControlPush::RouteClosed {
14284                reason: RouteCloseReason::Restart,
14285                drained: true,
14286                ..
14287            }
14288        ));
14289    }
14290
14291    #[cfg(unix)]
14292    #[tokio::test(start_paused = true)]
14293    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14294        let mut fixture = route_closed_fixture().await;
14295        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14296        let started = tokio::time::Instant::now();
14297        fixture.cut().await;
14298        assert!(started.elapsed() >= Duration::from_secs(2));
14299        assert_eq!(fixture.route.flow.in_flight(), 1);
14300        assert!(matches!(
14301            fixture.disconnect(),
14302            ClientControlPush::RouteClosed {
14303                reason: RouteCloseReason::Restart,
14304                drained: false,
14305                ..
14306            }
14307        ));
14308    }
14309
14310    #[tokio::test]
14311    async fn module_connection_crash_reports_crash_and_not_drained() {
14312        let mut fixture = route_closed_fixture().await;
14313        // The server calls cleanup_connection on module EOF. Without a planned
14314        // daemon shutdown, that disconnect must remain a crash with no prior
14315        // route.closing notice, even when the route has no in-flight calls.
14316        assert!(matches!(
14317            fixture.disconnect(),
14318            ClientControlPush::RouteClosed {
14319                reason: RouteCloseReason::Crash,
14320                drained: false,
14321                ..
14322            }
14323        ));
14324    }
14325
14326    #[tokio::test]
14327    async fn dropping_router_connection_releases_registration() {
14328        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14329        let registry = Arc::new(Registry::default());
14330        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14331        let router = Router::with_control_handler(Arc::clone(&control));
14332        let connection = router.begin_connection();
14333        let connection_id = connection.id();
14334        let (ctx, mut rx) = route_ctx(connection_id);
14335
14336        router
14337            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
14338            .await
14339            .unwrap();
14340        let response = rx.recv().await.unwrap();
14341        let ack = parse_ack(&response);
14342        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14343        assert!(registry.get_module("aft").unwrap().is_some());
14344
14345        drop(connection);
14346
14347        assert!(registry.get_module("aft").unwrap().is_none());
14348        assert_eq!(registry.active_registration_count().unwrap(), 0);
14349
14350        control
14351            .cleanup_connection(ConnectionId::new(u64::MAX))
14352            .unwrap();
14353        let captured = crate::router::test_log::captured_logs(&logs);
14354        println!("captured registry lifecycle logs:\n{captured}");
14355        let events: Vec<_> = captured
14356            .lines()
14357            .filter(|line| {
14358                line.contains("module registered module_id=aft ")
14359                    || line.contains("module registration ended")
14360            })
14361            .collect();
14362        assert_eq!(
14363            events.len(),
14364            2,
14365            "unexpected registry lifecycle logs: {captured}"
14366        );
14367        assert!(events[0].contains("module registered module_id=aft "));
14368        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
14369        assert!(events[1].contains(&format!(
14370            "module_id=aft connection_id={}",
14371            connection_id.get()
14372        )));
14373        assert!(events[1].contains("reason=connection_closed"));
14374        assert!(events[1].contains("module registration ended"));
14375        assert_eq!(
14376            captured
14377                .lines()
14378                .filter(|line| line.contains("module registered module_id=aft "))
14379                .count(),
14380            1,
14381            "legacy registration admission line must appear once: {captured}"
14382        );
14383    }
14384
14385    #[tokio::test]
14386    async fn hello_registration_keeps_legacy_module_registered_line_once() {
14387        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14388        let registry = Arc::new(Registry::default());
14389        let control = ControlHandler::new(Arc::clone(&registry));
14390        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
14391        hello_via_sink(
14392            &control,
14393            &ctx,
14394            &mut rx,
14395            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
14396        )
14397        .await;
14398
14399        let captured = crate::router::test_log::captured_logs(&logs);
14400        let admissions: Vec<_> = captured
14401            .lines()
14402            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
14403            .collect();
14404        assert_eq!(
14405            admissions.len(),
14406            1,
14407            "legacy admission line must remain exactly once: {captured}"
14408        );
14409        assert!(admissions[0].contains("routable_provider=true"));
14410        assert!(admissions[0].contains("connection_id=777"));
14411    }
14412
14413    fn capability_manifest(
14414        module_id: &str,
14415        provides: &[&str],
14416        must_never_reach: &[&str],
14417    ) -> ModuleManifest {
14418        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
14419        manifest.capabilities = Some(CapabilityDeclarations {
14420            provides: provides
14421                .iter()
14422                .map(|capability| (*capability).to_string())
14423                .collect(),
14424            requires: Vec::new(),
14425            must_never_reach: must_never_reach
14426                .iter()
14427                .map(|capability| (*capability).to_string())
14428                .collect(),
14429        });
14430        manifest
14431    }
14432
14433    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14434        Frame::build(
14435            FrameType::Hello,
14436            control_flags(),
14437            0,
14438            0,
14439            corr,
14440            serde_json::to_vec(&ModuleHelloBody {
14441                protocol_ver: manifest.protocol_ver,
14442                manifest,
14443                control_ops: None,
14444                launch_nonce: None,
14445            })
14446            .expect("capability test HELLO serializes"),
14447        )
14448        .expect("capability test HELLO frame builds")
14449    }
14450
14451    fn catalog_update_with_capabilities_frame(
14452        corr: u64,
14453        capabilities: CapabilityDeclarations,
14454    ) -> Frame {
14455        Frame::build(
14456            FrameType::Request,
14457            control_flags(),
14458            0,
14459            0,
14460            corr,
14461            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14462                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14463                capabilities: Some(capabilities),
14464                ready: None,
14465            })
14466            .expect("capability catalog.update serializes"),
14467        )
14468        .expect("capability catalog.update frame builds")
14469    }
14470
14471    async fn register_capability_manifest(
14472        handler: &ControlHandler,
14473        ctx: &RouteCtx,
14474        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14475        manifest: ModuleManifest,
14476        corr: u64,
14477    ) {
14478        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14479    }
14480
14481    async fn open_route_for_capability_test(
14482        handler: &ControlHandler,
14483        target_ctx: &RouteCtx,
14484        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14485        client_connection_id: u64,
14486        corr: u64,
14487        target_module_id: &str,
14488        consumer_identity: Option<ConsumerIdentity>,
14489    ) -> (
14490        mpsc::Receiver<crate::router::OutboundFrame>,
14491        ModuleControlRequest,
14492    ) {
14493        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14494        let route_handler = handler.clone();
14495        let target_module_id = target_module_id.to_string();
14496        let route_task = tokio::spawn(async move {
14497            route_handler
14498                .handle_control_frame(
14499                    &client_ctx,
14500                    route_open_frame_with_admission_facts(
14501                        corr,
14502                        &target_module_id,
14503                        unique_project_root("admission-facts"),
14504                        consumer_identity,
14505                        None,
14506                    ),
14507                )
14508                .await
14509                .expect("capability test route.open succeeds")
14510        });
14511        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14512            .await
14513            .expect("capability test route.open must reach route.bind")
14514            .expect("target control receiver stays open");
14515        let bind_request: ModuleControlRequest =
14516            serde_json::from_slice(&bind.body).expect("route.bind decodes");
14517        handler
14518            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14519            .await
14520            .expect("capability test route.bind ACK succeeds");
14521        assert!(route_task.await.expect("route.open task joins").is_empty());
14522        let opened = client_rx
14523            .recv()
14524            .await
14525            .expect("successful route.open publishes a response");
14526        assert!(matches!(
14527            serde_json::from_slice::<ClientControlResponse>(&opened.body),
14528            Ok(ClientControlResponse::RouteOpen { .. })
14529        ));
14530        (client_rx, bind_request)
14531    }
14532
14533    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
14534        assert_eq!(frame.header.ty, FrameType::Push);
14535        assert_eq!(frame.header.channel, 0);
14536        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14537            .expect("route.closed control push decodes");
14538        let ClientControlPush::RouteClosed { channels, .. } = &push else {
14539            panic!("expected route.closed");
14540        };
14541        assert_eq!(channels.len(), 1, "exactly one violating route closed");
14542        let channels = channels.clone();
14543        assert_eq!(
14544            push,
14545            ClientControlPush::RouteClosed {
14546                module_id: target_module_id.to_string(),
14547                channels,
14548                reason: RouteCloseReason::CapabilityDenied,
14549                drained: false,
14550                abandoned: 0,
14551                excluded_subscriptions: 0,
14552                terminal: Some(false),
14553            }
14554        );
14555    }
14556
14557    #[tokio::test]
14558    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14559        let registry = Arc::new(Registry::default());
14560        let forwarding = Arc::new(ForwardingTable::default());
14561        let supervisor = SupervisorHandle::new();
14562        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14563        let handler =
14564            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14565                .with_supervisor(supervisor);
14566        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14567        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14568        register_capability_manifest(
14569            &handler,
14570            &target_ctx,
14571            &mut target_rx,
14572            capability_manifest("target", &["credentials-provider/v1"], &[]),
14573            1,
14574        )
14575        .await;
14576        register_capability_manifest(
14577            &handler,
14578            &opener_ctx,
14579            &mut opener_rx,
14580            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14581            2,
14582        )
14583        .await;
14584
14585        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
14586        let replies = handler
14587            .handle_control_frame(
14588                &client_ctx,
14589                route_open_frame_with_admission_facts(
14590                    3,
14591                    "target",
14592                    unique_project_root("admission-facts"),
14593                    Some(ConsumerIdentity {
14594                        module_id: "opener".to_string(),
14595                        launch_nonce: "opener-nonce".to_string(),
14596                    }),
14597                    None,
14598                ),
14599            )
14600            .await
14601            .expect("denied route.open returns a typed frame");
14602        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14603        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14604        assert!(
14605            target_rx.try_recv().is_err(),
14606            "forbidden route.open must not relay route.bind"
14607        );
14608    }
14609
14610    #[tokio::test]
14611    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
14612        let registry = Arc::new(Registry::default());
14613        let forwarding = Arc::new(ForwardingTable::default());
14614        let supervisor = SupervisorHandle::new();
14615        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14616        let handler =
14617            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14618                .with_supervisor(supervisor);
14619        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
14620        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
14621        register_capability_manifest(
14622            &handler,
14623            &target_ctx,
14624            &mut target_rx,
14625            capability_manifest("target", &["credentials-provider/v1"], &[]),
14626            1,
14627        )
14628        .await;
14629        register_capability_manifest(
14630            &handler,
14631            &old_opener_ctx,
14632            &mut old_opener_rx,
14633            capability_manifest("opener", &[], &[]),
14634            2,
14635        )
14636        .await;
14637        let (mut client_rx, _) = open_route_for_capability_test(
14638            &handler,
14639            &target_ctx,
14640            &mut target_rx,
14641            712,
14642            3,
14643            "target",
14644            Some(ConsumerIdentity {
14645                module_id: "opener".to_string(),
14646                launch_nonce: "opener-nonce".to_string(),
14647            }),
14648        )
14649        .await;
14650        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14651
14652        handler
14653            .cleanup_connection(old_opener_ctx.connection_id)
14654            .expect("old opener registration cleans up");
14655        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
14656        register_capability_manifest(
14657            &handler,
14658            &new_opener_ctx,
14659            &mut new_opener_rx,
14660            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14661            4,
14662        )
14663        .await;
14664
14665        assert_capability_denied_push(
14666            client_rx
14667                .try_recv()
14668                .expect("HELLO deny addition must emit route.closed")
14669                .frame,
14670            "target",
14671        );
14672        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14673        assert!(matches!(
14674            target_rx.try_recv(),
14675            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14676        ));
14677    }
14678
14679    #[tokio::test]
14680    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
14681        let registry = Arc::new(Registry::default());
14682        let forwarding = Arc::new(ForwardingTable::default());
14683        let supervisor = SupervisorHandle::new();
14684        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14685        let handler =
14686            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14687                .with_supervisor(supervisor);
14688        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
14689        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
14690        register_capability_manifest(
14691            &handler,
14692            &target_ctx,
14693            &mut target_rx,
14694            capability_manifest("target", &[], &[]),
14695            1,
14696        )
14697        .await;
14698        register_capability_manifest(
14699            &handler,
14700            &opener_ctx,
14701            &mut opener_rx,
14702            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14703            2,
14704        )
14705        .await;
14706        let (mut client_rx, _) = open_route_for_capability_test(
14707            &handler,
14708            &target_ctx,
14709            &mut target_rx,
14710            722,
14711            3,
14712            "target",
14713            Some(ConsumerIdentity {
14714                module_id: "opener".to_string(),
14715                launch_nonce: "opener-nonce".to_string(),
14716            }),
14717        )
14718        .await;
14719        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14720
14721        let replies = handler
14722            .handle_control_frame(
14723                &target_ctx,
14724                catalog_update_with_capabilities_frame(
14725                    4,
14726                    CapabilityDeclarations {
14727                        provides: vec!["credentials-provider/v1".to_string()],
14728                        requires: Vec::new(),
14729                        must_never_reach: Vec::new(),
14730                    },
14731                ),
14732            )
14733            .await
14734            .expect("claim catalog.update succeeds");
14735        assert!(matches!(
14736            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
14737            Ok(ModuleControlResponseToModule::CatalogUpdate {})
14738        ));
14739        assert_capability_denied_push(
14740            client_rx
14741                .try_recv()
14742                .expect("claim addition must emit route.closed")
14743                .frame,
14744            "target",
14745        );
14746        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14747        assert!(matches!(
14748            target_rx.try_recv(),
14749            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14750        ));
14751    }
14752
14753    #[tokio::test]
14754    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
14755        let registry = Arc::new(Registry::default());
14756        let forwarding = Arc::new(ForwardingTable::default());
14757        let supervisor = SupervisorHandle::new();
14758        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14759        let handler =
14760            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14761                .with_supervisor(supervisor);
14762        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
14763        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
14764        register_capability_manifest(
14765            &handler,
14766            &target_ctx,
14767            &mut target_rx,
14768            capability_manifest("target", &["credentials-provider/v1"], &[]),
14769            1,
14770        )
14771        .await;
14772        register_capability_manifest(
14773            &handler,
14774            &opener_ctx,
14775            &mut opener_rx,
14776            capability_manifest("opener", &[], &[]),
14777            2,
14778        )
14779        .await;
14780        let (mut client_rx, _) = open_route_for_capability_test(
14781            &handler,
14782            &target_ctx,
14783            &mut target_rx,
14784            732,
14785            3,
14786            "target",
14787            Some(ConsumerIdentity {
14788                module_id: "opener".to_string(),
14789                launch_nonce: "opener-nonce".to_string(),
14790            }),
14791        )
14792        .await;
14793        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14794
14795        handler
14796            .handle_control_frame(
14797                &target_ctx,
14798                catalog_update_with_capabilities_frame(
14799                    4,
14800                    CapabilityDeclarations {
14801                        provides: Vec::new(),
14802                        requires: Vec::new(),
14803                        must_never_reach: Vec::new(),
14804                    },
14805                ),
14806            )
14807            .await
14808            .expect("claim removal catalog.update succeeds");
14809        assert_eq!(
14810            forwarding.active_binding_count().unwrap(),
14811            1,
14812            "removing an attested target claim must leave the route census unchanged"
14813        );
14814        assert!(
14815            client_rx.try_recv().is_err(),
14816            "claim removal must not emit route.closed capability_denied"
14817        );
14818        assert!(
14819            target_rx.try_recv().is_err(),
14820            "claim removal must not send the target a route GOODBYE"
14821        );
14822    }
14823
14824    /// A direct client may open a route to a denied capability provider; this
14825    /// policy applies only to attested supervised module origins, not to direct clients.
14826    #[tokio::test]
14827    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
14828        let registry = Arc::new(Registry::default());
14829        let forwarding = Arc::new(ForwardingTable::default());
14830        let supervisor = SupervisorHandle::new();
14831        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14832        let handler =
14833            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14834                .with_supervisor(supervisor);
14835        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
14836        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
14837        register_capability_manifest(
14838            &handler,
14839            &target_ctx,
14840            &mut target_rx,
14841            capability_manifest("target", &["credentials-provider/v1"], &[]),
14842            1,
14843        )
14844        .await;
14845        register_capability_manifest(
14846            &handler,
14847            &opener_ctx,
14848            &mut opener_rx,
14849            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14850            2,
14851        )
14852        .await;
14853
14854        let (_client_rx, bind) = open_route_for_capability_test(
14855            &handler,
14856            &target_ctx,
14857            &mut target_rx,
14858            742,
14859            3,
14860            "target",
14861            None,
14862        )
14863        .await;
14864        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14865            panic!("direct scope-honesty route must bind");
14866        };
14867        assert_eq!(principal, Some(Principal::Direct));
14868        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14869    }
14870
14871    /// A module that denies a capability receives no self-route exemption when it
14872    /// also attestedly provides that capability.
14873    #[tokio::test]
14874    async fn must_never_reach_self_route_is_capability_forbidden() {
14875        let registry = Arc::new(Registry::default());
14876        let forwarding = Arc::new(ForwardingTable::default());
14877        let supervisor = SupervisorHandle::new();
14878        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
14879        let handler =
14880            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14881                .with_supervisor(supervisor);
14882        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
14883        register_capability_manifest(
14884            &handler,
14885            &self_ctx,
14886            &mut self_rx,
14887            capability_manifest(
14888                "self-provider",
14889                &["credentials-provider/v1"],
14890                &["credentials-provider/v1"],
14891            ),
14892            1,
14893        )
14894        .await;
14895
14896        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
14897        let replies = handler
14898            .handle_control_frame(
14899                &client_ctx,
14900                route_open_frame_with_admission_facts(
14901                    2,
14902                    "self-provider",
14903                    unique_project_root("admission-facts"),
14904                    Some(ConsumerIdentity {
14905                        module_id: "self-provider".to_string(),
14906                        launch_nonce: "self-nonce".to_string(),
14907                    }),
14908                    None,
14909                ),
14910            )
14911            .await
14912            .expect("self-route refusal returns a typed frame");
14913        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14914        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14915        assert!(
14916            self_rx.try_recv().is_err(),
14917            "self denial must not relay route.bind"
14918        );
14919    }
14920
14921    #[test]
14922    fn unsupported_channel_zero_frame_returns_error() {
14923        let handler = ControlHandler::default();
14924        let request = Frame::build(
14925            FrameType::Request,
14926            control_flags(),
14927            0,
14928            0,
14929            21,
14930            b"opaque".to_vec(),
14931        )
14932        .unwrap();
14933
14934        let response = handler
14935            .handle_control(ConnectionId::new(1), request)
14936            .unwrap();
14937
14938        assert_eq!(response[0].header.ty, FrameType::Error);
14939        assert_eq!(
14940            parse_error(&response[0])["code"],
14941            "unsupported_control_frame"
14942        );
14943    }
14944
14945    /// Blue/green swap at the control-plane boundary. The supervisor that opens
14946    /// a swap is not wired yet, so the candidate is registered here directly
14947    /// into the registry and forwarding candidate slots, the way the swap's
14948    /// HELLO admission will.
14949    mod swap {
14950        use super::*;
14951
14952        const INCUMBENT: ConnectionId = ConnectionId::new(30);
14953        const CANDIDATE: ConnectionId = ConnectionId::new(40);
14954
14955        struct Swap {
14956            registry: Arc<Registry>,
14957            forwarding: Arc<ForwardingTable>,
14958            handler: ControlHandler,
14959            incumbent_ctx: RouteCtx,
14960            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14961            candidate_ctx: RouteCtx,
14962            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14963        }
14964
14965        async fn swap_with_incumbent() -> Swap {
14966            let registry = Arc::new(Registry::default());
14967            let forwarding = Arc::new(ForwardingTable::default());
14968            let handler =
14969                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14970            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14971            hello_via_sink(
14972                &handler,
14973                &incumbent_ctx,
14974                &mut incumbent_rx,
14975                hello_frame("aft", PROTOCOL_VERSION, 7),
14976            )
14977            .await;
14978            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14979            Swap {
14980                registry,
14981                forwarding,
14982                handler,
14983                incumbent_ctx,
14984                incumbent_rx,
14985                candidate_ctx,
14986                candidate_rx,
14987            }
14988        }
14989
14990        fn register_candidate(swap: &Swap, ready: Option<bool>) {
14991            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14992            candidate_manifest.ready = ready;
14993            let registration = swap
14994                .registry
14995                .register_candidate_with_control_ops(
14996                    candidate_manifest,
14997                    PROTOCOL_VERSION,
14998                    CANDIDATE,
14999                    module_baseline_control_ops(),
15000                )
15001                .unwrap();
15002            swap.forwarding
15003                .register_candidate_module_connection(
15004                    CANDIDATE,
15005                    "aft".to_string(),
15006                    PROTOCOL_VERSION,
15007                    manifest_concurrency(&registration.manifest),
15008                    swap.candidate_ctx.egress.clone(),
15009                )
15010                .unwrap();
15011        }
15012
15013        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15014            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15015            swap.registry.promote_candidate("aft").unwrap().unwrap();
15016            cutover.incumbent.unwrap()
15017        }
15018
15019        /// A swap candidate's HELLO reserves its candidate registry entry
15020        /// before installing its candidate forwarding endpoint. The swap
15021        /// supervisor promotes a ready candidate as soon as it reads it as
15022        /// registered, and promotion cuts forwarding over to the candidate
15023        /// endpoint, so the lookup the supervisor reads
15024        /// (`Registry::published_registration`) must not return the candidate
15025        /// until that endpoint exists. Otherwise the cutover finds no candidate
15026        /// endpoint and the swap fails as if the candidate had disconnected.
15027        #[tokio::test]
15028        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15029            let mut swap = swap_with_incumbent().await;
15030            let hold = Arc::new(std::sync::Barrier::new(2));
15031            let handler = swap
15032                .handler
15033                .clone()
15034                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15035            handler
15036                .supervisor
15037                .open_swap("aft", "candidate-nonce".to_string());
15038            let hello = {
15039                let handler = handler.clone();
15040                let candidate_ctx = swap.candidate_ctx.clone();
15041                std::thread::spawn(move || {
15042                    handler
15043                        .handle_hello(
15044                            candidate_ctx.connection_id,
15045                            Some(candidate_ctx.egress.clone()),
15046                            hello_frame_with_nonce(
15047                                "aft",
15048                                PROTOCOL_VERSION,
15049                                8,
15050                                Some("candidate-nonce"),
15051                            ),
15052                        )
15053                        .unwrap()
15054                })
15055            };
15056
15057            hold.wait();
15058            // Parked: the candidate slot is claimed but not published, and
15059            // forwarding has no candidate endpoint.
15060            assert!(swap
15061                .registry
15062                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15063                .unwrap()
15064                .is_some());
15065            assert!(swap
15066                .registry
15067                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15068                .unwrap()
15069                .is_none());
15070            assert!(swap
15071                .forwarding
15072                .module_endpoint_for_connection(CANDIDATE)
15073                .unwrap()
15074                .is_none());
15075
15076            hold.wait();
15077            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15078            assert_eq!(
15079                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15080                FrameType::HelloAck
15081            );
15082            let published = swap
15083                .registry
15084                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15085                .unwrap()
15086                .expect("candidate is published once its HELLO completes");
15087            assert_eq!(published.connection_id, CANDIDATE);
15088            assert!(swap
15089                .forwarding
15090                .module_endpoint_for_connection(CANDIDATE)
15091                .unwrap()
15092                .is_some());
15093        }
15094
15095        fn keyed_total(counters: &Value, key: &str) -> u64 {
15096            counters[key]
15097                .as_object()
15098                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15099                .unwrap_or(0)
15100        }
15101
15102        #[tokio::test]
15103        async fn replacement_logs_old_end_and_new_admission_once() {
15104            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15105            let mut swap = swap_with_incumbent().await;
15106            swap.handler
15107                .supervisor
15108                .open_swap("aft", "candidate-nonce".to_string());
15109            hello_via_sink(
15110                &swap.handler,
15111                &swap.candidate_ctx,
15112                &mut swap.candidate_rx,
15113                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15114            )
15115            .await;
15116            cutover(&swap);
15117            swap.handler.cleanup_connection(INCUMBENT).unwrap();
15118
15119            let captured = crate::router::test_log::captured_logs(&logs);
15120            println!("captured replacement registry logs:\n{captured}");
15121            let new_admissions: Vec<_> = captured
15122                .lines()
15123                .filter(|line| {
15124                    line.contains("connection_id=40")
15125                        && line.contains("swap candidate registered; not routable until cutover")
15126                })
15127                .collect();
15128            assert_eq!(
15129                new_admissions.len(),
15130                1,
15131                "unexpected admission logs: {captured}"
15132            );
15133            assert!(new_admissions[0].contains("module_id=aft"));
15134            assert!(new_admissions[0].contains("connection_id=40"));
15135            assert!(
15136                new_admissions[0].contains("swap candidate registered; not routable until cutover")
15137            );
15138            assert!(new_admissions[0].contains("ready=true"));
15139
15140            let old_admissions: Vec<_> = captured
15141                .lines()
15142                .filter(|line| {
15143                    line.contains("module registered module_id=aft ")
15144                        && line.contains("connection_id=30")
15145                })
15146                .collect();
15147            assert_eq!(
15148                old_admissions.len(),
15149                1,
15150                "unexpected incumbent admission: {captured}"
15151            );
15152
15153            let old_ends: Vec<_> = captured
15154                .lines()
15155                .filter(|line| {
15156                    line.contains("connection_id=30") && line.contains("module registration ended")
15157                })
15158                .collect();
15159            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15160            assert!(old_ends[0].contains("module_id=aft"));
15161            assert!(old_ends[0].contains("connection_id=30"));
15162            assert!(old_ends[0].contains("reason=replaced"));
15163            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15164            assert_eq!(
15165                captured
15166                    .lines()
15167                    .filter(|line| line.contains("module registration promoted"))
15168                    .count(),
15169                1,
15170                "unexpected promotion logs: {captured}"
15171            );
15172        }
15173
15174        /// An ack from the incumbent for a bind it was sent before cutover,
15175        /// arriving before the incumbent is drained. The incumbent is the live
15176        /// connection carrying every other client's routes, so the ack must
15177        /// not end it: the waiting client is told to retry, the reservation is
15178        /// given back, and the incumbent is told to drop just that binding.
15179        #[tokio::test]
15180        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15181            let mut swap = swap_with_incumbent().await;
15182            let handler = swap.handler.clone();
15183
15184            // A co-tenant route, bound on the incumbent before the swap.
15185            let cotenant = ConnectionId::new(31);
15186            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15187            let (cotenant_task, cotenant_bind) = relay_route_open(
15188                &handler,
15189                cotenant,
15190                &cotenant_ctx.egress,
15191                &mut swap.incumbent_rx,
15192                100,
15193                "aft",
15194                "swap-cotenant",
15195            )
15196            .await;
15197            handler
15198                .handle_control_frame(
15199                    &swap.incumbent_ctx,
15200                    route_bind_ack(cotenant_bind.header.corr),
15201                )
15202                .await
15203                .unwrap();
15204            assert!(cotenant_task.await.unwrap().is_empty());
15205            let (cotenant_channel, cotenant_epoch) =
15206                published_route(&cotenant_rx.recv().await.unwrap());
15207
15208            // A second route.open, relayed to the incumbent and not yet acked.
15209            let caller = ConnectionId::new(32);
15210            let (caller_ctx, mut caller_rx) = route_ctx(caller);
15211            let (caller_task, caller_bind) = relay_route_open(
15212                &handler,
15213                caller,
15214                &caller_ctx.egress,
15215                &mut swap.incumbent_rx,
15216                101,
15217                "aft",
15218                "swap-caller",
15219            )
15220            .await;
15221            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15222
15223            register_candidate(&swap, None);
15224            cutover(&swap);
15225
15226            // The incumbent acks after promotion and before any drain.
15227            let ack = handler
15228                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15229                .await;
15230            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15231            if module_loop_error.is_some() {
15232                // What the connection loop does with an untranslated router
15233                // error: end the connection, releasing every route on it.
15234                handler.cleanup_connection(INCUMBENT).unwrap();
15235            }
15236
15237            // 1. The incumbent's other routes survive.
15238            assert!(
15239                cotenant_rx.try_recv().is_err(),
15240                "the co-tenant route on the incumbent was torn down by one late ack: \
15241                 {module_loop_error:?}"
15242            );
15243            assert!(matches!(
15244                swap.forwarding
15245                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15246                    .unwrap(),
15247                DataRoute::Client(DataRouteState::Bound(_))
15248            ));
15249            assert_eq!(module_loop_error, None);
15250            assert!(swap
15251                .registry
15252                .get_module_by_connection(INCUMBENT)
15253                .unwrap()
15254                .is_some());
15255
15256            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
15257            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15258                .await
15259                .expect("the incumbent is told to drop the abandoned binding")
15260                .unwrap()
15261                .frame;
15262            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15263            assert_eq!(goodbye.header.channel, abandoned_channel);
15264            assert_eq!(goodbye.header.epoch, abandoned_epoch);
15265            assert!(swap.incumbent_rx.try_recv().is_err());
15266
15267            // 3. The waiting client gets a retryable refusal and no route.
15268            let response = caller_task.await.unwrap();
15269            assert_eq!(response.len(), 1);
15270            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15271            assert!(caller_rx.try_recv().is_err());
15272
15273            // 4. The reservation pair is given back, and the pending bind
15274            //    settled exactly once: one accepted open (the co-tenant) and one
15275            //    refused open (the caller), nothing counted twice.
15276            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15277            let counters = handler.counters().snapshot();
15278            assert_eq!(
15279                keyed_total(&counters, "route_open_accepted_by_principal"),
15280                1
15281            );
15282            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15283            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15284        }
15285
15286        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
15287        /// id would resolve to the promoted candidate and every new route.open
15288        /// would be refused as reloading, leaving neither process routable.
15289        #[tokio::test]
15290        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15291            let mut swap = swap_with_incumbent().await;
15292            register_candidate(&swap, None);
15293            let incumbent = cutover(&swap);
15294            swap.forwarding
15295                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15296                .unwrap()
15297                .expect("the incumbent is still registered");
15298
15299            let client = ConnectionId::new(33);
15300            let (client_ctx, mut client_rx) = route_ctx(client);
15301            let route_handler = swap.handler.clone();
15302            let open_ctx = RouteCtx {
15303                connection_id: client,
15304                egress: client_ctx.egress.clone(),
15305            };
15306            let mut route_task = tokio::spawn(async move {
15307                route_handler
15308                    .handle_control_frame(
15309                        &open_ctx,
15310                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
15311                    )
15312                    .await
15313                    .unwrap()
15314            });
15315            let bind = tokio::select! {
15316                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
15317                response = &mut route_task => {
15318                    let response = response.unwrap();
15319                    panic!(
15320                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
15321                        parse_error(&response[0])["code"]
15322                    );
15323                }
15324            };
15325            swap.handler
15326                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
15327                .await
15328                .unwrap();
15329            assert!(route_task.await.unwrap().is_empty());
15330            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
15331            match swap
15332                .forwarding
15333                .lookup_data_route(client, channel, epoch)
15334                .unwrap()
15335            {
15336                DataRoute::Client(DataRouteState::Bound(route)) => {
15337                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
15338                }
15339                other => panic!("expected a bound route on the candidate, got {other:?}"),
15340            }
15341            assert!(swap.incumbent_rx.try_recv().is_err());
15342        }
15343
15344        /// A candidate declares itself ready with `catalog.update` on its own
15345        /// connection. If the connection-keyed registry lookups searched only the
15346        /// active slot, this would answer `not_registered` and the candidate
15347        /// would never become ready.
15348        #[tokio::test]
15349        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
15350            let swap = swap_with_incumbent().await;
15351            register_candidate(&swap, Some(false));
15352            let update = Frame::build(
15353                FrameType::Request,
15354                control_flags(),
15355                0,
15356                0,
15357                55,
15358                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15359                    provides: manifest("aft", PROTOCOL_VERSION).provides,
15360                    capabilities: None,
15361                    ready: Some(true),
15362                })
15363                .unwrap(),
15364            )
15365            .unwrap();
15366
15367            let replies = swap
15368                .handler
15369                .handle_control_frame(&swap.candidate_ctx, update)
15370                .await
15371                .unwrap();
15372
15373            assert_eq!(replies.len(), 1);
15374            assert_eq!(
15375                replies[0].header.ty,
15376                FrameType::Response,
15377                "candidate catalog.update was refused: {:?}",
15378                serde_json::from_slice::<Value>(&replies[0].body).ok()
15379            );
15380            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
15381            assert_eq!(
15382                swap.registry
15383                    .get_module("aft")
15384                    .unwrap()
15385                    .unwrap()
15386                    .connection_id,
15387                INCUMBENT
15388            );
15389        }
15390    }
15391
15392    /// The HELLO gate while the supervisor has a swap open: only the nonce it
15393    /// minted for the candidate admits a second process, into the candidate
15394    /// slot, and that check runs ahead of the reserved-module gate.
15395    mod swap_admission {
15396        use super::*;
15397
15398        const INCUMBENT_NONCE: &str = "incumbent-nonce";
15399        const CANDIDATE_NONCE: &str = "candidate-nonce";
15400
15401        fn handler_with_incumbent(
15402            module_id: &str,
15403            reserved: bool,
15404        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
15405            let registry = Arc::new(Registry::default());
15406            let supervisor = SupervisorHandle::new();
15407            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
15408            if reserved {
15409                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
15410            }
15411            let handler =
15412                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
15413            let incumbent = handler
15414                .handle_control(
15415                    ConnectionId::new(1),
15416                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
15417                )
15418                .unwrap();
15419            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
15420            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
15421            (registry, supervisor, handler)
15422        }
15423
15424        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
15425        /// admits every nonce, so while a swap is open the swap gate is the only
15426        /// thing between a key-holder and the candidate slot. A nonce the
15427        /// supervisor did not mint, or none at all, is refused, and neither the
15428        /// incumbent's registration nor the candidate slot moves.
15429        #[test]
15430        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15431            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15432
15433            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15434                let replies = handler
15435                    .handle_control(
15436                        ConnectionId::new(connection),
15437                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15438                    )
15439                    .unwrap();
15440                assert_eq!(replies[0].header.ty, FrameType::Error);
15441                assert_eq!(
15442                    parse_error(&replies[0])["code"],
15443                    "swap_token_invalid",
15444                    "nonce {nonce:?}"
15445                );
15446            }
15447            assert!(registry.get_candidate("aft").unwrap().is_none());
15448            assert_eq!(
15449                registry.get_module("aft").unwrap().unwrap().connection_id,
15450                ConnectionId::new(1)
15451            );
15452
15453            // Control: the minted token is admitted, into the candidate slot,
15454            // and only once.
15455            let admitted = handler
15456                .handle_control(
15457                    ConnectionId::new(4),
15458                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15459                )
15460                .unwrap();
15461            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15462            assert_eq!(
15463                registry
15464                    .get_candidate("aft")
15465                    .unwrap()
15466                    .unwrap()
15467                    .connection_id,
15468                ConnectionId::new(4)
15469            );
15470            assert_eq!(
15471                registry.get_module("aft").unwrap().unwrap().connection_id,
15472                ConnectionId::new(1),
15473                "the candidate must not take the active slot"
15474            );
15475            let replayed = handler
15476                .handle_control(
15477                    ConnectionId::new(5),
15478                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15479                )
15480                .unwrap();
15481            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15482
15483            // The case only this gate covers: the incumbent has died mid-swap,
15484            // so its duplicate refusal is gone too, and without the gate a
15485            // key-holder would take the id's ACTIVE slot.
15486            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15487            let squatter = handler
15488                .handle_control(
15489                    ConnectionId::new(6),
15490                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15491                )
15492                .unwrap();
15493            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15494            assert!(
15495                registry.get_module("aft").unwrap().is_none(),
15496                "a squatter took the active slot of an id being swapped"
15497            );
15498        }
15499
15500        /// Design mutation arm (iii). A reserved module's candidate presents a
15501        /// nonce the reserved gate has never seen (that gate holds the
15502        /// incumbent's), so the swap gate must run first or the candidate is
15503        /// refused `reserved_module` and a reserved module can never be swapped.
15504        #[test]
15505        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15506            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15507
15508            let replies = handler
15509                .handle_control(
15510                    ConnectionId::new(2),
15511                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15512                )
15513                .unwrap();
15514
15515            assert_eq!(
15516                replies[0].header.ty,
15517                FrameType::HelloAck,
15518                "reserved candidate refused: {:?}",
15519                serde_json::from_slice::<Value>(&replies[0].body).ok()
15520            );
15521            assert_eq!(
15522                registry
15523                    .get_candidate("vault")
15524                    .unwrap()
15525                    .unwrap()
15526                    .connection_id,
15527                ConnectionId::new(2)
15528            );
15529        }
15530
15531        /// With no swap open the gate is inert: the incumbent's reserved gate
15532        /// and duplicate refusal behave exactly as before.
15533        #[test]
15534        fn without_an_open_swap_the_ordinary_gates_decide() {
15535            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
15536            supervisor.close_swap("vault");
15537
15538            let candidate = handler
15539                .handle_control(
15540                    ConnectionId::new(2),
15541                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15542                )
15543                .unwrap();
15544            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
15545            let duplicate = handler
15546                .handle_control(
15547                    ConnectionId::new(3),
15548                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15549                )
15550                .unwrap();
15551            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15552            assert!(registry.get_candidate("vault").unwrap().is_none());
15553        }
15554    }
15555
15556    /// `scope.sync` and `scope.describe` through the real control handler: who
15557    /// may sync is decided by the registration and launch nonce of the module
15558    /// connection, never by the request body.
15559    mod scopes {
15560        mod idle_tests {
15561            use super::*;
15562            include!("scope_idle_tests.rs");
15563        }
15564        mod route_open_timings {
15565            use super::*;
15566            include!("route_open_timing_tests.rs");
15567        }
15568        mod activity {
15569            include!("activity_scope_tests.rs");
15570        }
15571        mod delta {
15572            include!("scope_handler_delta_tests.rs");
15573        }
15574        mod call_modes {
15575            include!("scope_call_mode_tests.rs");
15576        }
15577        #[tokio::test]
15578        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
15579            let rig = rig().await;
15580            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
15581                "s".into(),
15582                1,
15583                "tool".into(),
15584            ))
15585            .unwrap();
15586            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
15587            let (direct, _rx) = wide_ctx(999);
15588            for (ctx, expected) in [
15589                (&rig.owner, FrameType::Response),
15590                (&direct, FrameType::Error),
15591            ] {
15592                let frame =
15593                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
15594                        .unwrap();
15595                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
15596                assert_eq!(replies[0].header.ty, expected);
15597                if expected == FrameType::Response {
15598                    let reply: subc_control::ScopeServedByReply =
15599                        serde_json::from_slice(&replies[0].body).unwrap();
15600                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
15601                } else {
15602                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
15603                }
15604            }
15605            let reply = call(
15606                &rig.handler,
15607                &rig.owner,
15608                &ModuleControlRequestFromModule::ScopeDescribe {
15609                    owner: Principal::Reserved {
15610                        module_id: OWNER.into(),
15611                    },
15612                    scope_ref: "s".into(),
15613                },
15614            )
15615            .await;
15616            assert!(matches!(
15617                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
15618                ModuleControlResponseToModule::ScopeDescribe { .. }
15619            ));
15620        }
15621
15622        use subc_protocol::scope::{
15623            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
15624            ScopeStatus,
15625        };
15626
15627        use super::*;
15628
15629        const OWNER: &str = "prefrontal-core";
15630
15631        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
15632            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
15633        }
15634
15635        async fn call(
15636            handler: &ControlHandler,
15637            ctx: &RouteCtx,
15638            request: &ModuleControlRequestFromModule,
15639        ) -> Frame {
15640            let body = serde_json::to_vec(request).unwrap();
15641            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
15642            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
15643            assert_eq!(replies.len(), 1, "{replies:?}");
15644            replies.pop().unwrap()
15645        }
15646
15647        async fn sync(
15648            handler: &ControlHandler,
15649            ctx: &RouteCtx,
15650            generation: u64,
15651            scopes: Vec<ScopeRecord>,
15652        ) -> Result<ModuleControlResponseToModule, String> {
15653            let reply = call(
15654                handler,
15655                ctx,
15656                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
15657            )
15658            .await;
15659            match reply.header.ty {
15660                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
15661                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
15662            }
15663        }
15664
15665        async fn describe(
15666            handler: &ControlHandler,
15667            ctx: &RouteCtx,
15668            owner: &str,
15669            scope_ref: &str,
15670        ) -> ModuleControlResponseToModule {
15671            let reply = call(
15672                handler,
15673                ctx,
15674                &ModuleControlRequestFromModule::ScopeDescribe {
15675                    owner: Principal::Reserved {
15676                        module_id: owner.to_string(),
15677                    },
15678                    scope_ref: scope_ref.to_string(),
15679                },
15680            )
15681            .await;
15682            assert_eq!(
15683                reply.header.ty,
15684                FrameType::Response,
15685                "{:?}",
15686                parse_error(&reply)
15687            );
15688            serde_json::from_slice(&reply.body).unwrap()
15689        }
15690
15691        #[tokio::test]
15692        async fn scope_describe_never_echoes_bind_proof_or_hash() {
15693            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15694            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15695            let mut rig = rig().await;
15696            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15697                .await;
15698            let _bound = rig
15699                .bound(
15700                    Some(AFT),
15701                    PLEXUS,
15702                    Some(
15703                        rig.selector("s", Some(1))
15704                            .with_bind_proof(Some(PROOF.into())),
15705                    ),
15706                )
15707                .await;
15708            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
15709            let output = serde_json::to_string(&reply).unwrap();
15710            assert!(output.contains("\"status\":\"live\""));
15711            assert!(!output.contains(PROOF));
15712            assert!(!output.contains(HASH));
15713        }
15714
15715        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
15716        async fn module(
15717            handler: &ControlHandler,
15718            connection: u64,
15719            module_id: &str,
15720            nonce: Option<&str>,
15721        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15722            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
15723            hello_via_sink(
15724                handler,
15725                &ctx,
15726                &mut rx,
15727                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
15728            )
15729            .await;
15730            (ctx, rx)
15731        }
15732
15733        /// `direct` and every other client connection has no registration, so
15734        /// it can neither sync nor own a scope.
15735        #[tokio::test]
15736        async fn a_client_connection_cannot_sync_or_describe() {
15737            let handler = ControlHandler::new(Arc::new(Registry::default()));
15738            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
15739            for request in [
15740                ModuleControlRequestFromModule::ScopeSync {
15741                    generation: 1,
15742                    scopes: vec![head("s", 1)],
15743                },
15744                ModuleControlRequestFromModule::ScopeDescribe {
15745                    owner: Principal::Direct,
15746                    scope_ref: "s".to_string(),
15747                },
15748            ] {
15749                let reply = call(&handler, &ctx, &request).await;
15750                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
15751            }
15752            assert!(
15753                !handler
15754                    .scopes
15755                    .read()
15756                    .unwrap()
15757                    .describe(
15758                        &Principal::Reserved {
15759                            module_id: OWNER.to_string()
15760                        },
15761                        "s"
15762                    )
15763                    .owner_synced
15764            );
15765        }
15766
15767        /// A module the supervisor did not spawn registers without a launch
15768        /// nonce, so it is never an owner's current launch.
15769        #[tokio::test]
15770        async fn a_module_without_a_supervised_launch_cannot_sync() {
15771            let handler = ControlHandler::new(Arc::new(Registry::default()));
15772            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
15773            assert_eq!(
15774                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
15775                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15776            );
15777        }
15778
15779        #[tokio::test]
15780        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
15781            let supervisor = SupervisorHandle::new();
15782            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15783            let handler = ControlHandler::new(Arc::new(Registry::default()))
15784                .with_supervisor(supervisor.clone());
15785            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15786            sync(&handler, &incumbent, 1, vec![head("s", 1)])
15787                .await
15788                .expect("the current launch syncs");
15789
15790            // A swap candidate registers with the swap token and is refused
15791            // while the incumbent keeps syncing.
15792            supervisor.open_swap(OWNER, "n2".to_string());
15793            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
15794            assert_eq!(
15795                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15796                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15797            );
15798            sync(&handler, &incumbent, 2, vec![head("s", 1)])
15799                .await
15800                .expect("the serving owner syncs during the swap");
15801
15802            // The swap fails and is rolled back. The candidate never held sync
15803            // authority, and still cannot sync.
15804            supervisor.close_swap(OWNER);
15805            assert_eq!(
15806                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15807                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15808            );
15809            sync(&handler, &incumbent, 3, vec![head("s", 1)])
15810                .await
15811                .expect("the serving owner syncs after the rollback");
15812            handler.cleanup_connection(candidate.connection_id).unwrap();
15813
15814            // A swap that cuts over. Promotion records the candidate's nonce as
15815            // the module's spawn nonce, which is what `set_spawn_nonce` does
15816            // here; the promoted connection then takes authority at any
15817            // generation and the superseded incumbent is refused.
15818            supervisor.open_swap(OWNER, "n3".to_string());
15819            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
15820            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
15821            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
15822                .await
15823                .expect("the promoted launch takes authority");
15824            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15825                panic!("unexpected reply {reply:?}");
15826            };
15827            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
15828            assert_eq!(
15829                sync(&handler, &incumbent, 4, Vec::new()).await,
15830                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15831            );
15832        }
15833
15834        /// Authority dies with its connection: the cleanup path releases it,
15835        /// so the owner's next connection takes it at any generation.
15836        #[tokio::test]
15837        async fn closing_the_authority_connection_frees_sync_authority() {
15838            let supervisor = SupervisorHandle::new();
15839            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15840            let handler = ControlHandler::new(Arc::new(Registry::default()))
15841                .with_supervisor(supervisor.clone());
15842            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15843            sync(&handler, &first, 10, vec![head("s", 1)])
15844                .await
15845                .unwrap();
15846            handler.cleanup_connection(first.connection_id).unwrap();
15847
15848            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
15849            sync(&handler, &second, 1, vec![head("s", 1)])
15850                .await
15851                .expect("the next connection takes the released authority");
15852        }
15853
15854        #[tokio::test]
15855        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
15856            let supervisor = SupervisorHandle::new();
15857            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15858            let handler =
15859                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
15860            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
15861            sync(&handler, &first, 10, vec![head("s", 1)])
15862                .await
15863                .unwrap();
15864            handler
15865                .handle_control_frame(
15866                    &first,
15867                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
15868                )
15869                .await
15870                .unwrap();
15871            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
15872            sync(&handler, &second, 1, vec![head("s", 1)])
15873                .await
15874                .expect("GOODBYE releases authority even if the old socket remains open");
15875        }
15876
15877        #[tokio::test]
15878        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
15879            let registry = Arc::new(Registry::default());
15880            let supervisor_handle = SupervisorHandle::new();
15881            let supervisor =
15882                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
15883                    .with_handle(supervisor_handle.clone())
15884                    .with_daemon_incarnation("incarnation-7".to_string());
15885            // Configured with enabled: false, so the supervisor lists the
15886            // module without spawning a process for it.
15887            supervisor
15888                .supervise_configured(
15889                    ModuleSpec {
15890                        module_id: OWNER.to_string(),
15891                        program: PathBuf::from("/nonexistent/prefrontal-core"),
15892                        args: Vec::new(),
15893                        env: Vec::new(),
15894                        reserved: false,
15895                        reserved_prefixes: Vec::new(),
15896                        protocol: ModuleProtocol::Subc,
15897                        overlap: Default::default(),
15898                    },
15899                    false,
15900                )
15901                .unwrap();
15902            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
15903            let handler =
15904                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
15905            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
15906
15907            // Configured but not yet synced: a reader waits for the owner.
15908            let ModuleControlResponseToModule::ScopeDescribe {
15909                status,
15910                daemon_incarnation,
15911                owner_synced,
15912                owner_configured,
15913                scope,
15914                ..
15915            } = describe(&handler, &reader, OWNER, "s").await
15916            else {
15917                panic!("not a describe reply");
15918            };
15919            assert_eq!(status, ScopeStatus::NotLive);
15920            assert_eq!(daemon_incarnation, "incarnation-7");
15921            assert!(!owner_synced);
15922            assert!(owner_configured);
15923            assert!(scope.is_none());
15924
15925            // Not a supervised module: the owner will never sync, and a reader
15926            // refuses rather than waits.
15927            let ModuleControlResponseToModule::ScopeDescribe {
15928                status,
15929                owner_configured,
15930                ..
15931            } = describe(&handler, &reader, "ghost", "s").await
15932            else {
15933                panic!("not a describe reply");
15934            };
15935            assert_eq!(status, ScopeStatus::NotLive);
15936            assert!(!owner_configured);
15937
15938            // Live, with the stamp fields and the computed owner_authorized.
15939            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
15940            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
15941            let ModuleControlResponseToModule::ScopeDescribe {
15942                status,
15943                scope_epoch,
15944                owner_synced,
15945                scope,
15946                ..
15947            } = describe(&handler, &reader, OWNER, "s").await
15948            else {
15949                panic!("not a describe reply");
15950            };
15951            assert_eq!(status, ScopeStatus::Live);
15952            assert_eq!(scope_epoch, Some(4));
15953            assert!(owner_synced);
15954            let stamp = scope.expect("a live scope carries its stamp");
15955            assert!(
15956                stamp.owner_authorized,
15957                "prefrontal-core is the default authority"
15958            );
15959            assert_eq!(stamp.kind, ScopeKind::Head);
15960        }
15961
15962        #[tokio::test]
15963        async fn scope_authority_owners_decides_owner_authorized() {
15964            let supervisor = SupervisorHandle::new();
15965            supervisor.set_spawn_nonce("broca", "b1".to_string());
15966            let handler = ControlHandler::new(Arc::new(Registry::default()))
15967                .with_supervisor(supervisor)
15968                .with_scope_authority_owners(vec!["broca".to_string()]);
15969            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
15970            let mut gated = head("s", 1);
15971            gated.attributes.agent_id = Some("agent".to_string());
15972            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
15973            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
15974                describe(&handler, &broca, "broca", "s").await
15975            else {
15976                panic!("not a describe reply");
15977            };
15978            assert!(scope.unwrap().owner_authorized);
15979        }
15980
15981        /// With route admission, the stamp, the commit re-check and drains in
15982        /// place, the feature is advertised: the module ops in HELLO_ACK, and
15983        /// `scopes/v1` in HELLO_ACK and `server.describe`.
15984        #[tokio::test]
15985        async fn scope_ops_and_the_scopes_capability_are_advertised() {
15986            let handler = ControlHandler::new(Arc::new(Registry::default()));
15987            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
15988            let ack = hello_via_sink(
15989                &handler,
15990                &ctx,
15991                &mut rx,
15992                hello_frame("m", PROTOCOL_VERSION, 1),
15993            )
15994            .await;
15995            let ack = parse_ack(&ack);
15996            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
15997                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
15998            }
15999            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16000
16001            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16002            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16003            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16004            let reply = handler
16005                .handle_control_frame(&client, frame)
16006                .await
16007                .unwrap()
16008                .pop()
16009                .unwrap();
16010            let ClientControlResponse::ServerDescribe { capabilities, .. } =
16011                serde_json::from_slice(&reply.body).unwrap()
16012            else {
16013                panic!("not a server.describe reply");
16014            };
16015            assert!(
16016                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16017                "{capabilities:?}"
16018            );
16019        }
16020
16021        // ---- route admission, stamps, commit re-check and drains ----------
16022
16023        const PLEXUS: &str = "plexus";
16024        const OTHER: &str = "other";
16025        const AFT: &str = "aft";
16026        const BROCA: &str = "broca";
16027        const MAGIC: &str = "magic-context";
16028
16029        fn nonce(module_id: &str) -> String {
16030            format!("nonce-{module_id}")
16031        }
16032
16033        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16034            let (tx, rx) = mpsc::channel(64);
16035            (
16036                RouteCtx {
16037                    connection_id: ConnectionId::new(connection),
16038                    egress: FrameSink::new(tx),
16039                },
16040                rx,
16041            )
16042        }
16043
16044        /// A daemon with a configured owner (prefrontal-core) registered on its
16045        /// own module connection, two routable targets (plexus, other), and
16046        /// launch nonces minted for the modules that open routes as carriers.
16047        struct Rig {
16048            handler: ControlHandler,
16049            forwarding: Arc<ForwardingTable>,
16050            owner: RouteCtx,
16051            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16052            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16053            generation: u64,
16054            next_connection: u64,
16055            _supervisor: Supervisor,
16056        }
16057
16058        async fn rig() -> Rig {
16059            rig_with_flow_support(true).await
16060        }
16061
16062        async fn rig_with_flow_support(flow_support: bool) -> Rig {
16063            rig_with_scope_capabilities(flow_support, false).await
16064        }
16065
16066        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16067            let mut provides = Vec::new();
16068            if flow_support {
16069                provides.push("flow-scopes/v1");
16070            }
16071            if run_support {
16072                provides.push("agent-run-scopes/v1");
16073            }
16074            rig_with_target_capabilities(&provides).await
16075        }
16076
16077        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16078            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16079            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16080            // Decoder versions alone cannot admit scoped calls: only a capability
16081            // declaration promises the required run or flow behaviour.
16082            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16083            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16084            Frame::build(
16085                FrameType::Hello,
16086                control_flags(),
16087                0,
16088                0,
16089                connection,
16090                serde_json::to_vec(&body).unwrap(),
16091            )
16092            .unwrap()
16093        }
16094
16095        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16096            let registry = Arc::new(Registry::default());
16097            let forwarding = Arc::new(ForwardingTable::default());
16098            let supervisor_handle = SupervisorHandle::new();
16099            let supervisor =
16100                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16101                    .with_handle(supervisor_handle.clone());
16102            supervisor
16103                .supervise_configured(
16104                    ModuleSpec {
16105                        module_id: OWNER.to_string(),
16106                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16107                        args: Vec::new(),
16108                        env: Vec::new(),
16109                        reserved: false,
16110                        reserved_prefixes: Vec::new(),
16111                        protocol: ModuleProtocol::Subc,
16112                        overlap: Default::default(),
16113                    },
16114                    false,
16115                )
16116                .unwrap();
16117            for module_id in [OWNER, AFT, BROCA, MAGIC] {
16118                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16119            }
16120            let handler =
16121                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16122                    .with_supervisor(supervisor_handle);
16123            let (owner, mut owner_rx) = wide_ctx(1);
16124            hello_via_sink(
16125                &handler,
16126                &owner,
16127                &mut owner_rx,
16128                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16129            )
16130            .await;
16131            let mut modules = BTreeMap::new();
16132            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16133                let (ctx, mut rx) = wide_ctx(connection);
16134                let hello = hello_with_capabilities(module_id, connection, provides);
16135                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16136                modules.insert(module_id.to_string(), (ctx, rx));
16137            }
16138            Rig {
16139                handler,
16140                forwarding,
16141                owner,
16142                _owner_rx: owner_rx,
16143                modules,
16144                generation: 0,
16145                next_connection: 100,
16146                _supervisor: supervisor,
16147            }
16148        }
16149
16150        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16151            ScopeCarrier::new(Principal::Reserved {
16152                module_id: module_id.to_string(),
16153            })
16154            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16155        }
16156
16157        /// The scope most tests open under: aft carries to any module, broca
16158        /// only to plexus and other, and the owner delegates as agent-1.
16159        fn session(scope_epoch: u64) -> ScopeRecord {
16160            let mut record = head("s", scope_epoch);
16161            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16162            record.attributes.agent_id = Some("agent-1".to_string());
16163            record.attributes.delegates = true;
16164            record
16165        }
16166
16167        impl Rig {
16168            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16169                self.generation += 1;
16170                sync(&self.handler, &self.owner, self.generation, scopes)
16171                    .await
16172                    .expect("the owner's sync is accepted");
16173            }
16174
16175            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16176                ScopeSelector::new(
16177                    Principal::Reserved {
16178                        module_id: OWNER.to_string(),
16179                    },
16180                    scope_ref.to_string(),
16181                    scope_epoch,
16182                )
16183            }
16184
16185            fn open_frame(
16186                &mut self,
16187                opener: Option<&str>,
16188                target: &str,
16189                scope: Option<ScopeSelector>,
16190            ) -> (
16191                RouteCtx,
16192                mpsc::Receiver<crate::router::OutboundFrame>,
16193                Frame,
16194            ) {
16195                self.next_connection += 1;
16196                let (ctx, rx) = wide_ctx(self.next_connection);
16197                let root = unique_project_root("scoped-open");
16198                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16199                    target: RouteTarget::ToolProvider {
16200                        module_id: target.to_string(),
16201                    },
16202                    identity: BindIdentity::new(
16203                        root.path().to_path_buf(),
16204                        "unit".to_string(),
16205                        "session".to_string(),
16206                    ),
16207                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
16208                        module_id: module_id.to_string(),
16209                        launch_nonce: nonce(module_id),
16210                    }),
16211                    consumer_capabilities: None,
16212                    role_versions: None,
16213                    admission_facts: None,
16214                    scope,
16215                })
16216                .unwrap();
16217                let frame = Frame::build(
16218                    FrameType::Request,
16219                    control_flags(),
16220                    0,
16221                    0,
16222                    self.next_connection,
16223                    body,
16224                )
16225                .unwrap();
16226                (ctx, rx, frame)
16227            }
16228
16229            /// Open and expect a refusal before anything is relayed.
16230            async fn refused(
16231                &mut self,
16232                opener: Option<&str>,
16233                target: &str,
16234                scope: Option<ScopeSelector>,
16235            ) -> String {
16236                self.refusal_body(opener, target, scope).await["code"]
16237                    .as_str()
16238                    .unwrap()
16239                    .to_string()
16240            }
16241
16242            async fn refusal_body(
16243                &mut self,
16244                opener: Option<&str>,
16245                target: &str,
16246                scope: Option<ScopeSelector>,
16247            ) -> Value {
16248                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16249                let replies = tokio::time::timeout(
16250                    Duration::from_secs(2),
16251                    self.handler.handle_control_frame(&ctx, frame),
16252                )
16253                .await
16254                .expect("the open must be refused before waiting for a bind ack")
16255                .unwrap();
16256                assert_eq!(replies.len(), 1, "{replies:?}");
16257                assert_eq!(replies[0].header.ty, FrameType::Error);
16258                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16259                assert!(
16260                    module_rx.try_recv().is_err(),
16261                    "a refused open relays nothing"
16262                );
16263                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16264                parse_error(&replies[0])
16265            }
16266
16267            /// Start an open and return its task and the bind the target got.
16268            async fn relayed(
16269                &mut self,
16270                opener: Option<&str>,
16271                target: &str,
16272                scope: Option<ScopeSelector>,
16273            ) -> Relayed {
16274                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16275                let handler = self.handler.clone();
16276                let task_ctx = ctx.clone();
16277                let task = tokio::spawn(async move {
16278                    handler
16279                        .handle_control_frame(&task_ctx, frame)
16280                        .await
16281                        .unwrap()
16282                });
16283                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16284                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16285                    .await
16286                    .expect("the target receives the relayed route.bind")
16287                    .unwrap()
16288                    .frame;
16289                Relayed {
16290                    target: target.to_string(),
16291                    client: ctx,
16292                    client_rx: rx,
16293                    task,
16294                    bind,
16295                }
16296            }
16297
16298            async fn ack(&self, relayed: &Relayed) {
16299                let (module, _) = &self.modules[&relayed.target];
16300                self.handler
16301                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16302                    .await
16303                    .unwrap();
16304            }
16305
16306            /// Open, ack and return the bound route.
16307            async fn bound(
16308                &mut self,
16309                opener: Option<&str>,
16310                target: &str,
16311                scope: Option<ScopeSelector>,
16312            ) -> Bound {
16313                let relayed = self.relayed(opener, target, scope).await;
16314                self.ack(&relayed).await;
16315                let Relayed {
16316                    target,
16317                    client,
16318                    mut client_rx,
16319                    task,
16320                    bind,
16321                } = relayed;
16322                assert!(
16323                    task.await.unwrap().is_empty(),
16324                    "the open is answered by commit"
16325                );
16326                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
16327                Bound {
16328                    target,
16329                    client,
16330                    client_rx,
16331                    channel,
16332                    epoch,
16333                    bind,
16334                }
16335            }
16336
16337            fn live(&self, route: &Bound) -> bool {
16338                matches!(
16339                    self.forwarding
16340                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16341                        .unwrap(),
16342                    DataRoute::Client(DataRouteState::Bound(_))
16343                )
16344            }
16345        }
16346
16347        struct Relayed {
16348            target: String,
16349            client: RouteCtx,
16350            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16351            task: tokio::task::JoinHandle<Vec<Frame>>,
16352            bind: Frame,
16353        }
16354
16355        struct Bound {
16356            target: String,
16357            client: RouteCtx,
16358            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16359            channel: u16,
16360            epoch: u32,
16361            bind: Frame,
16362        }
16363
16364        impl Bound {
16365            /// The reason of the `route.closed` this client was sent, after
16366            /// checking it also got a GOODBYE on exactly this route.
16367            fn closed_reason(&mut self) -> RouteCloseReason {
16368                let mut reason = None;
16369                let mut goodbye = false;
16370                while let Ok(outbound) = self.client_rx.try_recv() {
16371                    let frame = outbound.frame;
16372                    match frame.header.ty {
16373                        FrameType::Goodbye => {
16374                            assert_eq!(
16375                                (frame.header.channel, frame.header.epoch),
16376                                (self.channel, self.epoch)
16377                            );
16378                            goodbye = true;
16379                        }
16380                        FrameType::Push => {
16381                            let ClientControlPush::RouteClosed {
16382                                reason: r,
16383                                module_id,
16384                                ..
16385                            } = serde_json::from_slice(&frame.body).unwrap()
16386                            else {
16387                                panic!("unexpected push");
16388                            };
16389                            assert_eq!(module_id, self.target);
16390                            reason = Some(r);
16391                        }
16392                        other => panic!("unexpected frame {other:?}"),
16393                    }
16394                }
16395                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
16396                reason.expect("the client is told why the route closed")
16397            }
16398
16399            fn untouched(&mut self) -> bool {
16400                self.client_rx.try_recv().is_err()
16401            }
16402
16403            fn stamp(&self) -> Option<ScopeStamp> {
16404                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
16405                    ModuleControlRequest::RouteBind { scope, .. } => scope,
16406                    other => panic!("expected a route.bind, got {other:?}"),
16407                }
16408            }
16409        }
16410
16411        #[tokio::test]
16412        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
16413        ) {
16414            let mut rig = rig().await;
16415            let mut record = session(1);
16416            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
16417            record.child_owners = vec![Principal::Reserved {
16418                module_id: MAGIC.to_string(),
16419            }];
16420            rig.sync(vec![record]).await;
16421            let scope = || Some(rig_selector("s", Some(1)));
16422
16423            // Admitted: the owner, a bare carrier to any module, a targeted
16424            // carrier to its listed module.
16425            rig.bound(Some(OWNER), PLEXUS, scope()).await;
16426            rig.bound(Some(AFT), OTHER, scope()).await;
16427            rig.bound(Some(BROCA), PLEXUS, scope()).await;
16428
16429            // Refused scope_not_carrier: a targeted carrier to an unlisted
16430            // module, a module that is not listed at all (a child owner is not
16431            // a carrier), and a direct key-holder.
16432            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16433                assert_eq!(
16434                    rig.refused(opener, target, scope()).await,
16435                    error_codes::SCOPE_NOT_CARRIER,
16436                    "{opener:?} -> {target}"
16437                );
16438            }
16439        }
16440
16441        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16442            ScopeSelector::new(
16443                Principal::Reserved {
16444                    module_id: OWNER.to_string(),
16445                },
16446                scope_ref.to_string(),
16447                scope_epoch,
16448            )
16449        }
16450
16451        #[tokio::test]
16452        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16453            let mut rig = rig().await;
16454            rig.sync(vec![session(1)]).await;
16455            for opener in [OWNER, AFT] {
16456                assert_eq!(
16457                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16458                        .await,
16459                    error_codes::SCOPE_EPOCH_REQUIRED,
16460                    "{opener}"
16461                );
16462            }
16463        }
16464
16465        #[tokio::test]
16466        async fn admission_separates_not_synced_not_live_and_ended() {
16467            let mut rig = rig().await;
16468            // Before the configured owner's first sync: retryable.
16469            let code = rig
16470                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16471                .await;
16472            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16473            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16474
16475            // An owner that is not configured will never sync: terminal.
16476            let ghost = ScopeSelector::new(
16477                Principal::Reserved {
16478                    module_id: "ghost".to_string(),
16479                },
16480                "s".to_string(),
16481                Some(1),
16482            );
16483            assert_eq!(
16484                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16485                error_codes::SCOPE_NOT_LIVE
16486            );
16487
16488            rig.sync(vec![session(2)]).await;
16489            assert_eq!(
16490                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16491                    .await,
16492                error_codes::SCOPE_NOT_LIVE
16493            );
16494            for epoch in [1, 3] {
16495                assert_eq!(
16496                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16497                        .await,
16498                    error_codes::SCOPE_ENDED,
16499                    "epoch {epoch}"
16500                );
16501            }
16502            // Control: the live epoch is admitted.
16503            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16504                .await;
16505        }
16506
16507        #[tokio::test]
16508        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16509            let mut rig = rig().await;
16510            rig.sync(vec![session(1)]).await;
16511            let route = rig
16512                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16513                .await;
16514            let stamp = route.stamp().expect("a scoped bind carries the stamp");
16515            assert_eq!(stamp.scope_ref, "s");
16516            assert_eq!(stamp.scope_epoch, 1);
16517            assert_eq!(stamp.kind, ScopeKind::Head);
16518            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16519            assert!(stamp.attributes.delegates);
16520            assert!(stamp.owner_authorized);
16521            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16522            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16523
16524            // broca owns a scope of its own on its own module connection; it is
16525            // not in scope_authority_owners, so its stamp is not authorized.
16526            let (broca, mut broca_rx) = wide_ctx(50);
16527            hello_via_sink(
16528                &rig.handler,
16529                &broca,
16530                &mut broca_rx,
16531                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
16532            )
16533            .await;
16534            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
16535                .await
16536                .unwrap();
16537            let own = ScopeSelector::new(
16538                Principal::Reserved {
16539                    module_id: BROCA.to_string(),
16540                },
16541                "b".to_string(),
16542                Some(1),
16543            );
16544            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
16545            assert!(!route.stamp().unwrap().owner_authorized);
16546        }
16547
16548        #[tokio::test]
16549        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16550            let mut rig = rig().await;
16551            let mut record = head("s", 1);
16552            record.carriers = vec![carrier(AFT, None)];
16553            let flow_id = "Flow:run-7/step_2!~";
16554            record.attributes.flow_id = Some(flow_id.to_string());
16555            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16556                .await
16557                .unwrap();
16558            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16559                panic!("not a sync reply");
16560            };
16561            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
16562            let route = rig
16563                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16564                .await;
16565            assert!(rig.live(&route), "the stamped bind committed");
16566            let stamp = route.stamp().expect("a flow scope carries a stamp");
16567            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
16568            assert_eq!(stamp.attributes.agent_id, None);
16569            assert!(!stamp.attributes.delegates);
16570            assert!(stamp.owner_authorized);
16571            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16572            assert_eq!(unscoped.stamp(), None);
16573        }
16574
16575        #[tokio::test]
16576        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
16577            let mut rig = rig_with_flow_support(false).await;
16578            let mut record = session(1);
16579            record.attributes.flow_id = Some("flow:7".to_string());
16580            rig.sync(vec![record]).await;
16581            for opener in [OWNER, AFT] {
16582                let body = rig
16583                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16584                    .await;
16585                assert_eq!(body["code"], "target_flow_unsupported");
16586                let message = body["message"].as_str().unwrap();
16587                for required in [PLEXUS, "flow-scopes/v1"] {
16588                    assert!(message.contains(required), "{message}");
16589                }
16590            }
16591        }
16592
16593        #[tokio::test]
16594        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
16595            let mut rig = rig_with_flow_support(true).await;
16596            let mut record = session(1);
16597            record.attributes.flow_id = Some("flow:7".to_string());
16598            rig.sync(vec![record]).await;
16599            for opener in [OWNER, AFT] {
16600                let route = rig
16601                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16602                    .await;
16603                assert!(rig.live(&route));
16604                assert_eq!(
16605                    route.stamp().unwrap().attributes.flow_id.as_deref(),
16606                    Some("flow:7")
16607                );
16608            }
16609        }
16610
16611        #[tokio::test]
16612        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
16613            use std::future::Future;
16614
16615            let mut rig = rig_with_flow_support(true).await;
16616            let mut record = session(1);
16617            record.attributes.flow_id = Some("flow:7".to_string());
16618            rig.sync(vec![record]).await;
16619            let (client, mut client_rx, frame) =
16620                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
16621            // Hold the route-open response permit so admission sees the first
16622            // target but relay reservation cannot capture an endpoint yet.
16623            for _ in 0..64 {
16624                client.egress.try_send(route_bind_ack(1)).unwrap();
16625            }
16626            let handler = rig.handler.clone();
16627            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16628            std::future::poll_fn(|cx| {
16629                assert!(open.as_mut().poll(cx).is_pending());
16630                std::task::Poll::Ready(())
16631            })
16632            .await;
16633            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16634
16635            let old_connection = rig.modules[PLEXUS].0.connection_id;
16636            rig.handler.cleanup_connection(old_connection).unwrap();
16637            let (replacement, mut replacement_rx) = wide_ctx(200);
16638            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
16639            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16640            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16641            let hello = Frame::build(
16642                FrameType::Hello,
16643                control_flags(),
16644                0,
16645                0,
16646                200,
16647                serde_json::to_vec(&body).unwrap(),
16648            )
16649            .unwrap();
16650            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
16651
16652            client_rx.try_recv().unwrap();
16653            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16654                .await
16655                .expect("the replacement is refused without waiting for a bind ack")
16656                .unwrap();
16657            assert_eq!(replies.len(), 1);
16658            let body = parse_error(&replies[0]);
16659            assert_eq!(body["code"], "target_flow_unsupported");
16660            for required in [PLEXUS, "flow-scopes/v1"] {
16661                assert!(body["message"].as_str().unwrap().contains(required));
16662            }
16663            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
16664            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16665            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16666            assert!(rig
16667                .handler
16668                .registry
16669                .get_module_by_connection(replacement.connection_id)
16670                .unwrap()
16671                .is_some());
16672        }
16673
16674        #[tokio::test]
16675        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
16676            use std::future::Future;
16677            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16678            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16679            const NEXT_HASH: &str =
16680                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16681            let mut rig = rig().await;
16682            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
16683            record.carriers.push(ScopeCarrier::new(Principal::Direct));
16684            rig.sync(vec![record.clone()]).await;
16685            let (client, mut rx, frame) = rig.open_frame(
16686                None,
16687                PLEXUS,
16688                Some(
16689                    rig.selector("s", Some(1))
16690                        .with_bind_proof(Some(PROOF.into())),
16691                ),
16692            );
16693            // Hold egress reservation while the scope owner changes its bind-proof
16694            // hash, so a proof valid at admission must be checked again at relay.
16695            for _ in 0..64 {
16696                client.egress.try_send(route_bind_ack(1)).unwrap();
16697            }
16698            let handler = rig.handler.clone();
16699            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16700            std::future::poll_fn(|cx| {
16701                assert!(open.as_mut().poll(cx).is_pending());
16702                std::task::Poll::Ready(())
16703            })
16704            .await;
16705            record.bind_proof_sha256 = Some(NEXT_HASH.into());
16706            rig.sync(vec![record]).await;
16707            rx.try_recv().unwrap();
16708            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16709                .await
16710                .unwrap()
16711                .unwrap();
16712            assert_eq!(
16713                parse_error(&replies[0])["code"],
16714                error_codes::SCOPE_BIND_PROOF_MISMATCH
16715            );
16716            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
16717            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16718            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16719        }
16720
16721        #[tokio::test]
16722        async fn scope_bind_proof_reserved_openers_ignore_proof() {
16723            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16724            let mut rig = rig().await;
16725            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16726                .await;
16727            for opener in [OWNER, AFT] {
16728                for proof in [None, Some("malformed+proof=")] {
16729                    let route = rig
16730                        .bound(
16731                            Some(opener),
16732                            PLEXUS,
16733                            Some(
16734                                rig.selector("s", Some(1))
16735                                    .with_bind_proof(proof.map(str::to_string)),
16736                            ),
16737                        )
16738                        .await;
16739                    assert!(rig.live(&route));
16740                }
16741            }
16742        }
16743
16744        #[tokio::test]
16745        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
16746            use std::future::Future;
16747            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16748            const NEXT_HASH: &str =
16749                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16750            let mut rig = rig().await;
16751            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16752                .await;
16753            let (client, mut rx, frame) =
16754                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
16755            for _ in 0..64 {
16756                client.egress.try_send(route_bind_ack(1)).unwrap();
16757            }
16758            let handler = rig.handler.clone();
16759            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16760            std::future::poll_fn(|cx| {
16761                assert!(open.as_mut().poll(cx).is_pending());
16762                std::task::Poll::Ready(())
16763            })
16764            .await;
16765            rig.sync(vec![
16766                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
16767            ])
16768            .await;
16769            rx.try_recv().unwrap();
16770            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16771            let bind = std::future::poll_fn(|cx| {
16772                assert!(open.as_mut().poll(cx).is_pending());
16773                module_rx.poll_recv(cx)
16774            })
16775            .await
16776            .unwrap()
16777            .frame;
16778            handler
16779                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
16780                .await
16781                .unwrap();
16782            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16783                .await
16784                .unwrap()
16785                .unwrap();
16786            assert_eq!(replies.len(), 1);
16787            // Proof rechecking allowed the relay. Independent tag settlement
16788            // still asks the opener to retry after any same-epoch content change.
16789            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
16790            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16791            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
16792            let route = rig
16793                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
16794                .await;
16795            assert!(rig.live(&route));
16796        }
16797
16798        #[tokio::test]
16799        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
16800        {
16801            let mut rig = rig_with_flow_support(false).await;
16802            rig.sync(vec![session(1)]).await;
16803            let route = rig
16804                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16805                .await;
16806            assert!(rig.live(&route));
16807            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
16808            let mut record = session(1);
16809            record.attributes.flow_id = Some("flow:7".to_string());
16810            rig.sync(vec![record]).await;
16811            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
16812            assert!(rig.live(&unscoped));
16813            assert_eq!(unscoped.stamp(), None);
16814        }
16815
16816        #[tokio::test]
16817        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
16818            let mut rig = rig().await;
16819            let mut record = session(1);
16820            record.attributes.flow_id = Some("flow:7".to_string());
16821            rig.sync(vec![record.clone()]).await;
16822            let before = rig.forwarding.published_scope_tag(OWNER, "s");
16823            let mut owner_route = rig
16824                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16825                .await;
16826            let mut carrier_route = rig
16827                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
16828                .await;
16829            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16830            rig.sync(vec![record.clone()]).await;
16831            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16832            assert!(rig.live(&owner_route) && owner_route.untouched());
16833            assert!(rig.live(&carrier_route) && carrier_route.untouched());
16834
16835            record.attributes.flow_id = Some("flow:8".to_string());
16836            rig.sync(vec![record]).await;
16837            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
16838            let before = before.unwrap();
16839            assert_eq!(after.scope_epoch, before.scope_epoch);
16840            assert!(after.version > before.version);
16841            for route in [&mut owner_route, &mut carrier_route] {
16842                assert!(!rig.live(route));
16843                assert_eq!(
16844                    route.closed_reason(),
16845                    RouteCloseReason::ScopeDelegationChanged
16846                );
16847            }
16848            assert!(rig.live(&unscoped) && unscoped.untouched());
16849            // Each provider also receives a GOODBYE for its drained route;
16850            // consume it before expecting the next route.bind on that sink.
16851            for target in [PLEXUS, OTHER] {
16852                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
16853                let goodbye = module_rx
16854                    .try_recv()
16855                    .expect("the provider sees the drain")
16856                    .frame;
16857                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16858                assert!(module_rx.try_recv().is_err());
16859            }
16860            let rebound = rig
16861                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16862                .await;
16863            assert_eq!(
16864                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
16865                Some("flow:8")
16866            );
16867        }
16868
16869        /// The owner's sync lands between admission and the module's ack. The
16870        /// open is refused by name, the module's other routes stay up, and the
16871        /// reserved pair is released. Changed content is retryable; an ended
16872        /// scope is not.
16873        #[tokio::test]
16874        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
16875            let mut rig = rig().await;
16876            rig.sync(vec![session(1)]).await;
16877            let mut cotenant = rig
16878                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16879                .await;
16880
16881            let mut changed = session(1);
16882            changed.child_owners.push(Principal::Reserved {
16883                module_id: MAGIC.to_string(),
16884            });
16885            let mut ended = None;
16886            for (code, next) in [
16887                (error_codes::SCOPE_CHANGED, vec![changed]),
16888                (error_codes::SCOPE_ENDED, Vec::new()),
16889            ] {
16890                let relayed = rig
16891                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16892                    .await;
16893                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
16894                ended = Some(next.is_empty());
16895                rig.sync(next).await;
16896                rig.ack(&relayed).await;
16897                let replies = relayed.task.await.unwrap();
16898                assert_eq!(replies.len(), 1, "{replies:?}");
16899                assert_eq!(parse_error(&replies[0])["code"], code);
16900                assert_eq!(
16901                    subc_protocol::error_codes::is_retryable_route_open(code),
16902                    code == error_codes::SCOPE_CHANGED
16903                );
16904                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16905                // The module is told to drop just the binding it created.
16906                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16907                // Collected, because ending the scope also closes the co-tenant
16908                // route, whose GOODBYE comes first.
16909                let mut goodbyes = Vec::new();
16910                while let Ok(outbound) = plexus_rx.try_recv() {
16911                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
16912                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
16913                }
16914                assert!(
16915                    goodbyes.contains(&(bind_channel, bind_epoch)),
16916                    "{goodbyes:?}"
16917                );
16918                assert!(rig
16919                    .handler
16920                    .registry
16921                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
16922                    .unwrap()
16923                    .is_some());
16924            }
16925            assert_eq!(ended, Some(true));
16926            // The co-tenant stayed up through the change, and closed only when
16927            // the scope ended, by the drain rule rather than by the commit.
16928            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
16929        }
16930
16931        /// Each row of the drain table on one set of routes: the owner's, a
16932        /// bare carrier's, and a targeted carrier's to each of its targets.
16933        #[tokio::test]
16934        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
16935            struct Case {
16936                name: &'static str,
16937                change: fn(&mut ScopeRecord),
16938                /// Closed routes by index: owner->plexus, aft->plexus,
16939                /// broca->plexus, broca->other.
16940                closed: [Option<RouteCloseReason>; 4],
16941            }
16942            use RouteCloseReason::*;
16943            let cases = [
16944                Case {
16945                    name: "a carrier entry removed",
16946                    change: |r| {
16947                        r.carriers.retain(|c| {
16948                            c.principal
16949                                != Principal::Reserved {
16950                                    module_id: AFT.to_string(),
16951                                }
16952                        })
16953                    },
16954                    closed: [None, Some(ScopeCarrierRemoved), None, None],
16955                },
16956                Case {
16957                    name: "a target removed from a carrier",
16958                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
16959                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
16960                },
16961                Case {
16962                    name: "a bare carrier narrowed to targets",
16963                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
16964                    closed: [None, Some(ScopeCarrierRemoved), None, None],
16965                },
16966                Case {
16967                    name: "delegates turned off",
16968                    change: |r| r.attributes.delegates = false,
16969                    closed: [Some(ScopeDelegationChanged); 4],
16970                },
16971                Case {
16972                    name: "agent_id changed",
16973                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
16974                    closed: [Some(ScopeDelegationChanged); 4],
16975                },
16976                Case {
16977                    name: "a carrier added, child owners changed, the record re-sent",
16978                    change: |r| {
16979                        r.carriers.push(carrier(MAGIC, None));
16980                        r.child_owners.push(Principal::Reserved {
16981                            module_id: MAGIC.to_string(),
16982                        });
16983                    },
16984                    closed: [None; 4],
16985                },
16986                Case {
16987                    name: "a target added",
16988                    change: |r| {
16989                        r.carriers[1]
16990                            .targets
16991                            .as_mut()
16992                            .unwrap()
16993                            .push("third".to_string())
16994                    },
16995                    closed: [None; 4],
16996                },
16997                Case {
16998                    name: "delegates turned on",
16999                    change: |r| r.attributes.delegates = true,
17000                    closed: [None; 4],
17001                },
17002            ];
17003            for case in cases {
17004                let mut rig = rig().await;
17005                rig.sync(vec![session(1)]).await;
17006                let scope = || Some(rig_selector("s", Some(1)));
17007                let mut routes = [
17008                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
17009                    rig.bound(Some(AFT), PLEXUS, scope()).await,
17010                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
17011                    rig.bound(Some(BROCA), OTHER, scope()).await,
17012                ];
17013                let mut record = session(1);
17014                (case.change)(&mut record);
17015                rig.sync(vec![record]).await;
17016                for (index, expected) in case.closed.iter().enumerate() {
17017                    let route = &mut routes[index];
17018                    match expected {
17019                        Some(reason) => {
17020                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
17021                            assert_eq!(
17022                                route.closed_reason(),
17023                                *reason,
17024                                "{}: route {index}",
17025                                case.name
17026                            );
17027                        }
17028                        None => {
17029                            assert!(rig.live(route), "{}: route {index} closed", case.name);
17030                            assert!(
17031                                route.untouched(),
17032                                "{}: route {index} was told something",
17033                                case.name
17034                            );
17035                        }
17036                    }
17037                }
17038            }
17039        }
17040
17041        #[tokio::test]
17042        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17043            // Removed, and replaced by a higher epoch.
17044            for next in [Vec::new(), vec![session(2)]] {
17045                let mut rig = rig().await;
17046                rig.sync(vec![session(1)]).await;
17047                let mut route = rig
17048                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17049                    .await;
17050                rig.sync(next).await;
17051                assert!(!rig.live(&route));
17052                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17053            }
17054
17055            // A child whose parent ends: its routes close as parent-ended, the
17056            // child stays live, and routes under the parent close as ended.
17057            let mut rig = rig().await;
17058            let mut child = session(1);
17059            child.scope_ref = "child".to_string();
17060            child.kind = ScopeKind::Worker;
17061            child.parent = Some(ScopeParent::new(
17062                Principal::Reserved {
17063                    module_id: OWNER.to_string(),
17064                },
17065                "s".to_string(),
17066                1,
17067            ));
17068            rig.sync(vec![session(1), child.clone()]).await;
17069            let mut child_route = rig
17070                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17071                .await;
17072            assert_eq!(
17073                child_route.stamp().unwrap().parent_state,
17074                Some(ParentState::Linked)
17075            );
17076            rig.sync(vec![child]).await;
17077            assert!(!rig.live(&child_route));
17078            assert_eq!(
17079                child_route.closed_reason(),
17080                RouteCloseReason::ScopeParentEnded
17081            );
17082        }
17083
17084        #[tokio::test]
17085        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17086        ) {
17087            let mut rig = rig().await;
17088            rig.sync(vec![session(1)]).await;
17089            let mut route = rig
17090                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17091                .await;
17092            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17093            rig.sync(vec![session(1)]).await;
17094            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17095            assert!(rig.live(&route) && route.untouched());
17096
17097            // A call in flight on the route when another carrier is added. A
17098            // forwarded REQUEST holds one credit on the route's flow until the
17099            // module answers; the router takes it exactly like this.
17100            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17101                .forwarding
17102                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17103                .unwrap()
17104            else {
17105                panic!("the route is bound");
17106            };
17107            binding.flow.acquire_tagged(9, false).await.unwrap();
17108            let mut widened = session(1);
17109            widened.carriers.push(carrier(MAGIC, None));
17110            rig.sync(vec![widened]).await;
17111            assert!(rig.live(&route) && route.untouched());
17112            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17113            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17114            // The call's credit is still held on an open flow, so its answer
17115            // will be delivered: closing the route would have closed the flow.
17116            assert_eq!(binding.flow.in_flight(), 1);
17117            binding
17118                .flow
17119                .acquire_tagged(10, false)
17120                .await
17121                .expect("the flow is still open");
17122        }
17123
17124        /// A swap's superseded endpoint keeps its routes until drained; ending
17125        /// the scope closes them there too.
17126        #[tokio::test]
17127        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17128            let mut rig = rig().await;
17129            rig.sync(vec![session(1)]).await;
17130            let mut on_incumbent = rig
17131                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17132                .await;
17133
17134            // Swap plexus: register a candidate and cut over, leaving the
17135            // incumbent superseded with the route still on it.
17136            let (candidate, _candidate_rx) = wide_ctx(9);
17137            let registration = rig
17138                .handler
17139                .registry
17140                .register_candidate_with_control_ops(
17141                    manifest(PLEXUS, PROTOCOL_VERSION),
17142                    PROTOCOL_VERSION,
17143                    candidate.connection_id,
17144                    module_baseline_control_ops(),
17145                )
17146                .unwrap();
17147            rig.forwarding
17148                .register_candidate_module_connection(
17149                    candidate.connection_id,
17150                    PLEXUS.to_string(),
17151                    PROTOCOL_VERSION,
17152                    manifest_concurrency(&registration.manifest),
17153                    candidate.egress.clone(),
17154                )
17155                .unwrap();
17156            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17157            rig.handler
17158                .registry
17159                .promote_candidate(PLEXUS)
17160                .unwrap()
17161                .unwrap();
17162            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17163
17164            rig.sync(Vec::new()).await;
17165            assert!(!rig.live(&on_incumbent));
17166            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17167            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17168            let goodbye = incumbent_rx
17169                .try_recv()
17170                .expect("the superseded endpoint is told")
17171                .frame;
17172            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17173        }
17174    }
17175}
17176
17177#[cfg(test)]
17178mod concurrency_default_exposure_tests {
17179    use super::*;
17180
17181    fn hello_body(role_json: &str) -> Vec<u8> {
17182        format!(
17183            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17184        )
17185        .into_bytes()
17186    }
17187
17188    fn manifest_from(body: &[u8]) -> ModuleManifest {
17189        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17190        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17191            .expect("manifest parses")
17192    }
17193
17194    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17195
17196    #[test]
17197    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17198        let body = hello_body(&format!(
17199            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17200        ));
17201        let manifest = manifest_from(&body);
17202        // Precondition: serde really resolved it to the default, so the typed
17203        // manifest alone cannot answer the question this probe exists for.
17204        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17205        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17206    }
17207
17208    #[test]
17209    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17210        let body = hello_body(&format!(
17211            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17212        ));
17213        let manifest = manifest_from(&body);
17214        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17215        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17216    }
17217
17218    #[test]
17219    fn non_management_roles_are_never_reported() {
17220        let body = hello_body(
17221            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17222        );
17223        let manifest = manifest_from(&body);
17224        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17225    }
17226}