Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_event_declarations,
25        validate_hello_rich_ui_descriptor, validate_hello_self_signal_declarations,
26        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
27        ProviderRole,
28    },
29    scope::{
30        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
31        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
32        SCOPE_SYNC_OP,
33    },
34    session::{
35        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
36        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
37        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
38        ROLE_VERSIONS_FIELD,
39    },
40    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
41    Priority, RouteTarget, PROTOCOL_VERSION,
42};
43use tokio::time::{timeout_at, Instant};
44use tracing::{debug, info, warn};
45
46use crate::{
47    caller_identity::CallerIdentity,
48    capability_requirements::{
49        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
50        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
51        RuntimeModule,
52    },
53    child_roster::DaemonShutdownFlag,
54    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
55    forwarding::{
56        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
57        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
58        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
59    },
60    observability::{
61        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
62    },
63    privileged_prompt::{prompt_text, PrivilegedOperation},
64    provenance::{
65        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
66    },
67    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
68    router::{RouteCtx, RouterError},
69    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
70    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
71    stderr_tail::{CaptureState, TailEntry},
72    supervise::{
73        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
74        SupervisorHandle, SwapHelloAdmission,
75    },
76    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
77};
78
79/// Lowest envelope version this subc build will negotiate.
80///
81/// Module HELLO negotiation is exact: peers must use the daemon's locked
82/// protocol version. Older and newer peers receive `version_unsupported` and
83/// are not registered.
84pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
85
86const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
87const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
88const CAP_PING_PONG: &str = "ping_pong_v1";
89const CAP_SESSION_ATTACH: &str = "session_attach_v1";
90const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
91
92const SUBC_CONTROL_OPS: &[&str] = &[
93    ops::ACTIVITY_RECORD,
94    ops::ACTIVITY_QUERY,
95    ops::ACTIVITY_SUBSCRIBE,
96    ops::SERVER_DESCRIBE,
97    ops::CATALOG_LIST,
98    ops::ROUTE_OPEN,
99    ops::ROUTE_POLL,
100    ops::ROUTE_CLOSING,
101    ops::ROUTE_CLOSED,
102    ops::SUPERVISOR_LIST,
103    ops::SUPERVISOR_RESTART,
104    ops::SUPERVISOR_SWAP,
105    ops::SUPERVISOR_RELOAD,
106    ops::SUPERVISOR_RESCAN,
107    ops::SUPERVISOR_RELEASE_RESERVED,
108    ops::SUPERVISOR_SET_ENABLED,
109    ops::SUPERVISOR_HEALTH_PROBE,
110    ops::SUPERVISOR_HEALTH,
111    ops::SUPERVISOR_STDERR_TAIL,
112    ops::SUPERVISOR_TERMINALS,
113    ops::SUPERVISOR_ROUTES,
114    ops::SUPERVISOR_PROVENANCE,
115    ops::SUPERVISOR_SPAWN_SNAPSHOT,
116    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
117];
118
119/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
120/// announced by a capability in `server.describe` instead, and clients check
121/// that capability before sending it. `ck daemon --json` prints `subc_ops`
122/// verbatim and scripts read that output, so a new name there would change it.
123/// A malformed body for one of these is still refused `invalid_control_body`
124/// rather than as an unknown op.
125const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
126
127mod rescan_checked;
128use rescan_checked::{RescanMode, RescanPlan};
129
130const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
131    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
132    "supervisor.live_roots",
133    SCOPE_SYNC_OP,
134    SCOPE_APPLY_OP,
135    SCOPE_DESCRIBE_OP,
136    "operator.confirm",
137];
138
139/// Module-originated ops the daemon answers but does not advertise in
140/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
141/// belongs to is incomplete, so no module is told it works before it does.
142const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
143
144const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
145
146/// How long subc waits for a module to ack a relayed route.bind before returning
147/// `module_timeout`. The ack waits on the module's own configure, which for AFT
148/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
149/// and DB-open work — on a cold page cache or a large repo that legitimately
150/// exceeds a couple of seconds. The default is generous because rejecting a VALID
151/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
152/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
153pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
154
155/// How many CONSECUTIVE full-budget relay timeouts against one target module
156/// open that module's bind-relay breaker.
157///
158/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
159/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
160/// so three consecutive full-budget timeouts take ~36s to observe: every client
161/// whose open contributed to opening the breaker had already given up on its
162/// own. That is what makes opening the breaker unable to turn a call that would
163/// have succeeded into a refusal — it can only make an already-failing module
164/// fail faster.
165///
166/// Two would be reachable inside one default deadline. One would convict a
167/// module on a single cold-cache bind, which is exactly the valid-but-slow case
168/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
169pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
170
171/// How long a module's bind-relay breaker stays open before exactly one
172/// `route.open` is let through as a probe.
173///
174/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
175/// re-pays a full-budget stall almost continuously, and the breaker stops being
176/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
177/// deadline: a client that starts retrying after the module recovers has to get
178/// a probe opportunity inside its own deadline, or the breaker converts a
179/// recovered module into a failed call — the failure it exists to prevent,
180/// pointed the other way.
181///
182/// 20s sits between those with room on both sides, and it caps what a wedged
183/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
184/// rather than one per `route.open` per connection. The stall that motivated
185/// this, with its measurements, is written up in
186/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
187/// waited the whole budget out.
188pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
189
190const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
191const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
192
193fn reload_verdict(
194    configured: &Path,
195    spawned_from: Option<&Path>,
196    image: subc_control::RunningImageAgreement,
197) -> PendingReloadVerdict {
198    let path = match spawned_from {
199        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
200            ReloadPathAgreement::Match
201        }
202        Some(spawned_from) => ReloadPathAgreement::Mismatch {
203            configured: configured.to_path_buf(),
204            spawned_from: spawned_from.to_path_buf(),
205        },
206        None => ReloadPathAgreement::Unavailable {
207            reason: if matches!(
208                image,
209                subc_control::RunningImageAgreement::Unavailable {
210                    reason: subc_control::RunningImageUnavailableReason::NotRunning
211                }
212            ) {
213                ReloadPathUnavailableReason::NotRunning
214            } else {
215                ReloadPathUnavailableReason::SpawnedPathUnavailable
216            },
217        },
218    };
219    PendingReloadVerdict {
220        path,
221        image,
222        spec: None,
223    }
224}
225
226fn launch_spec_agreement(
227    configured: &crate::supervise::ModuleSpec,
228    running: bool,
229    spawned: Option<&crate::supervise::LaunchSpecDigest>,
230) -> subc_control::LaunchSpecAgreement {
231    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
232    if !running {
233        return Agreement::Unavailable {
234            reason: Reason::NotRunning,
235        };
236    }
237    match spawned {
238        None => Agreement::Unavailable {
239            reason: Reason::SpawnedSpecUnavailable,
240        },
241        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
242            Agreement::Match
243        }
244        Some(_) => Agreement::Mismatch,
245    }
246}
247
248fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
249    if configured == spawned_from {
250        return true;
251    }
252    #[cfg(windows)]
253    {
254        // Windows spawn records the resolved verbatim path (\\?\...), while
255        // configuration keeps its original spelling. Resolve both with the OS
256        // instead of stripping prefixes or folding case, which could conflate
257        // distinct names in case-sensitive directories. Different resolved
258        // paths still differ even if their files have identical content.
259        match (
260            std::fs::canonicalize(configured),
261            std::fs::canonicalize(spawned_from),
262        ) {
263            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
264            _ => false,
265        }
266    }
267    #[cfg(not(windows))]
268    {
269        false
270    }
271}
272
273#[derive(Clone)]
274struct DaemonProvenanceFacts {
275    build: DaemonBuildProvenance,
276    pid: Option<u32>,
277    started_at_ms: Option<u64>,
278    start_clock: Option<crate::clock::StartClock>,
279    executable_path: Option<PathBuf>,
280    executable_identity: Option<SpawnedFileIdentity>,
281    process_start_time: Option<u64>,
282    probe: ExecutableIdentityProbe,
283}
284
285impl Default for DaemonProvenanceFacts {
286    fn default() -> Self {
287        Self {
288            build: DaemonBuildProvenance {
289                build_git_sha: None,
290                build_lock_digest: None,
291            },
292            pid: None,
293            started_at_ms: None,
294            start_clock: None,
295            executable_path: None,
296            executable_identity: None,
297            process_start_time: None,
298            probe: ExecutableIdentityProbe::default(),
299        }
300    }
301}
302
303#[derive(Debug, Clone)]
304struct SupervisorRescanContext {
305    supervisor: Supervisor,
306    config_path: PathBuf,
307    configured_port: Option<u16>,
308    storage_config: Option<crate::daemon_config::StorageConfig>,
309    admission_facts_carrier_module_id: Option<String>,
310    admission_facts_targets: Option<Vec<String>>,
311    scope_authority_owners: Vec<String>,
312    privileged_operator_prompt: PrivilegedOperatorPrompt,
313    /// Counts executed rescans that set out to change the supervised modules'
314    /// staged or applied configuration, counted before the first change so a
315    /// rescan that fails partway still counts. With the daemon incarnation it
316    /// names the state a `supervisor.rescan_checked` preview was computed
317    /// against.
318    generation: Arc<std::sync::atomic::AtomicU64>,
319}
320
321/// Refusal labels passed to `observe_route_open_refusal` that mean the target
322/// module is not serving right now, and so open or extend an outage in the
323/// route outage tracker. Every one of them is only reachable after the target
324/// was found in the registry, which is what keeps an arbitrary client-chosen
325/// id from ever creating tracker state.
326///
327/// Deliberately absent: `not_registered` and `removed` (the id may be
328/// anything a client sent, and a removed module is gone on purpose),
329/// `protocol_none` (such a module never serves routes, so nothing is out),
330/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
331/// capability and admission-facts refusals (they refuse the caller, not a
332/// module outage), and `relay_reservation_failed` (its code ranges over
333/// capacity limits as well as a vanished connection). Capacity, breaker,
334/// relay-timeout and module-rejection refusals do not pass through that
335/// function at all; the breaker logs its own transitions.
336///
337/// The two not-serving refusals that bypass that function record themselves
338/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
339/// `required_capability_unprovided` is not tracked: the module itself is up,
340/// and the outage belongs to the missing provider.
341const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
342    "reloading",
343    "supervisor_not_live",
344    "registration_not_active",
345    "no_forwarding_connection",
346    "relay_send_failed",
347];
348
349/// Real channel-0 control handler for subc itself.
350#[derive(Clone)]
351pub struct ControlHandler {
352    registry: Arc<Registry>,
353    activity: Option<crate::activity::ActivityLedger>,
354    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
355    forwarding: Arc<ForwardingTable>,
356    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
357    supervisor: SupervisorHandle,
358    subc_capabilities: Arc<[String]>,
359    /// Daemon-wide route.bind relay budget. Used as the fallback when the
360    /// target module has no per-module override in
361    /// `route_bind_relay_timeouts`.
362    route_bind_relay_timeout: Duration,
363    /// Per-module route.bind relay budget overrides, keyed by module id. When
364    /// `handle_route_open` resolves the deadline for a target module, a
365    /// per-module entry wins over the daemon-wide value above.
366    route_bind_relay_timeouts: BTreeMap<String, Duration>,
367    /// Per-target-module bind-relay breaker state. Shared with the forwarding
368    /// table, which is where a new module connection resets it.
369    route_bind_breakers: RouteBindBreakers,
370    /// Live relay admissions keyed by target module. Shared through the
371    /// forwarding table so cloned or separately built handlers enforce one cap.
372    route_bind_concurrency: RouteBindConcurrency,
373    /// Start and end of each module's not-serving period as seen by
374    /// `route.open`, so an outage gets one line at each edge instead of only
375    /// the per-refusal INFO lines. Taken from the forwarding table, so every
376    /// handler built over one table shares it.
377    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
378    /// Consecutive relay timeouts that open a module's breaker.
379    route_bind_breaker_threshold: u32,
380    /// How long a breaker stays open before one probe is admitted.
381    route_bind_breaker_cooldown: Duration,
382    health_probe_timeout: Duration,
383    /// Central storage policy. When set, each registering module receives its
384    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
385    storage_config: Option<crate::daemon_config::StorageConfig>,
386    /// The machine id established at boot, served on every HELLO_ACK and on
387    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
388    /// changes the file, never this value. `None` serves no id.
389    machine_id: Option<crate::machine_id::MachineId>,
390    admission_facts_carrier_module_id: Option<String>,
391    admission_facts_targets: Option<Vec<String>>,
392    /// Scope records with their sync authorities and tombstones; see
393    /// `crate::scopes`. Shared by clones of this handler, so every connection
394    /// reads and writes one table.
395    scopes: Arc<RwLock<ScopeTable>>,
396    /// Wakes the expiry loop when an accepted scope change may have added a
397    /// deadline, so the loop can wait without ticking while no live scope has
398    /// one. A permit is stored when nothing waits, so no wake is lost.
399    scope_deadline_added: Arc<tokio::sync::Notify>,
400    /// The configured `scope_authority_owners`, kept so a rescan can report a
401    /// changed value as needing a daemon restart; rescan never applies it.
402    scope_authority_owners: Vec<String>,
403    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
404    privileged_operator_prompt: PrivilegedOperatorPrompt,
405    /// The local and peer address of each accepted TCP client connection, read
406    /// from the daemon's own socket, so a privileged request can name its caller.
407    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
408    #[cfg(test)]
409    caller_lookup: Option<TestCallerLookup>,
410    /// The launch nonce each module connection presented at HELLO, which is how
411    /// a `scope.sync` is matched to the owner's current launch.
412    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
413    rescan: Option<SupervisorRescanContext>,
414    connected_clients: ConnectedClients,
415    counters: DaemonCounters,
416    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
417    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
418    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
419    /// their usual severity (see [`log_requirement_events`]). A handler built
420    /// without one has a flag that is never set.
421    daemon_shutdown: DaemonShutdownFlag,
422    daemon_provenance: DaemonProvenanceFacts,
423    #[cfg(test)]
424    control_dispatch_delay: Option<Duration>,
425    #[cfg(test)]
426    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
427    /// Holds `route.open` after the route.bind is queued to the module and
428    /// before it is marked enqueued, so a test can make the module's reply (or
429    /// a client teardown) land exactly inside that window.
430    #[cfg(test)]
431    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
432    /// Holds a HELLO after its registry entry is reserved and before its
433    /// forwarding endpoint is installed, so a test can read the catalog and
434    /// open routes while the module is registered but not yet routable. A
435    /// blocking barrier because the HELLO handler is synchronous; the test
436    /// runs the HELLO on its own thread.
437    #[cfg(test)]
438    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
439}
440
441impl fmt::Debug for ControlHandler {
442    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443        f.debug_struct("ControlHandler")
444            .field("registry", &self.registry)
445            .field("forwarding", &self.forwarding)
446            .field("process_liveness", &self.process_liveness.is_some())
447            .field("supervisor", &self.supervisor)
448            .field("subc_capabilities", &self.subc_capabilities)
449            .finish()
450    }
451}
452
453struct RouteOpenRequest {
454    target: RouteTarget,
455    identity: BindIdentity,
456    consumer_identity: Option<ConsumerIdentity>,
457    consumer_capabilities: Option<Vec<String>>,
458    role_versions: Option<BTreeMap<String, String>>,
459    admission_facts: Option<serde_json::Value>,
460    scope: Option<ScopeSelector>,
461}
462
463#[derive(Clone, Copy)]
464enum RouteOpenPhase {
465    Admission,
466    ProjectRoot,
467    EgressReservation,
468    BindSend,
469    BindAckWait,
470}
471
472/// Store elapsed time for each route-open phase, including attempts that end
473/// in an early refusal or backend error. Admission also includes rechecks and
474/// commit bookkeeping performed between the other timed phases.
475struct RouteOpenTimings {
476    module_id: String,
477    principal: String,
478    admission: Duration,
479    project_root: Duration,
480    egress_reservation: Duration,
481    bind_send: Duration,
482    bind_ack_wait: Duration,
483    phase: Option<RouteOpenPhase>,
484    phase_started_at: Instant,
485}
486
487impl RouteOpenTimings {
488    fn new() -> Self {
489        Self {
490            module_id: String::new(),
491            // Refusals before attestation must not claim a verified principal.
492            principal: "unresolved".to_string(),
493            admission: Duration::ZERO,
494            project_root: Duration::ZERO,
495            egress_reservation: Duration::ZERO,
496            bind_send: Duration::ZERO,
497            bind_ack_wait: Duration::ZERO,
498            phase: Some(RouteOpenPhase::Admission),
499            phase_started_at: Instant::now(),
500        }
501    }
502
503    fn finish(&mut self) {
504        let elapsed = self.phase_started_at.elapsed();
505        match self.phase.take() {
506            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
507            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
508            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
509            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
510            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
511            None => {}
512        }
513    }
514
515    fn enter(&mut self, phase: RouteOpenPhase) {
516        self.finish();
517        self.phase = Some(phase);
518        self.phase_started_at = Instant::now();
519    }
520}
521
522struct RouteBindReservationGuard {
523    forwarding: Arc<ForwardingTable>,
524    endpoint: ModuleEndpointId,
525    relay_corr: u64,
526    armed: bool,
527}
528
529struct ModuleControlRpcGuard {
530    forwarding: Arc<ForwardingTable>,
531    endpoint: ModuleEndpointId,
532    corr: u64,
533    armed: bool,
534}
535
536impl ModuleControlRpcGuard {
537    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
538        Self {
539            forwarding,
540            endpoint,
541            corr,
542            armed: true,
543        }
544    }
545
546    fn disarm(&mut self) {
547        self.armed = false;
548    }
549}
550
551impl Drop for ModuleControlRpcGuard {
552    fn drop(&mut self) {
553        if self.armed {
554            let _ = self
555                .forwarding
556                .cancel_module_control_rpc(self.endpoint, self.corr);
557        }
558    }
559}
560
561impl RouteBindReservationGuard {
562    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
563        Self {
564            forwarding,
565            endpoint,
566            relay_corr,
567            armed: true,
568        }
569    }
570
571    fn release_and_disarm(&mut self) {
572        if !self.armed {
573            return;
574        }
575        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
576            self.endpoint,
577            self.relay_corr,
578            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
579        ) {
580            send_goodbye_target_best_effort(
581                &self.forwarding.counters(),
582                &target,
583                "canceled route.bind",
584            );
585        }
586        self.armed = false;
587    }
588
589    fn disarm(&mut self) {
590        self.armed = false;
591    }
592}
593
594impl Drop for RouteBindReservationGuard {
595    fn drop(&mut self) {
596        self.release_and_disarm();
597    }
598}
599
600/// Per-target-module circuit breaker around the `route.bind` relay.
601///
602/// The connection reader is serial per connection, so a module whose `on_bind`
603/// sits on the ack blocks every LATER frame on the connections that call it,
604/// including calls to unrelated modules. This does not make any module's bind
605/// fast; it stops the daemon paying the full budget again and again for a
606/// condition it has already observed.
607///
608/// State is keyed by TARGET MODULE and shared by every connection: a wedged
609/// module wedges everyone, so what one connection learned should protect the
610/// rest.
611///
612/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
613/// relay to that module has actually timed out, and is removed again when a
614/// relay is accepted or the module reconnects, so it cannot grow with traffic
615/// or with modules that behave.
616///
617/// # Why a `std` mutex here is not the head-of-line defect again
618///
619/// Acquisition never awaits. The critical section is a hash lookup plus a few
620/// integer updates, with no I/O and no `.await` inside it, so a reader task
621/// cannot be descheduled behind it the way it can behind
622/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
623/// primitive, held for the same kind of work, as the refusal counter this very
624/// path already increments.
625///
626/// It is also NOT on the data-plane splice path: only `route.open` and module
627/// registration touch it, so bound-route frames gain no state check and no
628/// contention.
629#[derive(Debug, Clone, Default)]
630pub(crate) struct RouteBindBreakers {
631    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
632}
633
634#[derive(Debug, Clone, Default)]
635pub(crate) struct RouteBindConcurrency {
636    modules: Arc<Mutex<HashMap<String, usize>>>,
637}
638
639struct RouteBindConcurrencyGuard {
640    concurrency: RouteBindConcurrency,
641    module_id: String,
642}
643
644impl RouteBindConcurrency {
645    /// Admit without waiting. Waiting here would move the bind stall from the
646    /// module reply to a semaphore and restore reader head-of-line blocking.
647    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
648        let mut modules = self
649            .modules
650            .lock()
651            .expect("route.bind concurrency mutex poisoned");
652        let in_flight = modules.entry(module_id.to_string()).or_default();
653        if *in_flight >= limit {
654            return Err(*in_flight);
655        }
656        *in_flight += 1;
657        Ok(RouteBindConcurrencyGuard {
658            concurrency: self.clone(),
659            module_id: module_id.to_string(),
660        })
661    }
662}
663
664impl Drop for RouteBindConcurrencyGuard {
665    fn drop(&mut self) {
666        let mut modules = self
667            .concurrency
668            .modules
669            .lock()
670            .expect("route.bind concurrency mutex poisoned");
671        let remove = {
672            let in_flight = modules
673                .get_mut(&self.module_id)
674                .expect("admitted route.bind has a concurrency entry");
675            *in_flight -= 1;
676            *in_flight == 0
677        };
678        if remove {
679            modules.remove(&self.module_id);
680        }
681    }
682}
683
684#[derive(Debug, Default)]
685struct ModuleBreakerState {
686    /// Relay timeouts observed with no accepted relay in between.
687    consecutive_timeouts: u32,
688    /// `Some` while the breaker is open: the instant the cooldown expires and
689    /// the next arrival may probe. `None` means closed.
690    cooldown_until: Option<Instant>,
691    /// A half-open probe has been admitted and has not settled yet. This is
692    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
693    /// that read the cooldown, so concurrent opens arriving at the moment the
694    /// cooldown expires cannot all decide that they are the probe.
695    probe_in_flight: Option<Arc<()>>,
696}
697
698/// What the breaker decided for one `route.open`, before any relay work.
699enum RouteBindAdmission<'a> {
700    Admitted {
701        guard: RouteBindBreakerGuard<'a>,
702        /// This open is the single half-open probe, so the transition is worth
703        /// one log line.
704        probe: bool,
705    },
706    Refused {
707        consecutive_timeouts: u32,
708        /// What is left of the cooldown. Zero when the refusal is because the
709        /// one probe is already in flight rather than because the cooldown has
710        /// not elapsed.
711        retry_in: Duration,
712        probe_in_flight: bool,
713    },
714}
715
716/// An outstanding admission, which must be told how its relay settled.
717///
718/// `Drop` settles it as inconclusive, so an early return between admission and
719/// the relay -- or the whole handler being cancelled when the client
720/// disconnects -- releases a half-open probe slot instead of leaving the
721/// breaker wedged half-open with no further probes.
722struct RouteBindBreakerGuard<'a> {
723    breakers: RouteBindBreakers,
724    module_id: &'a str,
725    probe_token: Option<Arc<()>>,
726    settled: bool,
727}
728
729impl RouteBindBreakerGuard<'_> {
730    /// The module answered within the budget and took the bind. THE ONLY
731    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
732    /// breaker, which is a transition worth logging.
733    fn record_accepted(&mut self) -> bool {
734        self.settled = true;
735        self.breakers.record_accepted(self.module_id)
736    }
737
738    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
739    /// COUNTS TOWARD OPENING.
740    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
741        self.settled = true;
742        self.breakers.record_timeout(
743            self.module_id,
744            self.probe_token.as_ref(),
745            threshold,
746            cooldown,
747        )
748    }
749
750    /// Everything else: the module REJECTED the bind, its connection went away
751    /// mid-relay, or the waiter was cancelled.
752    ///
753    /// None of these is evidence that a module is slow, and each already has
754    /// its own refusal with its own code. A module that rejects a bind in
755    /// microseconds is healthy and must never be convicted for it; a module
756    /// that died has said nothing about the module that replaces it. So these
757    /// neither increment nor reset the count -- they only release a probe slot.
758    fn record_inconclusive(&mut self) {
759        self.settled = true;
760        self.breakers
761            .record_inconclusive(self.module_id, self.probe_token.as_ref());
762    }
763}
764
765impl Drop for RouteBindBreakerGuard<'_> {
766    fn drop(&mut self) {
767        if !self.settled {
768            self.breakers
769                .record_inconclusive(self.module_id, self.probe_token.as_ref());
770        }
771    }
772}
773
774/// The breaker moved to open, reported so the caller can log it outside the
775/// lock. Opening is rare and load-bearing; the refusals that follow are
776/// frequent and are counted rather than logged.
777struct BreakerOpened {
778    consecutive_timeouts: u32,
779    /// True when a failed probe re-opened an already-open breaker, which reads
780    /// very differently in a log from a first opening.
781    reopened_after_probe: bool,
782}
783
784impl RouteBindBreakers {
785    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
786        self.modules
787            .lock()
788            .expect("route.bind breaker mutex poisoned")
789    }
790
791    /// Decide whether this `route.open` may attempt its relay. Takes the map
792    /// lock and nothing else, and never awaits.
793    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
794        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
795            probe: probe_token.is_some(),
796            guard: RouteBindBreakerGuard {
797                breakers: self.clone(),
798                module_id,
799                probe_token,
800                settled: false,
801            },
802        };
803
804        let mut modules = self.lock();
805        let Some(state) = modules.get_mut(module_id) else {
806            return admitted(None);
807        };
808        let Some(cooldown_until) = state.cooldown_until else {
809            return admitted(None);
810        };
811        if state.probe_in_flight.is_some() {
812            return RouteBindAdmission::Refused {
813                consecutive_timeouts: state.consecutive_timeouts,
814                retry_in: Duration::ZERO,
815                probe_in_flight: true,
816            };
817        }
818        let now = Instant::now();
819        if now < cooldown_until {
820            return RouteBindAdmission::Refused {
821                consecutive_timeouts: state.consecutive_timeouts,
822                retry_in: cooldown_until - now,
823                probe_in_flight: false,
824            };
825        }
826        let token = Arc::new(());
827        state.probe_in_flight = Some(Arc::clone(&token));
828        admitted(Some(token))
829    }
830
831    fn record_accepted(&self, module_id: &str) -> bool {
832        self.lock()
833            .remove(module_id)
834            .is_some_and(|state| state.cooldown_until.is_some())
835    }
836
837    fn record_timeout(
838        &self,
839        module_id: &str,
840        probe_token: Option<&Arc<()>>,
841        threshold: u32,
842        cooldown: Duration,
843    ) -> Option<BreakerOpened> {
844        let mut modules = self.lock();
845        let state = modules.entry(module_id.to_string()).or_default();
846        let was_open = state.cooldown_until.is_some();
847        let was_probe = Self::owns_probe(state, probe_token);
848        if was_probe {
849            state.probe_in_flight = None;
850        }
851        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
852        if state.consecutive_timeouts < threshold {
853            return None;
854        }
855        state.cooldown_until = Some(Instant::now() + cooldown);
856        Some(BreakerOpened {
857            consecutive_timeouts: state.consecutive_timeouts,
858            reopened_after_probe: was_open && was_probe,
859        })
860    }
861
862    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
863        // A relay can finish after the breaker was reset or after it opened
864        // again and started a new probe. Only the guard whose token matches the
865        // active probe may release it, so a late relay never frees a newer probe.
866        state
867            .probe_in_flight
868            .as_ref()
869            .zip(token)
870            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
871    }
872
873    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
874        if let Some(state) = self.lock().get_mut(module_id) {
875            if Self::owns_probe(state, probe_token) {
876                state.probe_in_flight = None;
877            }
878        }
879    }
880
881    /// Discard what was learned about a module, because the process it was
882    /// learned about is gone. Returns the discarded count when it was non-zero.
883    ///
884    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
885    /// `module_id` is a configuration identity that outlives any particular
886    /// child; what the breaker observed was the process behind the module
887    /// connection of the moment. When a new connection registers under that id
888    /// the verdict's subject no longer exists, so the verdict is stale by
889    /// construction rather than merely likely to be wrong. Keeping it would
890    /// apply a dead process's record to a live one, which is the same defect
891    /// class this breaker exists to stop the daemon committing.
892    ///
893    /// A half-open probe in flight is discarded with the rest: it was a
894    /// question about the old process.
895    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
896        self.lock()
897            .remove(module_id)
898            .map(|state| state.consecutive_timeouts)
899            .filter(|discarded| *discarded > 0)
900    }
901
902    /// Open breakers, for the `server.describe` counters object. `None` when
903    /// none is open, so the key stays absent rather than present-and-empty.
904    ///
905    /// This is the operator's answer to "is this module refusing instantly or
906    /// is it fine?", which look identical from a client that retries and then
907    /// succeeds.
908    fn open_snapshot(&self) -> Option<serde_json::Value> {
909        let now = Instant::now();
910        let modules = self.lock();
911        let open = modules
912            .iter()
913            .filter_map(|(module_id, state)| {
914                let cooldown_until = state.cooldown_until?;
915                Some((
916                    module_id.clone(),
917                    serde_json::json!({
918                        "consecutive_timeouts": state.consecutive_timeouts,
919                        "cooldown_remaining_ms":
920                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
921                        "probe_in_flight": state.probe_in_flight.is_some(),
922                    }),
923                ))
924            })
925            .collect::<serde_json::Map<String, serde_json::Value>>();
926        (!open.is_empty()).then_some(serde_json::Value::Object(open))
927    }
928}
929
930impl ControlHandler {
931    pub fn new(registry: Arc<Registry>) -> Self {
932        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
933    }
934
935    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
936        let counters = forwarding.counters();
937        // Taken from the forwarding table rather than created here, so that the
938        // breaker a `route.open` consults is the same one a module's
939        // registration resets, however many handlers are built over one table.
940        let route_bind_breakers = forwarding.route_bind_breakers();
941        let route_bind_concurrency = forwarding.route_bind_concurrency();
942        let route_outages = forwarding.route_outages();
943        Self {
944            registry,
945            activity: None,
946            activity_streams: Arc::new(Mutex::new(HashMap::new())),
947            forwarding,
948            process_liveness: None,
949            supervisor: SupervisorHandle::new(),
950            subc_capabilities: Arc::from([
951                CAP_MANIFEST_REGISTRATION.to_string(),
952                CAP_CHANNEL_LIFECYCLE.to_string(),
953                CAP_PING_PONG.to_string(),
954                CAP_SESSION_ATTACH.to_string(),
955                CAP_ADMISSION_FACTS_RELAY.to_string(),
956                CAP_SCOPES_V1.to_string(),
957                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
958                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
959                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
960                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
961                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
962                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
963                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
964                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
965                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
966            ]),
967            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
968            route_bind_relay_timeouts: BTreeMap::new(),
969            route_bind_breakers,
970            route_bind_concurrency,
971            route_outages,
972            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
973            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
974            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
975            storage_config: None,
976            machine_id: None,
977            admission_facts_carrier_module_id: None,
978            admission_facts_targets: None,
979            scopes: Arc::new(RwLock::new(ScopeTable::new(
980                crate::daemon_config::default_scope_authority_owners(),
981            ))),
982            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
983            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
984            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
985            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
986            #[cfg(test)]
987            caller_lookup: None,
988            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
989            rescan: None,
990            connected_clients: ConnectedClients::new(),
991            counters,
992            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
993            daemon_shutdown: DaemonShutdownFlag::default(),
994            daemon_provenance: DaemonProvenanceFacts::default(),
995            #[cfg(test)]
996            control_dispatch_delay: None,
997            #[cfg(test)]
998            provenance_probe_override: None,
999            #[cfg(test)]
1000            route_bind_relay_sent_hold: None,
1001            #[cfg(test)]
1002            hello_registry_to_forwarding_hold: None,
1003        }
1004    }
1005
1006    /// Set the central storage policy: registering modules then receive their
1007    /// resolved storage descriptor in HELLO_ACK.
1008    pub fn with_storage_config(
1009        mut self,
1010        storage_config: Option<crate::daemon_config::StorageConfig>,
1011    ) -> Self {
1012        self.storage_config = storage_config;
1013        self
1014    }
1015
1016    /// Open the ledger on its own writer thread; an unavailable sink does not
1017    /// prevent the daemon from serving routing or supervisor requests.
1018    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1019        self.activity = Some(crate::activity::ActivityLedger::open(
1020            run_dir.join("activity.jsonl"),
1021            incarnation,
1022        ));
1023        self.supervisor.set_activity(self.activity.clone());
1024        self
1025    }
1026
1027    /// Set the machine id served to every registering module (HELLO_ACK) and on
1028    /// `server.describe`.
1029    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1030        self.machine_id = machine_id;
1031        self
1032    }
1033
1034    /// Configure the exact reserved module and target ids permitted to relay
1035    /// opaque admission facts. Config-file loading validates this authority;
1036    /// this builder keeps the same policy available to embedded test daemons.
1037    pub fn with_admission_facts_config(
1038        mut self,
1039        carrier_module_id: Option<String>,
1040        targets: Option<Vec<String>>,
1041    ) -> Self {
1042        self.admission_facts_carrier_module_id = carrier_module_id;
1043        self.admission_facts_targets = targets;
1044        self
1045    }
1046
1047    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1048    /// Replaces the scope table with an empty one under the new list, so call it
1049    /// while building the handler, before any module can sync.
1050    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1051        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1052        self.scope_authority_owners = owners;
1053        self
1054    }
1055
1056    /// Set the startup policy before installing the rescan context or serving requests.
1057    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1058        self.privileged_operator_prompt = mode;
1059        self
1060    }
1061
1062    /// Override the route.bind relay timeout. Used by tests that assert the
1063    /// timeout path so they don't block on the production-safe default.
1064    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1065        self.route_bind_relay_timeout = timeout;
1066        self
1067    }
1068
1069    /// Install per-module route.bind relay budget overrides. A module id
1070    /// listed here wins over the daemon-wide default set via
1071    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1072    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1073    /// the same `Duration` the bind path will use.
1074    pub fn with_route_bind_relay_timeouts(
1075        mut self,
1076        timeouts: impl IntoIterator<Item = (String, Duration)>,
1077    ) -> Self {
1078        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1079        self
1080    }
1081
1082    /// Resolve the route.bind relay budget for a specific target module id.
1083    /// Per-module overrides win; the daemon-wide value (set via
1084    /// `with_route_bind_relay_timeout` or the built-in default) is the
1085    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1086    /// the same resolution `handle_route_open` will use.
1087    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1088        self.route_bind_relay_timeouts
1089            .get(module_id)
1090            .copied()
1091            .unwrap_or(self.route_bind_relay_timeout)
1092    }
1093
1094    /// Override the per-module bind-relay breaker policy.
1095    ///
1096    /// Used by tests, which cannot spend three production budgets opening a
1097    /// breaker or twenty seconds waiting for its cooldown. The production
1098    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1099    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1100    /// reasoning for the numbers.
1101    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1102        self.route_bind_breaker_threshold = threshold.max(1);
1103        self.route_bind_breaker_cooldown = cooldown;
1104        self
1105    }
1106
1107    #[cfg(test)]
1108    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1109        self.health_probe_timeout = timeout;
1110        self
1111    }
1112
1113    #[cfg(test)]
1114    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1115        self.control_dispatch_delay = Some(delay);
1116        self
1117    }
1118
1119    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1120    /// announce it is parked, once to be released. Build the barrier for two
1121    /// parties, the test being the other one.
1122    #[cfg(test)]
1123    pub(crate) fn with_route_bind_relay_sent_hold(
1124        mut self,
1125        hold: Arc<tokio::sync::Barrier>,
1126    ) -> Self {
1127        self.route_bind_relay_sent_hold = Some(hold);
1128        self
1129    }
1130
1131    /// A HELLO waits on `hold` twice between reserving its registry entry and
1132    /// installing its forwarding endpoint: once to announce it is parked, once
1133    /// to be released. Build the barrier for two parties, the test being the
1134    /// other one, and run the HELLO on a thread of its own.
1135    #[cfg(test)]
1136    pub(crate) fn with_hello_registry_to_forwarding_hold(
1137        mut self,
1138        hold: Arc<std::sync::Barrier>,
1139    ) -> Self {
1140        self.hello_registry_to_forwarding_hold = Some(hold);
1141        self
1142    }
1143
1144    #[cfg(test)]
1145    fn hold_between_registry_and_forwarding(&self) {
1146        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1147            hold.wait();
1148            hold.wait();
1149        }
1150    }
1151
1152    pub fn with_process_liveness(
1153        mut self,
1154        process_liveness: Arc<dyn ModuleProcessLiveness>,
1155    ) -> Self {
1156        self.process_liveness = Some(process_liveness);
1157        self
1158    }
1159
1160    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1161        self.supervisor = supervisor;
1162        self.supervisor.set_activity(self.activity.clone());
1163        self
1164    }
1165
1166    pub fn with_daemon_provenance(
1167        mut self,
1168        pid: u32,
1169        started_at_ms: u64,
1170        executable_path: Option<PathBuf>,
1171        build_git_sha: Option<String>,
1172        build_lock_digest: Option<String>,
1173    ) -> Self {
1174        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1175        let process_start_time = process_start_time(pid);
1176        self.daemon_provenance = DaemonProvenanceFacts {
1177            build: DaemonBuildProvenance {
1178                build_git_sha,
1179                build_lock_digest,
1180            },
1181            pid: Some(pid),
1182            started_at_ms: Some(started_at_ms),
1183            start_clock: None,
1184            executable_path,
1185            executable_identity,
1186            process_start_time,
1187            probe: ExecutableIdentityProbe::default(),
1188        };
1189        self
1190    }
1191
1192    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1193        self.daemon_provenance.start_clock = Some(clock);
1194        self
1195    }
1196
1197    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1198    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1199        self.scopes
1200            .write()
1201            .unwrap_or_else(|poisoned| poisoned.into_inner())
1202            .set_wall_clock(Arc::new(clock));
1203        self
1204    }
1205
1206    #[cfg(test)]
1207    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1208        self.provenance_probe_override = Some(result);
1209        self
1210    }
1211
1212    /// Install the configured module set and its reserved capability bindings.
1213    /// Bindings are configuration-scoped and may point at a provider that has not
1214    /// been installed yet, so this does not require the bound module to exist.
1215    pub fn with_capability_config(
1216        self,
1217        modules: impl IntoIterator<Item = (String, bool)>,
1218        reserved_capabilities: BTreeMap<String, String>,
1219    ) -> Self {
1220        self.capability_evaluator
1221            .configure(modules, reserved_capabilities);
1222        self
1223    }
1224
1225    /// Share the supervisor's daemon-shutdown flag with this handler, so
1226    /// requirement changes after the daemon has begun its announced shutdown
1227    /// are not reported as faults.
1228    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1229        self.daemon_shutdown = flag;
1230        self
1231    }
1232
1233    pub fn with_supervisor_rescan(
1234        mut self,
1235        supervisor: Supervisor,
1236        config_path: impl Into<PathBuf>,
1237        configured_port: Option<u16>,
1238    ) -> Self {
1239        self.rescan = Some(SupervisorRescanContext {
1240            supervisor,
1241            config_path: config_path.into(),
1242            configured_port,
1243            storage_config: self.storage_config.clone(),
1244            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1245            admission_facts_targets: self.admission_facts_targets.clone(),
1246            scope_authority_owners: self.scope_authority_owners.clone(),
1247            privileged_operator_prompt: self.privileged_operator_prompt,
1248            generation: Arc::default(),
1249        });
1250        self
1251    }
1252
1253    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1254        self.connected_clients = connected_clients;
1255        self
1256    }
1257
1258    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1259        Arc::clone(&self.forwarding)
1260    }
1261
1262    pub(crate) fn counters(&self) -> DaemonCounters {
1263        self.counters.clone()
1264    }
1265
1266    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1267    /// requirement event without depending on an operator polling a status command.
1268    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1269        tokio::spawn(async move {
1270            loop {
1271                self.capability_evaluator
1272                    .wait_for_change_or_deadline()
1273                    .await;
1274                self.refresh_capability_requirements();
1275            }
1276        });
1277    }
1278
1279    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1280    /// deadline passed during sleep is enforced on the first tick after wake.
1281    /// While no live scope carries a deadline the loop waits for an accepted
1282    /// scope change instead of ticking, so a daemon without deadlines never
1283    /// wakes for expiry.
1284    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1285        tokio::spawn(async move {
1286            loop {
1287                let has_deadlines = self
1288                    .scopes
1289                    .read()
1290                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1291                    .has_deadlines();
1292                if !has_deadlines {
1293                    self.scope_deadline_added.notified().await;
1294                    continue;
1295                }
1296                tokio::time::sleep(Duration::from_secs(1)).await;
1297                if let Err(error) = self.sweep_expired_scopes() {
1298                    warn!(%error, "scope expiry sweep failed");
1299                }
1300            }
1301        });
1302    }
1303
1304    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1305        let mut table = self
1306            .scopes
1307            .write()
1308            .unwrap_or_else(|poisoned| poisoned.into_inner());
1309        let swept = table.sweep_expired();
1310        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1311        drop(table);
1312        self.close_scope_drained_routes(drained);
1313        Ok(())
1314    }
1315
1316    fn publish_scope_changes(
1317        &self,
1318        changes: &[crate::scopes::ScopeTagChange],
1319        expired: &[crate::scopes::ScopeExpired],
1320    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1321        if changes.is_empty() {
1322            return Ok(Vec::new());
1323        }
1324        let drained = self
1325            .forwarding
1326            .publish_scope_changes(changes)
1327            .map_err(RouterError::Forwarding)?;
1328        let mut counts = HashMap::new();
1329        for route in &drained {
1330            *counts
1331                .entry((
1332                    route.scope.owner.as_str(),
1333                    route.scope.scope_ref.as_str(),
1334                    route.scope.tag.scope_epoch,
1335                ))
1336                .or_insert(0usize) += 1;
1337        }
1338        for scope in expired {
1339            let routes_closed = counts
1340                .get(&(
1341                    scope.owner.as_str(),
1342                    scope.scope_ref.as_str(),
1343                    scope.scope_epoch,
1344                ))
1345                .copied()
1346                .unwrap_or(0);
1347            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1348                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1349                routes_closed, "scope expired");
1350        }
1351        Ok(drained)
1352    }
1353
1354    fn runtime_capability_snapshot(
1355        &self,
1356    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1357        let runtime = self
1358            .supervisor
1359            .list()
1360            .into_iter()
1361            .map(|module| {
1362                let status = module.status().map_err(|err| {
1363                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1364                })?;
1365                Ok(RuntimeModule {
1366                    module_id: status.module_id,
1367                    state: status.state,
1368                    enabled: status.enabled,
1369                })
1370            })
1371            .collect::<Result<Vec<_>, RouterError>>()?;
1372        // Include every registration, published or not. The capability
1373        // evaluator counts a module as providing its capabilities as soon as it
1374        // has registered, even before `catalog.list` shows it, so a HELLO in
1375        // progress does not change other modules' required-capability
1376        // verdicts. See `Registry::list_registered_modules`.
1377        let registrations = self.registry.list_registered_modules().map_err(|err| {
1378            RouterError::backend(
1379                0,
1380                0,
1381                format!("failed to list capability registrations: {err}"),
1382            )
1383        })?;
1384        let registrations = registrations
1385            .into_iter()
1386            .map(|registration| RegisteredModule {
1387                module_id: registration.manifest.module_id,
1388                module_version: registration.manifest.module_version,
1389                capabilities: registration.manifest.capabilities,
1390            })
1391            .collect();
1392        Ok((runtime, registrations))
1393    }
1394
1395    /// The capability side effects of a module becoming the active registration
1396    /// for its id: cache its manifest (warning if its claims drifted), run the
1397    /// deny census when its declarations call for one, and recompute the
1398    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1399    /// candidate's does not, and the supervisor does it at promotion instead,
1400    /// through [`crate::supervise::SwapPromotionObserver`].
1401    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1402        let cached_registration = RegisteredModule {
1403            module_id: registration.manifest.module_id.clone(),
1404            module_version: registration.manifest.module_version.clone(),
1405            capabilities: registration.manifest.capabilities.clone(),
1406        };
1407        if self.capability_evaluator.record_hello(&cached_registration) {
1408            warn!(
1409                module_id = %cached_registration.module_id,
1410                "capability claims drifted from the cached manifest"
1411            );
1412        }
1413        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1414            self.enforce_capability_denies();
1415        }
1416        self.refresh_capability_requirements();
1417    }
1418
1419    /// Point the shared supervisor handle at this handler for swap promotions.
1420    /// Called wherever a handler is put behind the `Arc` the router serves, so
1421    /// it can be held weakly.
1422    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1423        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1424            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1425        self.supervisor.set_swap_promotion_observer(observer);
1426    }
1427
1428    pub fn refresh_capability_requirements(&self) {
1429        match self.runtime_capability_snapshot() {
1430            Ok((runtime, registrations)) => {
1431                log_requirement_events(
1432                    self.capability_evaluator
1433                        .evaluate_now(&runtime, &registrations),
1434                    &self.daemon_shutdown,
1435                );
1436            }
1437            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1438        }
1439    }
1440
1441    /// Reconcile only live, attested route bindings after a capability deny edge
1442    /// or target claim was added. This is deliberately a control-plane census:
1443    /// the opaque forwarding hot path must not grow a per-frame capability check.
1444    fn enforce_capability_denies(&self) {
1445        let registrations = match self.registry.list_registered_modules() {
1446            Ok(snapshot) => snapshot,
1447            Err(err) => {
1448                warn!(error = %err, "failed to read registrations for capability deny census");
1449                return;
1450            }
1451        };
1452        let manifests = registrations
1453            .into_iter()
1454            .map(|registration| {
1455                (
1456                    registration.manifest.module_id.clone(),
1457                    registration.manifest,
1458                )
1459            })
1460            .collect::<BTreeMap<_, _>>();
1461        let census = match self.forwarding.route_census(None) {
1462            Ok(census) => census,
1463            Err(err) => {
1464                warn!(error = %err, "failed to read route census for capability deny enforcement");
1465                return;
1466            }
1467        };
1468
1469        for (target_module_id, routes) in census {
1470            let Some(target_manifest) = manifests.get(&target_module_id) else {
1471                continue;
1472            };
1473            let mut closed_routes = Vec::new();
1474            let mut module_goodbyes = Vec::new();
1475            for route in routes {
1476                let Principal::Reserved {
1477                    module_id: opening_module_id,
1478                } = &route.principal
1479                else {
1480                    continue;
1481                };
1482                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1483                    continue;
1484                };
1485                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1486                    continue;
1487                };
1488
1489                match self.forwarding.release_client_route(
1490                    route.goodbye_target.connection_id,
1491                    route.goodbye_target.channel,
1492                    route.goodbye_target.epoch,
1493                ) {
1494                    Ok(RouteRelease::Removed(module_goodbye)) => {
1495                        warn!(
1496                            opening_module_id,
1497                            target_module_id,
1498                            capability,
1499                            "force-closing route because an attested capability deny edge now matches"
1500                        );
1501                        closed_routes.push(route);
1502                        module_goodbyes.push(module_goodbye);
1503                    }
1504                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1505                    Err(err) => warn!(
1506                        opening_module_id,
1507                        target_module_id,
1508                        capability,
1509                        error = %err,
1510                        "failed to force-close capability-denied route"
1511                    ),
1512                }
1513            }
1514
1515            if closed_routes.is_empty() {
1516                continue;
1517            }
1518            send_route_control_pushes(
1519                &self.forwarding,
1520                closed_routes,
1521                ClientControlPush::RouteClosed {
1522                    module_id: target_module_id,
1523                    channels: Vec::new(),
1524                    reason: RouteCloseReason::CapabilityDenied,
1525                    drained: false,
1526                    abandoned: 0,
1527                    excluded_subscriptions: 0,
1528                    terminal: Some(false),
1529                },
1530            );
1531            self.emit_route_goodbyes(module_goodbyes);
1532        }
1533    }
1534
1535    /// Why a registered module is not accepting new route binds, or `None` when
1536    /// it is. This is the module's effective readiness: its declared readiness
1537    /// first, then every `need: required` capability it declares evaluating to
1538    /// `provided`. `route.open` and `catalog.list` both read it here so the
1539    /// catalog never reports a module routable that `route.open` would refuse.
1540    fn not_ready_reason(
1541        &self,
1542        registration: &crate::registry::ModuleRegistration,
1543    ) -> Option<NotReadyReason> {
1544        if !registration.ready {
1545            return Some(NotReadyReason {
1546                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1547                capability: None,
1548            });
1549        }
1550        self.first_unprovided_required_capability(registration)
1551            .map(|capability| NotReadyReason {
1552                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1553                capability: Some(capability),
1554            })
1555    }
1556
1557    /// The lexicographically first capability this registration declares
1558    /// `need: required` whose evaluator verdict is not `provided`.
1559    ///
1560    /// The verdicts are the capability evaluator's own; nothing here decides
1561    /// what "provided" means. The evaluator counts a capability provided as
1562    /// soon as a module claiming it has REGISTERED, not once that module is
1563    /// ready. That distinction is what keeps two modules that require each
1564    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1565    /// is ready", each would wait for the other to become ready first and
1566    /// neither ever would. Do not tighten it to readiness.
1567    ///
1568    /// A required capability with no verdict at all means this registration's
1569    /// HELLO or catalog.update landed after the last recompute; recompute once
1570    /// rather than let a missing verdict read as either answer. If it is still
1571    /// missing (the recompute itself failed) the capability counts as
1572    /// unprovided: the refusal is retryable, and routing a module whose
1573    /// required provider is unknown is the outcome this check exists to stop.
1574    fn first_unprovided_required_capability(
1575        &self,
1576        registration: &crate::registry::ModuleRegistration,
1577    ) -> Option<String> {
1578        let required = registration
1579            .manifest
1580            .capabilities
1581            .iter()
1582            .flat_map(|declarations| declarations.requires.iter())
1583            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1584            .map(|requirement| requirement.capability.as_str())
1585            .collect::<BTreeSet<_>>();
1586        if required.is_empty() {
1587            return None;
1588        }
1589        let module_id = registration.manifest.module_id.as_str();
1590        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1591        if required
1592            .iter()
1593            .any(|capability| verdict(capability).is_none())
1594        {
1595            self.refresh_capability_requirements();
1596        }
1597        required
1598            .into_iter()
1599            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1600            .map(str::to_string)
1601    }
1602
1603    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1604        self.capability_evaluator
1605            .statuses()
1606            .into_iter()
1607            .map(capability_requirement_status)
1608            .collect()
1609    }
1610
1611    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1612    /// registration-release watch. The signal is what the supervisor waits on
1613    /// before spawning a replacement, so it must only fire once forwarding
1614    /// teardown is also done (see [`Self::cleanup_connection`] /
1615    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1616    /// admitted but its forwarding endpoint could not be installed.
1617    fn deregister_connection(
1618        &self,
1619        connection_id: ConnectionId,
1620        reason: RegistrationEndReason,
1621    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1622        self.registry
1623            .deregister_connection_with_reason(connection_id, reason)
1624    }
1625
1626    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1627        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1628            return None;
1629        }
1630        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1631            parse_client_control_request(&frame.body)
1632        else {
1633            return None;
1634        };
1635        Some(target_module_id(&target).to_string())
1636    }
1637
1638    pub(crate) fn route_open_capacity_refusal(
1639        &self,
1640        ctx: &RouteCtx,
1641        frame: &Frame,
1642        target_module_id: &str,
1643        in_flight: usize,
1644        limit: usize,
1645    ) -> Result<Frame, RouterError> {
1646        self.route_open_admission_refusal_frame(
1647            ctx,
1648            frame,
1649            target_module_id,
1650            "open_admission_full",
1651            (in_flight, limit),
1652            format!(
1653                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1654            ),
1655        )
1656    }
1657
1658    fn route_open_target_capacity_refusal(
1659        &self,
1660        ctx: &RouteCtx,
1661        frame: &Frame,
1662        target_module_id: &str,
1663        in_flight: usize,
1664    ) -> Result<Frame, RouterError> {
1665        self.route_open_admission_refusal_frame(
1666            ctx,
1667            frame,
1668            target_module_id,
1669            "target_binds_full",
1670            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1671            format!(
1672                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1673            ),
1674        )
1675    }
1676
1677    /// Admission pressure clears as existing binds settle, so its refusal must
1678    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1679    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1680    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1681    /// cannot currently reach its target; `module_timeout` would falsely claim
1682    /// that a wait expired. A new, cleaner code would be terminal to deployed
1683    /// clients, so it requires a client-tolerance rollout before daemon emission.
1684    fn route_open_admission_refusal_frame(
1685        &self,
1686        ctx: &RouteCtx,
1687        frame: &Frame,
1688        target_module_id: &str,
1689        reason: &'static str,
1690        (in_flight, limit): (usize, usize),
1691        message: impl Into<String>,
1692    ) -> Result<Frame, RouterError> {
1693        let code = error_codes::TARGET_UNAVAILABLE;
1694        self.counters.increment_route_open_refused(code);
1695        info!(
1696            target: "control",
1697            code,
1698            reason,
1699            module_id = ?target_module_id,
1700            connection_id = ctx.connection_id.get(),
1701            in_flight,
1702            limit,
1703            "route.open refused"
1704        );
1705        control_error_frame(frame, code, message.into())
1706    }
1707
1708    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1709    ///
1710    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1711    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1712    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1713    #[cfg(test)]
1714    pub fn handle_control(
1715        &self,
1716        connection_id: ConnectionId,
1717        frame: Frame,
1718    ) -> Result<Vec<Frame>, RouterError> {
1719        match frame.header.ty {
1720            FrameType::Ping => Ok(vec![pong(&frame)?]),
1721            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1722            FrameType::Goodbye => self.handle_goodbye(connection_id),
1723            ty => Ok(vec![control_error_frame(
1724                &frame,
1725                "unsupported_control_frame",
1726                format!("unsupported channel-0 frame {ty:?}"),
1727            )?]),
1728        }
1729    }
1730
1731    pub async fn handle_control_frame(
1732        &self,
1733        ctx: &RouteCtx,
1734        frame: Frame,
1735    ) -> Result<Vec<Frame>, RouterError> {
1736        self.handle_control_frame_timed(ctx, frame, None).await
1737    }
1738
1739    pub(crate) fn record_caller_endpoints(
1740        &self,
1741        connection: ConnectionId,
1742        endpoints: Option<(SocketAddr, SocketAddr)>,
1743    ) {
1744        let mut callers = self
1745            .caller_endpoints
1746            .lock()
1747            .unwrap_or_else(|p| p.into_inner());
1748        callers.remove(&connection);
1749        if let Some(endpoints) = endpoints {
1750            callers.insert(connection, endpoints);
1751        }
1752    }
1753
1754    #[cfg(test)]
1755    pub(crate) fn with_caller_lookup_for_test(
1756        mut self,
1757        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1758    ) -> Self {
1759        self.caller_lookup = Some(Arc::new(lookup));
1760        self
1761    }
1762
1763    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1764        let endpoints = self
1765            .caller_endpoints
1766            .lock()
1767            .unwrap_or_else(|p| p.into_inner())
1768            .get(&connection)
1769            .copied();
1770        #[cfg(test)]
1771        let lookup = self.caller_lookup.clone();
1772        tokio::task::spawn_blocking(move || {
1773            #[cfg(test)]
1774            if let Some(lookup) = lookup {
1775                return lookup(endpoints);
1776            }
1777            endpoints
1778                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1779                .unwrap_or_default()
1780        })
1781        .await
1782        .unwrap_or_default()
1783    }
1784
1785    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1786        frame.header.channel == 0
1787            && frame.header.ty == FrameType::Request
1788            && self
1789                .forwarding
1790                .module_endpoint_for_connection(ctx.connection_id)
1791                .is_ok_and(|endpoint| endpoint.is_none())
1792            && parse_client_control_request(&frame.body)
1793                .ok()
1794                .is_some_and(|request| privileged_operation(&request).is_some())
1795    }
1796
1797    pub(crate) async fn handle_control_frame_timed(
1798        &self,
1799        ctx: &RouteCtx,
1800        frame: Frame,
1801        dispatch_started_at: Option<StdInstant>,
1802    ) -> Result<Vec<Frame>, RouterError> {
1803        match frame.header.ty {
1804            FrameType::Ping => Ok(vec![pong(&frame)?]),
1805            FrameType::Hello => {
1806                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1807            }
1808            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1809            FrameType::Cancel => {
1810                // A Cancel on channel 0 names a waiting operator.confirm or a
1811                // subscription. Successful cancellation has no ordinary reply.
1812                if self
1813                    .forwarding
1814                    .operator_confirms()
1815                    .cancel(ctx.connection_id, frame.header.corr)
1816                    || self
1817                        .supervisor
1818                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1819                    || self.cancel_activity_subscription(ctx, &frame)?
1820                {
1821                    Ok(Vec::new())
1822                } else {
1823                    Ok(vec![control_error_frame(
1824                        &frame,
1825                        "unknown_subscription",
1826                        "no subscription has this correlation id",
1827                    )?])
1828                }
1829            }
1830            FrameType::Request => {
1831                // This additive operation is not part of the existing exhaustive
1832                // module-control enum. Probe the op before decoding that enum.
1833                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1834                if op
1835                    .as_ref()
1836                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1837                {
1838                    return self.handle_activity_record(ctx, frame).await;
1839                }
1840                if op.as_ref().is_some_and(|probe| {
1841                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1842                }) {
1843                    return self.handle_activity_read(ctx, frame).await;
1844                }
1845                if op
1846                    .as_ref()
1847                    .is_some_and(|probe| probe.op == "operator.confirm")
1848                {
1849                    return self.handle_operator_confirm(ctx, frame);
1850                }
1851                if op
1852                    .as_ref()
1853                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1854                {
1855                    return self.handle_scope_served_by(ctx.connection_id, frame);
1856                }
1857                if self
1858                    .forwarding
1859                    .module_endpoint_for_connection(ctx.connection_id)
1860                    .map_err(RouterError::Forwarding)?
1861                    .is_some()
1862                {
1863                    if !is_known_module_request_op(&frame.body) {
1864                        return Ok(vec![control_error_frame(
1865                            &frame,
1866                            "unsupported_control_frame",
1867                            "module-originated channel-0 REQUEST is not supported",
1868                        )?]);
1869                    }
1870                    let request = match parse_module_control_request_from_module(&frame.body) {
1871                        Ok(request) => request,
1872                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1873                            return Ok(vec![control_error_frame(
1874                                &frame,
1875                                "unsupported_control_frame",
1876                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1877                            )?])
1878                        }
1879                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1880                            return Ok(vec![control_error_frame(
1881                                &frame,
1882                                "invalid_control_body",
1883                                format!("malformed module control body: {err}"),
1884                            )?])
1885                        }
1886                    };
1887                    let op = module_control_request_op(&request);
1888                    let corr = frame.header.corr;
1889                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1890                    let result =
1891                        self.handle_module_control_request(ctx.connection_id, frame, request);
1892                    log_slow_control_dispatch(
1893                        dispatch_started_at,
1894                        op,
1895                        ctx.connection_id,
1896                        corr,
1897                        None,
1898                    );
1899                    return result;
1900                }
1901
1902                if is_known_module_request_op(&frame.body) {
1903                    return Ok(vec![control_error_frame(
1904                        &frame,
1905                        "not_registered",
1906                        "catalog.update requires an active module registration owned by this connection",
1907                    )?]);
1908                }
1909
1910                let request = match parse_client_control_request(&frame.body) {
1911                    Ok(request) => request,
1912                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1913                        return Ok(vec![control_error_frame(
1914                            &frame,
1915                            "unknown_control_op",
1916                            format!("unknown client control op: {err}"),
1917                        )?])
1918                    }
1919                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1920                        return Ok(vec![control_error_frame(
1921                            &frame,
1922                            "invalid_control_body",
1923                            format!("malformed client control body: {err}"),
1924                        )?])
1925                    }
1926                };
1927                let op = client_control_request_op(&request);
1928                let corr = frame.header.corr;
1929                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1930                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1931                #[cfg(test)]
1932                if let Some(delay) = self.control_dispatch_delay {
1933                    tokio::time::sleep(delay).await;
1934                }
1935                let result = self
1936                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1937                    .await;
1938                if let Some(timings) = route_open_timings.as_mut() {
1939                    timings.finish();
1940                }
1941                log_slow_control_dispatch(
1942                    dispatch_started_at,
1943                    op,
1944                    ctx.connection_id,
1945                    corr,
1946                    route_open_timings.as_ref(),
1947                );
1948                result
1949            }
1950            FrameType::Push => {
1951                let Some(endpoint) = self
1952                    .forwarding
1953                    .module_endpoint_for_connection(ctx.connection_id)
1954                    .map_err(RouterError::Forwarding)?
1955                else {
1956                    return Ok(vec![control_error_frame(
1957                        &frame,
1958                        "unsupported_control_frame",
1959                        "client-originated channel-0 PUSH is not supported",
1960                    )?]);
1961                };
1962                self.handle_status_update(endpoint, frame)
1963            }
1964            FrameType::Response | FrameType::Error
1965                if self
1966                    .forwarding
1967                    .module_endpoint_for_connection(ctx.connection_id)
1968                    .map_err(RouterError::Forwarding)?
1969                    .is_some() =>
1970            {
1971                self.handle_module_relay_response(ctx.connection_id, frame)
1972            }
1973            ty => Ok(vec![control_error_frame(
1974                &frame,
1975                "unsupported_control_frame",
1976                format!("unsupported channel-0 frame {ty:?}"),
1977            )?]),
1978        }
1979    }
1980
1981    pub fn cleanup_connection(
1982        &self,
1983        connection_id: ConnectionId,
1984    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1985        self.caller_endpoints
1986            .lock()
1987            .unwrap_or_else(|p| p.into_inner())
1988            .remove(&connection_id);
1989        self.forwarding
1990            .operator_confirms()
1991            .connection_closed(connection_id);
1992        self.cleanup_connection_with_end_reason(
1993            connection_id,
1994            RegistrationEndReason::ConnectionClosed,
1995        )
1996    }
1997
1998    fn cleanup_connection_with_end_reason(
1999        &self,
2000        connection_id: ConnectionId,
2001        requested_reason: RegistrationEndReason,
2002    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2003        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2004            self.registry
2005                .get_module_by_connection(connection_id)?
2006                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2007                .and_then(|module| module.registration_end_reason().ok().flatten())
2008                .unwrap_or(requested_reason)
2009        } else {
2010            requested_reason
2011        };
2012        let crash_closed = self
2013            .registry
2014            .get_module_by_connection(connection_id)?
2015            .and_then(|registration| {
2016                self.forwarding
2017                    .module_endpoint_for_connection(connection_id)
2018                    .ok()
2019                    .flatten()
2020                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2021                    .map(|routes| (registration.manifest.module_id, routes))
2022            });
2023        let crash_closed = crash_closed.map(|(module_id, routes)| {
2024            let terminal = match self.supervisor.get(&module_id) {
2025                None => false,
2026                Some(module) => match module.will_recover_after_connection_loss() {
2027                    Ok(will_recover) => !will_recover,
2028                    Err(err) => {
2029                        warn!(
2030                            %module_id,
2031                            error = %err,
2032                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2033                        );
2034                        false
2035                    }
2036                },
2037            };
2038            // The forwarding table gates all providers at the start of daemon
2039            // shutdown, before their connections are closed. An ordinary
2040            // module disconnect still reports crash if that gate is not set.
2041            let (reason, drained) = match self
2042                .forwarding
2043                .daemon_drain_result_for_connection(connection_id)
2044            {
2045                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2046                Ok(None) => (RouteCloseReason::Crash, false),
2047                Err(err) => {
2048                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2049                    (RouteCloseReason::Crash, false)
2050                }
2051            };
2052            (module_id, routes, reason, drained, terminal)
2053        });
2054        let registrations = self.deregister_connection(connection_id, end_reason);
2055        let cleanup = if crash_closed.is_some() {
2056            self.forwarding.cleanup_connection_counted(connection_id)
2057        } else {
2058            // A module connection's teardown needs the count of abandoned
2059            // route.bind relays for its route.closed notice below. Any other
2060            // connection, such as a client's, sends no such notice and needs
2061            // only its routes released, so it uses the route-only wrapper and
2062            // reports zero.
2063            self.forwarding
2064                .cleanup_connection(connection_id)
2065                .map(|released| crate::forwarding::ConnectionCleanup {
2066                    released,
2067                    abandoned_relays: 0,
2068                })
2069        };
2070        // The route.closed push waits for forwarding teardown because only
2071        // teardown knows how many pending route.bind relays it aborted. It still
2072        // goes out before the GOODBYEs for the released routes, and its targets
2073        // were captured above, before teardown removed those routes.
2074        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2075            let abandoned = cleanup
2076                .as_ref()
2077                .map_or(0, |cleanup| cleanup.abandoned_relays);
2078            send_route_control_pushes(
2079                &self.forwarding,
2080                routes,
2081                ClientControlPush::RouteClosed {
2082                    module_id,
2083                    channels: Vec::new(),
2084                    reason,
2085                    drained,
2086                    abandoned,
2087                    excluded_subscriptions: 0,
2088                    terminal: Some(terminal),
2089                },
2090            );
2091        }
2092        if let Ok(cleanup) = cleanup {
2093            self.emit_route_goodbyes(cleanup.released);
2094        }
2095        // Signal the registration-release watch only now that BOTH registry and
2096        // forwarding teardown are done, so a supervisor waiting to spawn a
2097        // replacement never observes release while old routes still exist.
2098        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2099            crate::supervise::notify_registration_release();
2100            self.capability_evaluator.wake_deadline_loop();
2101            self.refresh_capability_requirements();
2102        }
2103        self.supervisor.remove_spawn_subscribers(connection_id);
2104        self.activity_streams
2105            .lock()
2106            .unwrap_or_else(|p| p.into_inner())
2107            .retain(|(id, _), task| {
2108                if *id == connection_id {
2109                    task.abort();
2110                    false
2111                } else {
2112                    true
2113                }
2114            });
2115        // Sync authority dies with its connection, so the owner's next
2116        // connection can take it; the owner's scopes stay as they are.
2117        self.hello_launch_nonces
2118            .lock()
2119            .unwrap_or_else(|poisoned| poisoned.into_inner())
2120            .forget(connection_id);
2121        self.scopes
2122            .write()
2123            .unwrap_or_else(|poisoned| poisoned.into_inner())
2124            .release_connection(connection_id);
2125        registrations
2126    }
2127
2128    pub(crate) fn handle_route_goodbye(
2129        &self,
2130        connection_id: ConnectionId,
2131        route_channel: u16,
2132        route_epoch: u32,
2133    ) -> Result<bool, RouterError> {
2134        debug!(
2135            connection_id = connection_id.get(),
2136            route_channel, route_epoch, "handling route GOODBYE"
2137        );
2138        let RouteRelease::Removed(released_route) = self
2139            .forwarding
2140            .release_client_route(connection_id, route_channel, route_epoch)
2141            .map_err(RouterError::Forwarding)?
2142        else {
2143            return Ok(false);
2144        };
2145        self.emit_route_goodbyes(vec![released_route]);
2146        Ok(true)
2147    }
2148
2149    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2150        for released in released_routes {
2151            let frame = match Frame::build_with_version(
2152                released.negotiated_ver,
2153                FrameType::Goodbye,
2154                control_flags(),
2155                released.channel,
2156                released.epoch,
2157                0,
2158                Vec::new(),
2159            ) {
2160                Ok(frame) => frame,
2161                Err(err) => {
2162                    warn!(
2163                        route_channel = released.channel,
2164                        error = %err,
2165                        "failed to build route GOODBYE frame"
2166                    );
2167                    continue;
2168                }
2169            };
2170            if !released.close_on_delivery_failure() {
2171                crate::forwarding::send_module_route_goodbye(
2172                    &self.counters,
2173                    &released.sink,
2174                    frame,
2175                    released.module_id.as_deref(),
2176                    "client route released",
2177                );
2178                continue;
2179            }
2180            if let Err(err) = released.sink.try_send(frame) {
2181                warn!(
2182                    target_connection_id = released.connection_id.get(),
2183                    route_channel = released.channel,
2184                    error = %err,
2185                    "route GOODBYE was not delivered to client; closing target connection"
2186                );
2187                if self
2188                    .forwarding
2189                    .escalate_client_delivery_failure(
2190                        released.connection_id,
2191                        released.channel,
2192                        released.epoch,
2193                        CloseReason::new(
2194                            "route_goodbye_delivery_failed",
2195                            format!(
2196                                "failed to enqueue route GOODBYE for channel {}: {err}",
2197                                released.channel
2198                            ),
2199                        ),
2200                        crate::forwarding::UndeliveredFrame {
2201                            module_id: released.module_id.as_deref(),
2202                            sink: &released.sink,
2203                        },
2204                    )
2205                    .unwrap_or(false)
2206                {
2207                    self.counters.increment_goodbye_relay_client_failed();
2208                }
2209            }
2210        }
2211    }
2212
2213    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2214    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2215    /// own commit failed after the module had already accepted). Without this, a
2216    /// module that accepts late keeps a binding subc has torn down, so a later
2217    /// frame on that module channel could misdeliver if the channel is reused.
2218    ///
2219    /// Never closes the shared module connection on failure: a dropped notification
2220    /// only wastes a bounded amount of warm module-side state, which the module's
2221    /// own idle reaper reclaims. Only call this once the route.bind relay was
2222    /// actually enqueued to the module — if the relay send itself failed, the
2223    /// module never created a binding and there is nothing to tear down.
2224    fn send_abandoned_route_bind_goodbye(
2225        &self,
2226        module_sink: &crate::FrameSink,
2227        negotiated_ver: u8,
2228        module_channel: u16,
2229        module_epoch: u32,
2230    ) {
2231        let frame = match Frame::build_with_version(
2232            negotiated_ver,
2233            FrameType::Goodbye,
2234            control_flags(),
2235            module_channel,
2236            module_epoch,
2237            0,
2238            Vec::new(),
2239        ) {
2240            Ok(frame) => frame,
2241            Err(err) => {
2242                warn!(
2243                    route_channel = module_channel,
2244                    error = %err,
2245                    "failed to build GOODBYE for abandoned route.bind"
2246                );
2247                return;
2248            }
2249        };
2250        crate::forwarding::send_module_route_goodbye(
2251            &self.counters,
2252            module_sink,
2253            frame,
2254            None,
2255            "abandoned route.bind",
2256        );
2257    }
2258
2259    fn handle_hello(
2260        &self,
2261        connection_id: ConnectionId,
2262        sink: Option<crate::FrameSink>,
2263        frame: Frame,
2264    ) -> Result<Vec<Frame>, RouterError> {
2265        debug!(
2266            connection_id = connection_id.get(),
2267            corr = frame.header.corr,
2268            "handling HELLO"
2269        );
2270        // A module connection has one identity for its entire lifetime. A second
2271        // registration would leave the old registry owner behind while replacing
2272        // its forwarding endpoint and launch nonce.
2273        if self
2274            .registry
2275            .get_module_by_connection(connection_id)
2276            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2277            .is_some()
2278        {
2279            return Ok(vec![control_error_frame(
2280                &frame,
2281                "invalid_hello",
2282                "connection is already registered as a module",
2283            )?]);
2284        }
2285        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2286            Ok(value) => value,
2287            Err(err) => {
2288                return Ok(vec![control_error_frame(
2289                    &frame,
2290                    "invalid_hello",
2291                    format!("malformed HELLO body: {err}"),
2292                )?])
2293            }
2294        };
2295        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2296            return Ok(vec![control_error_frame(
2297                &frame,
2298                "invalid_capability_grammar",
2299                err.to_string(),
2300            )?]);
2301        }
2302        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2303            return Ok(vec![control_error_frame(
2304                &frame,
2305                "invalid_manifest",
2306                err.to_string(),
2307            )?]);
2308        }
2309        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2310            return Ok(vec![control_error_frame(
2311                &frame,
2312                "invalid_event_declaration",
2313                err.to_string(),
2314            )?]);
2315        }
2316        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2317            return Ok(vec![control_error_frame(
2318                &frame,
2319                "invalid_manifest",
2320                err.to_string(),
2321            )?]);
2322        }
2323        if let Some(provenance) = hello_value
2324            .get("manifest")
2325            .and_then(|manifest| manifest.get("provenance"))
2326        {
2327            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2328                return Ok(vec![control_error_frame(
2329                    &frame,
2330                    "invalid_manifest",
2331                    format!("malformed manifest provenance: {err}"),
2332                )?]);
2333            }
2334        }
2335        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2336            Ok(hello) => hello,
2337            Err(err) => {
2338                return Ok(vec![control_error_frame(
2339                    &frame,
2340                    "invalid_hello",
2341                    format!("malformed HELLO body: {err}"),
2342                )?])
2343            }
2344        };
2345
2346        if hello.protocol_ver != hello.manifest.protocol_ver {
2347            return Ok(vec![control_error_frame(
2348                &frame,
2349                "invalid_manifest",
2350                format!(
2351                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2352                    hello.protocol_ver, hello.manifest.protocol_ver
2353                ),
2354            )?]);
2355        }
2356
2357        if hello.manifest.module_id.trim().is_empty() {
2358            return Ok(vec![control_error_frame(
2359                &frame,
2360                "invalid_manifest",
2361                "manifest module_id must not be empty",
2362            )?]);
2363        }
2364
2365        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2366            Ok(negotiated_ver) => negotiated_ver,
2367            Err(message) => {
2368                return Ok(vec![control_error_frame(
2369                    &frame,
2370                    "version_unsupported",
2371                    message,
2372                )?])
2373            }
2374        };
2375
2376        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2377        // swap is open for this id, the only HELLO admitted as a second process
2378        // is the one carrying the candidate's launch nonce (the swap token), and
2379        // it registers into the candidate slot rather than being refused as a
2380        // duplicate. Run after the reserved gate, a reserved module's candidate
2381        // would be refused `reserved_module` for presenting a nonce that gate
2382        // does not know. See `SupervisorHandle::swap_hello_admission`.
2383        let swap_admission = self
2384            .supervisor
2385            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2386        if swap_admission == SwapHelloAdmission::Refused {
2387            warn!(
2388                module_id = %hello.manifest.module_id,
2389                connection_id = connection_id.get(),
2390                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2391            );
2392            return Ok(vec![control_error_frame(
2393                &frame,
2394                "swap_token_invalid",
2395                format!(
2396                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2397                    hello.manifest.module_id
2398                ),
2399            )?]);
2400        }
2401        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2402
2403        // A swap candidate proves its identity with the candidate nonce above.
2404        // Other registrations receive this catalog fact only when reserved-module
2405        // policy required and accepted a nonce; unreserved registrations require
2406        // no nonce check.
2407        let nonce_verified = if swap_candidate {
2408            true
2409        } else {
2410            match self.supervisor.reserved_hello_verification(
2411                &hello.manifest.module_id,
2412                hello.launch_nonce.as_deref(),
2413            ) {
2414                Ok(verified) => verified,
2415                Err(rejection) => {
2416                    let message = match rejection {
2417                        ReservedHelloRejection::Exact { module_id } => format!(
2418                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2419                        ),
2420                        ReservedHelloRejection::Prefix {
2421                            prefix,
2422                            owner_module_id,
2423                        } => format!(
2424                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2425                            hello.manifest.module_id
2426                        ),
2427                    };
2428                    return Ok(vec![control_error_frame(
2429                        &frame,
2430                        "reserved_module",
2431                        message,
2432                    )?]);
2433                }
2434            }
2435        };
2436
2437        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2438            &hello.manifest.module_id,
2439            hello.manifest.capabilities.as_ref(),
2440        );
2441        if let Some(refusal) = reserved_capability_refusals.first() {
2442            let capability = refusal.capability.clone();
2443            let bound_module = refusal.claimants[0].clone();
2444            log_duplicate_claim_events(reserved_capability_refusals);
2445            return Ok(vec![control_error_frame(
2446                &frame,
2447                "reserved_capability",
2448                format!(
2449                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2450                    capability, bound_module, hello.manifest.module_id
2451                ),
2452            )?]);
2453        }
2454
2455        // A connection that already opened client routes must not also register as
2456        // a module: cleanup would then release only one side and leak the other.
2457        if self
2458            .forwarding
2459            .connection_has_client_routes(connection_id)
2460            .map_err(RouterError::Forwarding)?
2461        {
2462            return Ok(vec![control_error_frame(
2463                &frame,
2464                "invalid_hello",
2465                "connection has open client routes and cannot also register as a module",
2466            )?]);
2467        }
2468
2469        // Kept for scope sync authority, which goes only to the connection that
2470        // presented the module's current launch nonce. Recorded before the
2471        // registration is attempted: a connection whose registration then fails
2472        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2473        self.hello_launch_nonces
2474            .lock()
2475            .unwrap_or_else(|poisoned| poisoned.into_inner())
2476            .record(connection_id, hello.launch_nonce.as_deref());
2477        let control_ops = effective_module_control_ops(hello.control_ops);
2478        // Built before anything is registered so an encoding failure leaves no
2479        // registry or forwarding state behind.
2480        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2481        if swap_candidate {
2482            return self.register_swap_candidate(
2483                connection_id,
2484                sink,
2485                &frame,
2486                hello.manifest,
2487                negotiated_ver,
2488                control_ops,
2489                hello_ack,
2490            );
2491        }
2492        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2493        // is refused, but `catalog.list` does not show the module and the
2494        // catalog generation does not move until it is published below, once
2495        // its forwarding endpoint exists. Listing it now would advertise a
2496        // module that `route.open` still refuses for having no forwarding
2497        // connection.
2498        let registration = match self
2499            .registry
2500            .register_unpublished_verified_with_control_ops(
2501                hello.manifest,
2502                negotiated_ver,
2503                connection_id,
2504                control_ops,
2505                nonce_verified,
2506            ) {
2507            Ok(registration) => registration,
2508            Err(RegistryError::DuplicateModuleId { module_id }) => {
2509                return Ok(vec![control_error_frame(
2510                    &frame,
2511                    "duplicate_module_id",
2512                    format!(
2513                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2514                    ),
2515                )?])
2516            }
2517            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2518                return Ok(vec![control_error_frame(
2519                    &frame,
2520                    "invalid_module_id",
2521                    err.to_string(),
2522                )?])
2523            }
2524            Err(err) => {
2525                return Ok(vec![control_error_frame(
2526                    &frame,
2527                    "registry_error",
2528                    err.to_string(),
2529                )?])
2530            }
2531        };
2532        #[cfg(test)]
2533        self.hold_between_registry_and_forwarding();
2534
2535        let reply = if let Some(sink) = sink {
2536            // The forwarding table's module store is also the daemon-to-module
2537            // control-RPC lane, so every HELLO gets a live endpoint even when the
2538            // manifest has no routable provider role. Non-routable modules still
2539            // cannot receive route.bind in production: `handle_route_open` checks
2540            // the registry manifest with `target_has_required_role` before the
2541            // only production call to `begin_route_bind_relay_for` below that
2542            // route.open path. The remaining direct relay callers are unit tests
2543            // and benchmark harnesses that construct forwarding state explicitly.
2544            //
2545            // The HELLO_ACK is queued by the forwarding table itself, before the
2546            // endpoint becomes visible, and is NOT returned as a reply. A module
2547            // reads HELLO_ACK first and exits on anything else; a reply is only
2548            // written after this handler returns, by which time a route.open on
2549            // another connection could already have queued a route.bind request
2550            // for this module ahead of it.
2551            let concurrency = manifest_concurrency(&registration.manifest);
2552            if let Err(err) = self.forwarding.register_module_connection_acked(
2553                connection_id,
2554                registration.manifest.module_id.clone(),
2555                negotiated_ver,
2556                concurrency,
2557                sink,
2558                hello_ack,
2559            ) {
2560                // Forwarding registration failed, so there is no forwarding
2561                // state to tear down. Remove the registry entry, which was never
2562                // published and so never listed or counted in the catalog
2563                // generation, and signal the release watch directly.
2564                if matches!(
2565                    self.deregister_connection(
2566                        connection_id,
2567                        RegistrationEndReason::RegistrationFailed,
2568                    ),
2569                    Ok(r) if !r.is_empty()
2570                ) {
2571                    crate::supervise::notify_registration_release();
2572                }
2573                return Ok(vec![control_error_frame(
2574                    &frame,
2575                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2576                        "invalid_hello"
2577                    } else {
2578                        forwarding_error_code(&err)
2579                    },
2580                    err.to_string(),
2581                )?]);
2582            }
2583            Vec::new()
2584        } else {
2585            // No sink means no forwarding endpoint, so nothing can be routed
2586            // ahead of the ack; it goes out as the reply.
2587            vec![hello_ack]
2588        };
2589
2590        // The module is routable now (or, without a sink, as routable as it
2591        // will ever be), so the catalog may list it. Publishing bumps the
2592        // catalog generation in the same registry critical section that makes
2593        // the module listed.
2594        if !self
2595            .registry
2596            .publish_registration(connection_id)
2597            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2598        {
2599            debug!(
2600                module_id = %registration.manifest.module_id,
2601                connection_id = connection_id.get(),
2602                "registration was removed before it could be published"
2603            );
2604        }
2605
2606        // Exposure over assumption: Concurrency's serde default is pinned to the
2607        // pre-field behavior (ModuleManaged), so a management surface that is
2608        // genuinely Serial and just never declared it inherits concurrent
2609        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2610        // turns "no module has been bitten yet" into the checkable claim "no
2611        // module is exposed" -- one read of the boot log instead of a fleet
2612        // audit. Detected from the raw HELLO bytes because the serde default
2613        // deliberately erases the absent/declared distinction from the type.
2614        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2615            info!(
2616                module_id = %registration.manifest.module_id,
2617                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2618            );
2619        }
2620
2621        self.apply_registration_capabilities(&registration);
2622
2623        self.supervisor
2624            .record_activity_registration(&registration.manifest.module_id);
2625
2626        info!(
2627            module_id = %registration.manifest.module_id,
2628            module_version = %registration.manifest.module_version,
2629            negotiated_ver,
2630            routable_provider = manifest_provides_routable_role(&registration.manifest),
2631            connection_id = connection_id.get(),
2632            "module registered"
2633        );
2634
2635        Ok(reply)
2636    }
2637
2638    /// Register a HELLO the swap gate admitted into the candidate slot of the
2639    /// registry and of forwarding, where it is reachable over its own
2640    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2641    /// nothing routes to it until the supervisor cuts over.
2642    ///
2643    /// Registry first (unpublished), then forwarding, then publication, the
2644    /// same order as an ordinary HELLO; a forwarding failure removes the
2645    /// registry entry again. The swap supervisor treats a published, ready
2646    /// candidate as registered and promotes it, and promotion needs the
2647    /// candidate's forwarding endpoint, so the candidate is published only
2648    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2649    /// recompute capability verdicts or run the capability deny census
2650    /// (`enforce_capability_denies`): both are about the module in the active
2651    /// slot, which `route.open` can target. A candidate sits in the candidate
2652    /// slot, so it neither provides capabilities to other modules nor can be
2653    /// routed to until promotion moves it into the active slot.
2654    #[allow(clippy::too_many_arguments)]
2655    fn register_swap_candidate(
2656        &self,
2657        connection_id: ConnectionId,
2658        sink: Option<crate::FrameSink>,
2659        frame: &Frame,
2660        manifest: ModuleManifest,
2661        negotiated_ver: u8,
2662        control_ops: Vec<String>,
2663        hello_ack: Frame,
2664    ) -> Result<Vec<Frame>, RouterError> {
2665        let module_id = manifest.module_id.clone();
2666        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2667            manifest,
2668            negotiated_ver,
2669            connection_id,
2670            control_ops,
2671        ) {
2672            Ok(registration) => registration,
2673            Err(RegistryError::DuplicateModuleId { module_id }) => {
2674                return Ok(vec![control_error_frame(
2675                    frame,
2676                    "duplicate_module_id",
2677                    format!(
2678                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2679                    ),
2680                )?])
2681            }
2682            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2683                return Ok(vec![control_error_frame(
2684                    frame,
2685                    "invalid_module_id",
2686                    err.to_string(),
2687                )?])
2688            }
2689            Err(err) => {
2690                return Ok(vec![control_error_frame(
2691                    frame,
2692                    "registry_error",
2693                    err.to_string(),
2694                )?])
2695            }
2696        };
2697        #[cfg(test)]
2698        self.hold_between_registry_and_forwarding();
2699        let reply = if let Some(sink) = sink {
2700            // Same ordering as an ordinary HELLO: the forwarding table queues
2701            // the HELLO_ACK before the candidate endpoint is inserted, because
2702            // a module exits if its first frame after HELLO is anything else.
2703            let concurrency = manifest_concurrency(&registration.manifest);
2704            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2705                connection_id,
2706                module_id.clone(),
2707                negotiated_ver,
2708                concurrency,
2709                sink,
2710                hello_ack,
2711            ) {
2712                if matches!(
2713                    self.deregister_connection(
2714                        connection_id,
2715                        RegistrationEndReason::RegistrationFailed,
2716                    ),
2717                    Ok(r) if !r.is_empty()
2718                ) {
2719                    crate::supervise::notify_registration_release();
2720                }
2721                return Ok(vec![control_error_frame(
2722                    frame,
2723                    forwarding_error_code(&err),
2724                    err.to_string(),
2725                )?]);
2726            }
2727            Vec::new()
2728        } else {
2729            vec![hello_ack]
2730        };
2731        if !self
2732            .registry
2733            .publish_registration(connection_id)
2734            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2735        {
2736            debug!(
2737                module_id = %module_id,
2738                connection_id = connection_id.get(),
2739                "swap candidate registration was removed before it could be published"
2740            );
2741        }
2742        self.supervisor.mark_swap_candidate_admitted(&module_id);
2743        info!(
2744            module_id = %module_id,
2745            module_version = %registration.manifest.module_version,
2746            negotiated_ver,
2747            ready = registration.ready,
2748            connection_id = connection_id.get(),
2749            "swap candidate registered; not routable until cutover"
2750        );
2751        Ok(reply)
2752    }
2753
2754    fn build_hello_ack(
2755        &self,
2756        frame: &Frame,
2757        negotiated_ver: u8,
2758        module_id: &str,
2759    ) -> Result<Frame, RouterError> {
2760        let ack = ModuleHelloAckBody {
2761            negotiated_ver,
2762            subc_ops: module_subc_ops(),
2763            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2764            storage: self
2765                .storage_config
2766                .as_ref()
2767                .map(|cfg| cfg.descriptor_for(module_id)),
2768            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2769        };
2770        let body = serde_json::to_vec(&ack).map_err(|err| {
2771            RouterError::backend(
2772                0,
2773                frame.header.corr,
2774                format!("failed to encode HELLO_ACK: {err}"),
2775            )
2776        })?;
2777
2778        Frame::build_with_version(
2779            negotiated_ver,
2780            FrameType::HelloAck,
2781            control_flags(),
2782            0,
2783            0,
2784            frame.header.corr,
2785            body,
2786        )
2787        .map_err(RouterError::FrameBuild)
2788    }
2789
2790    async fn handle_client_control_request(
2791        &self,
2792        ctx: &RouteCtx,
2793        frame: Frame,
2794        request: ClientControlRequest,
2795        route_open_timings: &mut Option<RouteOpenTimings>,
2796    ) -> Result<Vec<Frame>, RouterError> {
2797        if let Some(operation) = privileged_operation(&request) {
2798            let caller = self.resolve_caller(ctx.connection_id).await;
2799            let op = client_control_request_op(&request);
2800            let text = prompt_text(operation, &caller);
2801            match self.privileged_operator_prompt {
2802                PrivilegedOperatorPrompt::Log => {
2803                    info!(target: "subc_daemon::privileged_prompt", operation = op,
2804                        target = ?operation, caller = %caller.description(), summary = %text,
2805                        "privileged request would prompt the operator");
2806                }
2807                PrivilegedOperatorPrompt::Enforce => {
2808                    let key =
2809                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2810                    let outcome = self
2811                        .forwarding
2812                        .operator_confirms()
2813                        .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2814                        .await;
2815                    if outcome.code != "confirmed" {
2816                        return Ok(vec![outcome.frame(&frame)]);
2817                    }
2818                }
2819            }
2820        }
2821        match request {
2822            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2823            ClientControlRequest::CatalogList { module_id } => {
2824                self.handle_catalog_list(frame, module_id)
2825            }
2826            ClientControlRequest::RouteOpen {
2827                target,
2828                identity,
2829                consumer_identity,
2830                consumer_capabilities,
2831                role_versions,
2832                admission_facts,
2833                scope,
2834            } => {
2835                self.handle_route_open(
2836                    ctx,
2837                    frame,
2838                    RouteOpenRequest {
2839                        target,
2840                        identity,
2841                        consumer_identity,
2842                        consumer_capabilities,
2843                        role_versions,
2844                        admission_facts,
2845                        scope,
2846                    },
2847                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2848                )
2849                .await
2850            }
2851            ClientControlRequest::RoutePoll {
2852                route_channel,
2853                route_epoch,
2854                kind,
2855            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2856            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2857            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2858                self.handle_supervisor_spawn_snapshot(frame)
2859            }
2860            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2861                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2862            }
2863            ClientControlRequest::SupervisorRestart {
2864                module_id,
2865                drain_timeout_ms,
2866            } => {
2867                self.log_supervisor_request_received(
2868                    ctx,
2869                    frame.header.corr,
2870                    ops::SUPERVISOR_RESTART,
2871                    Some(&module_id),
2872                    None,
2873                )?;
2874                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2875                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2876                    .await
2877            }
2878            ClientControlRequest::SupervisorSwap {
2879                module_id,
2880                ready_timeout_ms,
2881            } => {
2882                self.log_supervisor_request_received(
2883                    ctx,
2884                    frame.header.corr,
2885                    ops::SUPERVISOR_SWAP,
2886                    Some(&module_id),
2887                    None,
2888                )?;
2889                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2890                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2891                    .await
2892            }
2893            ClientControlRequest::SupervisorReload { module_id } => {
2894                self.log_supervisor_request_received(
2895                    ctx,
2896                    frame.header.corr,
2897                    ops::SUPERVISOR_RELOAD,
2898                    Some(&module_id),
2899                    None,
2900                )?;
2901                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2902                self.handle_supervisor_reload(frame, module_id, requester)
2903                    .await
2904            }
2905            ClientControlRequest::SupervisorRescan { preview } => {
2906                if !preview {
2907                    self.log_supervisor_request_received(
2908                        ctx,
2909                        frame.header.corr,
2910                        ops::SUPERVISOR_RESCAN,
2911                        None,
2912                        None,
2913                    )?;
2914                }
2915                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2916                self.handle_supervisor_rescan(
2917                    frame,
2918                    if preview {
2919                        RescanMode::Preview
2920                    } else {
2921                        RescanMode::Execute
2922                    },
2923                    requester,
2924                )
2925                .await
2926            }
2927            ClientControlRequest::SupervisorRescanChecked {
2928                preview,
2929                candidate,
2930                expect,
2931            } => {
2932                self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2933                    .await
2934            }
2935            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2936                self.log_supervisor_request_received(
2937                    ctx,
2938                    frame.header.corr,
2939                    ops::SUPERVISOR_RELEASE_RESERVED,
2940                    Some(&module_id),
2941                    None,
2942                )?;
2943                self.handle_supervisor_release_reserved(frame, module_id)
2944                    .await
2945            }
2946            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2947                self.log_supervisor_request_received(
2948                    ctx,
2949                    frame.header.corr,
2950                    ops::SUPERVISOR_SET_ENABLED,
2951                    Some(&module_id),
2952                    Some(enabled),
2953                )?;
2954                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2955                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2956                    .await
2957            }
2958            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2959                self.handle_supervisor_health_probe(frame, module_id).await
2960            }
2961            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2962            ClientControlRequest::SupervisorRoutes { module_id } => {
2963                self.handle_supervisor_routes(frame, module_id)
2964            }
2965            ClientControlRequest::SupervisorProvenance { module_id } => {
2966                self.handle_supervisor_provenance(frame, module_id).await
2967            }
2968            ClientControlRequest::SupervisorStderrTail {
2969                module_id,
2970                max_lines,
2971                max_bytes,
2972            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2973            ClientControlRequest::SupervisorTerminals { module_id } => {
2974                self.handle_supervisor_terminals(frame, module_id).await
2975            }
2976        }
2977    }
2978
2979    fn log_supervisor_request_received(
2980        &self,
2981        ctx: &RouteCtx,
2982        corr: u64,
2983        op: &'static str,
2984        module_id: Option<&str>,
2985        enabled: Option<bool>,
2986    ) -> Result<(), RouterError> {
2987        let caller = self.supervisor_requester(ctx, corr)?;
2988        let caller = principal_label(&caller);
2989        // `Option` fields are recorded only when present, so a request with no
2990        // module id (rescan) or no enabled flag simply omits that field.
2991        info!(
2992            target: "control",
2993            op,
2994            module_id,
2995            enabled,
2996            connection_id = ctx.connection_id.get(),
2997            caller = %caller,
2998            "supervisor request received"
2999        );
3000        Ok(())
3001    }
3002
3003    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3004        Ok(self
3005            .registry
3006            .get_module_by_connection(ctx.connection_id)
3007            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3008            .map(|registration| Principal::Reserved {
3009                module_id: registration.manifest.module_id,
3010            })
3011            .unwrap_or(Principal::Direct))
3012    }
3013
3014    fn handle_module_control_request(
3015        &self,
3016        connection_id: ConnectionId,
3017        frame: Frame,
3018        request: ModuleControlRequestFromModule,
3019    ) -> Result<Vec<Frame>, RouterError> {
3020        match request {
3021            ModuleControlRequestFromModule::CatalogUpdate {
3022                provides,
3023                capabilities,
3024                ready,
3025            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3026            ModuleControlRequestFromModule::LiveRoots {} => {
3027                let registered = self
3028                    .registry
3029                    .get_module_by_connection(connection_id)
3030                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3031                let Some(registration) = registered else {
3032                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3033                };
3034                let response = self
3035                    .forwarding
3036                    .live_roots(&registration.manifest.module_id)
3037                    .map_err(RouterError::Forwarding)?;
3038                Ok(vec![control_response_body_frame(
3039                    &frame,
3040                    &response,
3041                    "ModuleControlResponseToModule::LiveRoots",
3042                )?])
3043            }
3044            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3045                self.handle_scope_sync(connection_id, frame, generation, scopes)
3046            }
3047            ModuleControlRequestFromModule::ScopeApply {
3048                generation,
3049                upsert,
3050                end,
3051            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3052            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3053                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3054            }
3055        }
3056    }
3057
3058    fn handle_scope_served_by(
3059        &self,
3060        connection_id: ConnectionId,
3061        frame: Frame,
3062    ) -> Result<Vec<Frame>, RouterError> {
3063        let registered = self
3064            .registry
3065            .get_module_by_connection(connection_id)
3066            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3067        let Some(registered) = registered else {
3068            return Ok(vec![control_error_frame(
3069                &frame,
3070                "not_registered",
3071                "scope.served_by requires an active module registration owned by this connection",
3072            )?]);
3073        };
3074        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3075        {
3076            Ok(request) => request,
3077            Err(err) => {
3078                return Ok(vec![control_error_frame(
3079                    &frame,
3080                    "invalid_control_body",
3081                    format!("malformed scope.served_by body: {err}"),
3082                )?])
3083            }
3084        };
3085        let answer = self
3086            .scopes
3087            .read()
3088            .unwrap_or_else(|poisoned| poisoned.into_inner())
3089            .served_by(
3090                &registered.manifest.module_id,
3091                &request.scope_ref,
3092                request.scope_epoch,
3093                &request.module_id,
3094            );
3095        let reply = subc_control::ScopeServedByReply::new(
3096            answer,
3097            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3098        );
3099        Ok(vec![control_response_body_frame(
3100            &frame,
3101            &reply,
3102            "ScopeServedByReply",
3103        )?])
3104    }
3105
3106    fn handle_operator_confirm(
3107        &self,
3108        ctx: &RouteCtx,
3109        frame: Frame,
3110    ) -> Result<Vec<Frame>, RouterError> {
3111        use crate::operator_confirm::{audit, Outcome};
3112        let registration = self
3113            .registry
3114            .get_module_by_connection(ctx.connection_id)
3115            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3116        let Some(registration) = registration else {
3117            let outcome = Outcome::refusal("not_registered");
3118            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3119            return Ok(vec![outcome.frame(&frame)]);
3120        };
3121        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3122            Ok(request) => request,
3123            Err(_) => {
3124                let outcome = Outcome::refusal("invalid_control_body");
3125                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3126                return Ok(vec![outcome.frame(&frame)]);
3127            }
3128        };
3129        let module_id = registration.manifest.module_id;
3130        // Read the launch nonce (under its own lock) before taking the forwarding
3131        // table's lock below: holding forwarding while waiting on another daemon
3132        // lock risks a lock-order deadlock with paths that take them the other way.
3133        let nonce = self
3134            .hello_launch_nonces
3135            .lock()
3136            .unwrap_or_else(|p| p.into_inner())
3137            .nonce(ctx.connection_id)
3138            .map(str::to_owned);
3139        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3140            self.supervisor
3141                .spawned_consumer_authorized(&module_id, nonce)
3142        });
3143        let confirms = self.forwarding.operator_confirms();
3144        self.forwarding
3145            .with_operator_route(
3146                ctx.connection_id,
3147                request.route_channel,
3148                request.route_epoch,
3149                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3150            )
3151            .map_err(RouterError::Forwarding)
3152    }
3153
3154    /// Resolve folders before taking the scope-table lock: a slow filesystem
3155    /// must not hold up admissions for unrelated scoped routes.
3156    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3157        let mut scopes = ScopeRecords::from(scopes);
3158        for (index, record) in scopes.records.iter_mut().enumerate() {
3159            if let Some(refusal) = Self::canonicalize_workdir(record) {
3160                scopes.refusals.insert(index, refusal);
3161            }
3162        }
3163        scopes
3164    }
3165
3166    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3167        let workdir = record.workdir.as_mut()?;
3168        let path = std::path::Path::new(workdir);
3169        if !path.is_absolute() {
3170            return Some((
3171                error_codes::INVALID_WORKDIR,
3172                "workdir must be absolute".to_string(),
3173            ));
3174        }
3175        match ProjectRootId::from_path_allowing_missing(path) {
3176            Ok(root) => match root.as_path().to_str() {
3177                Some(canonical) => {
3178                    *workdir = canonical.to_string();
3179                    None
3180                }
3181                None => Some((
3182                    error_codes::INVALID_WORKDIR,
3183                    "canonical workdir is not UTF-8".to_string(),
3184                )),
3185            },
3186            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3187        }
3188    }
3189
3190    /// `scope.sync`: the owner is the module registered on this connection.
3191    /// A connection with no registration (every client connection, `direct`
3192    /// included) is refused `not_registered` before the table is consulted.
3193    fn handle_scope_sync(
3194        &self,
3195        connection_id: ConnectionId,
3196        frame: Frame,
3197        generation: u64,
3198        scopes: Vec<ScopeRecord>,
3199    ) -> Result<Vec<Frame>, RouterError> {
3200        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3201    }
3202
3203    fn handle_scope_change(
3204        &self,
3205        connection_id: ConnectionId,
3206        frame: Frame,
3207        generation: u64,
3208        scopes: Vec<ScopeRecord>,
3209        end: Option<Vec<ScopeEnd>>,
3210    ) -> Result<Vec<Frame>, RouterError> {
3211        let is_apply = end.is_some();
3212        let Some(registration) = self
3213            .registry
3214            .get_module_by_connection(connection_id)
3215            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3216        else {
3217            return Ok(vec![control_error_frame(
3218                &frame,
3219                "not_registered",
3220                "scope.sync and scope.apply require an active module registration owned by this connection",
3221            )?]);
3222        };
3223        let owner = registration.manifest.module_id;
3224        let scopes = Self::prepare_scope_records(scopes);
3225        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3226        let is_current_launch = |connection: ConnectionId| {
3227            self.hello_launch_nonces
3228                .lock()
3229                .unwrap_or_else(|poisoned| poisoned.into_inner())
3230                .presented(connection, current_nonce.as_deref())
3231        };
3232        // Lock order is the scope table, then the forwarding table: the new
3233        // tags are published, and the routes the change closes are selected,
3234        // while the scope table is still write-locked, so no admission can read
3235        // a record whose tag is not yet published.
3236        let mut table = self
3237            .scopes
3238            .write()
3239            .unwrap_or_else(|poisoned| poisoned.into_inner());
3240        let outcome = match end {
3241            Some(end) => table.apply(
3242                &owner,
3243                connection_id,
3244                is_current_launch,
3245                generation,
3246                scopes,
3247                end,
3248            ),
3249            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3250        };
3251        let drained = match &outcome {
3252            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3253            Err(_) => Vec::new(),
3254        };
3255        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3256        drop(table);
3257        if has_deadlines {
3258            self.scope_deadline_added.notify_one();
3259        }
3260        match outcome {
3261            Ok(applied) => {
3262                let counts = ScopeOutcomeCounts::of(&applied.results);
3263                info!(
3264                    owner = %owner,
3265                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3266                    generation,
3267                    records = applied.results.len(),
3268                    created = counts.created,
3269                    replaced = counts.replaced,
3270                    updated = counts.updated,
3271                    unchanged = counts.unchanged,
3272                    refused = counts.refused,
3273                    ended = applied.ended.len(),
3274                    tag_changes = applied.tag_changes.len(),
3275                    routes_closed = drained.len(),
3276                    "scope change accepted"
3277                );
3278                // An accepted scope change can still refuse individual records, and the
3279                // owner is the only party that sees the reply. Name them here so
3280                // an operator can tell a refused session from a missing one
3281                // without the owner's logs. Capped so a call that refuses
3282                // thousands cannot flood the log; the count above is complete.
3283                for refused in applied
3284                    .results
3285                    .iter()
3286                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3287                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3288                {
3289                    warn!(
3290                        owner = %owner,
3291                        generation,
3292                        scope_ref = %refused.scope_ref,
3293                        scope_epoch = refused.scope_epoch,
3294                        code = refused.code.as_deref().unwrap_or(""),
3295                        "scope record refused"
3296                    );
3297                }
3298                self.close_scope_drained_routes(drained);
3299                let response = if is_apply {
3300                    ModuleControlResponseToModule::ScopeApply {
3301                        generation,
3302                        results: applied.results,
3303                        end_results: applied.end_results,
3304                        ended: applied.ended,
3305                    }
3306                } else {
3307                    ModuleControlResponseToModule::ScopeSync {
3308                        generation,
3309                        results: applied.results,
3310                        ended: applied.ended,
3311                    }
3312                };
3313                Ok(vec![control_response_body_frame(
3314                    &frame,
3315                    &response,
3316                    "scope change response",
3317                )?])
3318            }
3319            Err(refusal) => {
3320                info!(
3321                    owner = %owner,
3322                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3323                    generation,
3324                    code = refusal.code,
3325                    "scope change refused"
3326                );
3327                Ok(vec![control_error_frame(
3328                    &frame,
3329                    refusal.code,
3330                    refusal.message,
3331                )?])
3332            }
3333        }
3334    }
3335
3336    /// Tell both ends of each route a scope change closed. The module gets a
3337    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3338    /// the client's route handle. The client also gets `route.closed` with the
3339    /// scope reason, one push per module and reason, so it can tell a revoked
3340    /// route from an ordinary close and not reopen it.
3341    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3342        if drained.is_empty() {
3343            return;
3344        }
3345        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3346            BTreeMap::new();
3347        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3348        for route in drained {
3349            warn!(
3350                module_id = %route.module_id,
3351                reason = ?route.reason,
3352                client_connection_id = route.client.connection_id.get(),
3353                route_channel = route.client.channel,
3354                "closing route because its scope changed"
3355            );
3356            pushes
3357                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3358                .or_insert_with(|| (route.reason, Vec::new()))
3359                .1
3360                .push(EndpointRoute {
3361                    goodbye_target: route.client.clone(),
3362                    principal: Principal::Unverified,
3363                    bound_at: Instant::now(),
3364                    draining: false,
3365                    drain_reason: None,
3366                });
3367            goodbyes.push(route.module);
3368            goodbyes.push(route.client);
3369        }
3370        for ((module_id, _), (reason, routes)) in pushes {
3371            send_route_control_pushes(
3372                &self.forwarding,
3373                routes,
3374                ClientControlPush::RouteClosed {
3375                    module_id,
3376                    channels: Vec::new(),
3377                    reason,
3378                    drained: false,
3379                    abandoned: 0,
3380                    excluded_subscriptions: 0,
3381                    terminal: Some(false),
3382                },
3383            );
3384        }
3385        self.emit_route_goodbyes(goodbyes);
3386    }
3387
3388    /// `scope.describe`: any registered module may read any scope, because a
3389    /// provider must read the scope a route it serves is stamped with.
3390    fn handle_scope_describe(
3391        &self,
3392        connection_id: ConnectionId,
3393        frame: Frame,
3394        owner: Principal,
3395        scope_ref: String,
3396    ) -> Result<Vec<Frame>, RouterError> {
3397        let registered = self
3398            .registry
3399            .get_module_by_connection(connection_id)
3400            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3401        let Some(registered) = registered else {
3402            return Ok(vec![control_error_frame(
3403                &frame,
3404                "not_registered",
3405                "scope.describe requires an active module registration owned by this connection",
3406            )?]);
3407        };
3408        let mut description = self
3409            .scopes
3410            .read()
3411            .unwrap_or_else(|poisoned| poisoned.into_inner())
3412            .describe(&owner, &scope_ref);
3413        // Describe readers also decode strict ScopeAttributes. Attribution is
3414        // safe to omit unless the requesting module declares it understands it.
3415        if !registered
3416            .manifest
3417            .capabilities
3418            .as_ref()
3419            .is_some_and(|caps| {
3420                caps.provides
3421                    .iter()
3422                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3423            })
3424        {
3425            if let Some(stamp) = description.stamp.as_mut() {
3426                stamp.attributes.on_behalf_of_agent = None;
3427            }
3428        }
3429        let owner_configured = match &owner {
3430            // Ask whether the owner is configured (`is_configured`), not
3431            // whether it is on the roster (`get(..).is_some()`): a supervised
3432            // module's process can register and describe a scope before the
3433            // supervisor has put it on the roster.
3434            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3435            _ => false,
3436        };
3437        let response = ModuleControlResponseToModule::ScopeDescribe {
3438            status: description.status,
3439            scope_epoch: description.scope_epoch,
3440            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3441            owner_synced: description.owner_synced,
3442            owner_configured,
3443            scope: description.stamp,
3444        };
3445        Ok(vec![control_response_body_frame(
3446            &frame,
3447            &response,
3448            "ModuleControlResponseToModule::ScopeDescribe",
3449        )?])
3450    }
3451
3452    fn handle_catalog_update(
3453        &self,
3454        connection_id: ConnectionId,
3455        frame: Frame,
3456        provides: Vec<ProviderRole>,
3457        capabilities: Option<CapabilityDeclarations>,
3458        ready: Option<bool>,
3459    ) -> Result<Vec<Frame>, RouterError> {
3460        self.refresh_capability_requirements();
3461        let Some(registration) = self
3462            .registry
3463            .get_module_by_connection(connection_id)
3464            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3465        else {
3466            return Ok(vec![control_error_frame(
3467                &frame,
3468                "not_registered",
3469                "catalog.update requires an active module registration owned by this connection",
3470            )?]);
3471        };
3472
3473        if let Some(message) =
3474            catalog_update_frozen_field_message(&registration.manifest, &provides)
3475        {
3476            return Ok(vec![control_error_frame(
3477                &frame,
3478                "catalog_update_frozen_field",
3479                message,
3480            )?]);
3481        }
3482
3483        let mut candidate = registration.manifest.clone();
3484        candidate.provides = provides.clone();
3485        candidate.capabilities = capabilities
3486            .clone()
3487            .or_else(|| registration.manifest.capabilities.clone());
3488        if let Some(rich_ui) = &candidate.rich_ui {
3489            if let Err(err) = rich_ui.validate() {
3490                return Ok(vec![control_error_frame(
3491                    &frame,
3492                    "invalid_manifest",
3493                    err.to_string(),
3494                )?]);
3495            }
3496        }
3497        if let Err(err) = candidate.validate_capability_grammar() {
3498            return Ok(vec![control_error_frame(
3499                &frame,
3500                "invalid_capability_grammar",
3501                err.to_string(),
3502            )?]);
3503        }
3504
3505        // Updates must honor the same reserved owner as initial registration;
3506        // otherwise an empty HELLO could acquire the claim after admission.
3507        let mut conflicts = self
3508            .capability_evaluator
3509            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3510        if let Some(conflict) = conflicts.first() {
3511            let message = format!(
3512                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3513                conflict.capability, conflict.claimants[0], candidate.module_id
3514            );
3515            for conflict in &mut conflicts {
3516                conflict.source = DuplicateClaimSource::CatalogUpdate;
3517            }
3518            log_duplicate_claim_events(conflicts);
3519            return Ok(vec![control_error_frame(
3520                &frame,
3521                "reserved_capability",
3522                message,
3523            )?]);
3524        }
3525
3526        let updated = self
3527            .registry
3528            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3529            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3530        if updated.is_none() {
3531            return Ok(vec![control_error_frame(
3532                &frame,
3533                "not_registered",
3534                "catalog.update requires an active module registration owned by this connection",
3535            )?]);
3536        }
3537        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3538            log_duplicate_claim_events(
3539                self.capability_evaluator
3540                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3541            );
3542        }
3543        if capability_census_trigger(
3544            registration.manifest.capabilities.as_ref(),
3545            updated
3546                .as_ref()
3547                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3548        ) {
3549            self.enforce_capability_denies();
3550        }
3551        self.refresh_capability_requirements();
3552
3553        let response = ModuleControlResponseToModule::CatalogUpdate {};
3554        control_response_body_frame(
3555            &frame,
3556            &response,
3557            "ModuleControlResponseToModule::CatalogUpdate",
3558        )
3559        .map(|frame| vec![frame])
3560    }
3561
3562    pub(crate) fn with_activity_ledger_config(
3563        mut self,
3564        run_dir: &Path,
3565        incarnation: String,
3566        config: crate::activity::ActivityConfig,
3567    ) -> Self {
3568        self.activity = Some(crate::activity::ActivityLedger::open_config(
3569            run_dir.join("activity.jsonl"),
3570            incarnation,
3571            config,
3572        ));
3573        self.supervisor.set_activity(self.activity.clone());
3574        self
3575    }
3576
3577    pub(crate) async fn configure_activity(
3578        &self,
3579        config: crate::activity::ActivityConfig,
3580    ) -> Result<(), &'static str> {
3581        match &self.activity {
3582            Some(ledger) => ledger.configure(config).await,
3583            None => Ok(()),
3584        }
3585    }
3586
3587    fn cancel_activity_subscription(
3588        &self,
3589        ctx: &RouteCtx,
3590        frame: &Frame,
3591    ) -> Result<bool, RouterError> {
3592        let task = self
3593            .activity_streams
3594            .lock()
3595            .unwrap_or_else(|p| p.into_inner())
3596            .remove(&(ctx.connection_id, frame.header.corr));
3597        let Some(task) = task else {
3598            return Ok(false);
3599        };
3600        task.abort();
3601        let end = Frame::build_with_version(
3602            response_version(frame),
3603            FrameType::StreamEnd,
3604            control_flags(),
3605            0,
3606            0,
3607            frame.header.corr,
3608            Vec::new(),
3609        )
3610        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3611        let sink = ctx.egress.clone();
3612        tokio::spawn(async move {
3613            let _ = sink.send(end).await;
3614        });
3615        Ok(true)
3616    }
3617
3618    async fn handle_activity_read(
3619        &self,
3620        ctx: &RouteCtx,
3621        frame: Frame,
3622    ) -> Result<Vec<Frame>, RouterError> {
3623        use subc_control::{
3624            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3625            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3626        };
3627        #[derive(Deserialize)]
3628        struct Identity {
3629            #[serde(default)]
3630            consumer_identity: Option<ConsumerIdentity>,
3631        }
3632        // A registered module connection already identifies its caller. Ignore
3633        // consumer_identity there so a module cannot borrow a mirror's identity.
3634        // Only separate consumer connections use the Direct/nonce trust boundary.
3635        let principal = match self
3636            .forwarding
3637            .module_id_for_connection(ctx.connection_id)
3638            .map_err(RouterError::Forwarding)?
3639        {
3640            Some(module_id) => Ok(Principal::Reserved { module_id }),
3641            None => {
3642                let identity: Identity = match serde_json::from_slice(&frame.body) {
3643                    Ok(identity) => identity,
3644                    Err(_) => {
3645                        return Ok(vec![control_error_frame(
3646                            &frame,
3647                            ACTIVITY_MALFORMED,
3648                            "malformed activity read",
3649                        )?])
3650                    }
3651                };
3652                self.route_open_principal(&frame, identity.consumer_identity)?
3653            }
3654        };
3655        let admitted = matches!(principal, Ok(Principal::Direct))
3656            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3657        if !admitted {
3658            return Ok(vec![control_error_frame(
3659                &frame,
3660                ACTIVITY_READ_FORBIDDEN,
3661                "activity reads require an operator or trusted mirror principal",
3662            )?]);
3663        }
3664        let Some(ledger) = &self.activity else {
3665            return Ok(vec![control_error_frame(
3666                &frame,
3667                ACTIVITY_UNAVAILABLE,
3668                "activity ledger unavailable",
3669            )?]);
3670        };
3671        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3672            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3673        {
3674            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3675                Ok(request) => ledger.query(request).await,
3676                Err(_) => Err(ACTIVITY_MALFORMED),
3677            };
3678            return match result {
3679                Ok(reply) => Ok(vec![control_response_body_frame(
3680                    &frame,
3681                    &reply,
3682                    ops::ACTIVITY_QUERY,
3683                )?]),
3684                Err(code) => Ok(vec![control_error_frame(
3685                    &frame,
3686                    code,
3687                    "activity query refused",
3688                )?]),
3689            };
3690        }
3691        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3692            Ok(request) => request,
3693            Err(_) => {
3694                return Ok(vec![control_error_frame(
3695                    &frame,
3696                    ACTIVITY_MALFORMED,
3697                    "malformed activity subscription",
3698                )?])
3699            }
3700        };
3701        let mut subscription = match ledger.subscribe(request.since).await {
3702            Ok(subscription) => subscription,
3703            Err(code) => {
3704                return Ok(vec![control_error_frame(
3705                    &frame,
3706                    code,
3707                    "activity subscription refused",
3708                )?])
3709            }
3710        };
3711        let key = (ctx.connection_id, frame.header.corr);
3712        let sink = ctx.egress.clone();
3713        let mut streams = self
3714            .activity_streams
3715            .lock()
3716            .unwrap_or_else(|p| p.into_inner());
3717        streams.retain(|_, task| !task.is_finished());
3718        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3719            return Ok(vec![control_error_frame(
3720                &frame,
3721                "duplicate_subscription",
3722                "activity subscription correlation id already in use",
3723            )?]);
3724        }
3725        let task = tokio::spawn(async move {
3726            loop {
3727                let entry = if let Some(entry) = subscription.replay.pop_front() {
3728                    Some(entry)
3729                } else {
3730                    subscription.receiver.recv().await
3731                };
3732                let Some(entry) = entry else {
3733                    break;
3734                };
3735                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3736                    return;
3737                };
3738                let Ok(next) = Frame::build_with_version(
3739                    response_version(&frame),
3740                    FrameType::StreamData,
3741                    control_flags(),
3742                    0,
3743                    0,
3744                    frame.header.corr,
3745                    body,
3746                ) else {
3747                    return;
3748                };
3749                if sink.send(next).await.is_err() {
3750                    return;
3751                }
3752            }
3753            // Dropping a full subscriber's sender drains its accepted entries
3754            // before the terminal error, preserving their order on the wire.
3755            if let Some(code) = subscription.terminal_code() {
3756                if let Ok(end) =
3757                    control_error_frame(&frame, code, "activity subscriber fell behind")
3758                {
3759                    let _ = sink.send(end).await;
3760                }
3761            }
3762        });
3763        streams.insert(key, task.abort_handle());
3764        Ok(Vec::new())
3765    }
3766
3767    async fn handle_activity_record(
3768        &self,
3769        ctx: &RouteCtx,
3770        frame: Frame,
3771    ) -> Result<Vec<Frame>, RouterError> {
3772        use subc_control::{
3773            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3774            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3775        };
3776        let result = async {
3777            let module = self
3778                .forwarding
3779                .module_id_for_connection(ctx.connection_id)
3780                .ok()
3781                .flatten()
3782                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3783            let request: ActivityRecordRequest =
3784                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3785            crate::activity::validate(&module, &request)?;
3786            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3787            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3788            let at_ms = std::time::SystemTime::now()
3789                .duration_since(std::time::UNIX_EPOCH)
3790                .unwrap_or_default()
3791                .as_millis() as u64;
3792            ledger.record(module, request, actor, at_ms).await
3793        }
3794        .await;
3795        match result {
3796            Ok(entry) => Ok(vec![control_response_body_frame(
3797                &frame,
3798                &ActivityRecordResponse { seq: entry.seq },
3799                "activity.record",
3800            )?]),
3801            Err(code) => Ok(vec![control_error_frame(
3802                &frame,
3803                code,
3804                "activity recording refused",
3805            )?]),
3806        }
3807    }
3808
3809    fn activity_actor(
3810        &self,
3811        connection: ConnectionId,
3812        module: &str,
3813        on: Option<&subc_control::ActivityOn>,
3814    ) -> Result<subc_control::ActivityActor, &'static str> {
3815        use crate::forwarding::{DataRoute, DataRouteState};
3816        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3817        let principal = |id: &str| Principal::Reserved {
3818            module_id: id.to_owned(),
3819        };
3820        let mut actor = ActivityActor {
3821            principal: principal(module),
3822            scope_owner: None,
3823            agent_id: None,
3824            run_id: None,
3825            flow_id: None,
3826            project_root: None,
3827            workdir: None,
3828            project_id: None,
3829        };
3830        let scope = match on {
3831            None => return Ok(actor),
3832            Some(ActivityOn::Scope(scope)) => {
3833                let selector = subc_protocol::scope::ScopeSelector::new(
3834                    principal(module),
3835                    scope.scope_ref.clone(),
3836                    Some(scope.epoch),
3837                );
3838                self.scopes
3839                    .read()
3840                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3841                    .attribute(&principal(module), module, &selector, true)
3842                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3843            }
3844            Some(ActivityOn::Route(route)) => {
3845                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3846                    .forwarding
3847                    .lookup_data_route(connection, route.channel, route.epoch)
3848                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3849                else {
3850                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3851                };
3852                if binding.module_id != module {
3853                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3854                }
3855                actor.principal = binding.principal.clone();
3856                actor.project_root = binding
3857                    .project_root
3858                    .as_ref()
3859                    .map(|root| root.as_path().to_path_buf());
3860                let Some(bound) = &binding.scope else {
3861                    return Ok(actor);
3862                };
3863                let owner = principal(&bound.owner);
3864                let selector = subc_protocol::scope::ScopeSelector::new(
3865                    owner.clone(),
3866                    bound.scope_ref.clone(),
3867                    Some(bound.tag.scope_epoch),
3868                );
3869                let admitted = self
3870                    .scopes
3871                    .read()
3872                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3873                    .attribute(&owner, module, &selector, true)
3874                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3875                if admitted.tag != bound.tag {
3876                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3877                }
3878                admitted
3879            }
3880        };
3881        actor.scope_owner = Some(scope.stamp.owner);
3882        actor.agent_id = scope.stamp.attributes.agent_id;
3883        actor.run_id = scope.stamp.attributes.run_id;
3884        actor.flow_id = scope.stamp.attributes.flow_id;
3885        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3886        actor.project_id = scope.stamp.project_id;
3887        Ok(actor)
3888    }
3889
3890    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3891        self.refresh_capability_requirements();
3892        // A bare connection count is ambiguous between many clients holding a
3893        // route each and one client accumulating hundreds, so publish the
3894        // concentration alongside it. Route state is best-effort here: a
3895        // diagnostic endpoint must still answer if the forwarding lock is
3896        // contended.
3897        let mut counters = self.counters.snapshot();
3898        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3899            obj.insert(
3900                "activity_unparseable_lines".into(),
3901                ledger.read_errors().into(),
3902            );
3903            obj.insert(
3904                "activity_lifecycle_append_failures".into(),
3905                ledger.lifecycle_failures().into(),
3906            );
3907            obj.insert(
3908                "activity_started_digest_skips".into(),
3909                ledger.lifecycle_digest_skips().into(),
3910            );
3911        }
3912        if let (Ok((connections_with_routes, max)), Some(obj)) = (
3913            self.forwarding.client_route_concentration(),
3914            counters.as_object_mut(),
3915        ) {
3916            obj.insert(
3917                "client_connections_with_routes".into(),
3918                connections_with_routes.into(),
3919            );
3920            obj.insert("max_routes_on_one_connection".into(), max.into());
3921        }
3922        // A module that is being fast-refused and a module that is fine look
3923        // identical from a client that retries and succeeds, so name the open
3924        // breakers here. This rides the existing free-form counters object
3925        // rather than a new wire field, so no sibling that deserializes
3926        // `ServerDescribe` has to be rebuilt to keep reading it.
3927        if let (Some(open_breakers), Some(obj)) = (
3928            self.route_bind_breakers.open_snapshot(),
3929            counters.as_object_mut(),
3930        ) {
3931            obj.insert("route_bind_breakers_open".into(), open_breakers);
3932        }
3933        let response = ClientControlResponse::ServerDescribe {
3934            protocol_ver: PROTOCOL_VERSION,
3935            subc_ops: subc_ops(),
3936            capabilities: self.describe_capabilities(),
3937            connected_clients: self.connected_clients.count(),
3938            counters: Some(counters),
3939            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3940            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3941            capability_requirements: self.capability_requirement_statuses(),
3942            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3943            config_path: self
3944                .rescan
3945                .as_ref()
3946                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3947        };
3948        Ok(vec![control_response_body_frame(
3949            &frame,
3950            &response,
3951            "ClientControlResponse::ServerDescribe",
3952        )?])
3953    }
3954
3955    fn handle_catalog_list(
3956        &self,
3957        frame: Frame,
3958        module_id: Option<String>,
3959    ) -> Result<Vec<Frame>, RouterError> {
3960        let (generation, modules) = self
3961            .registry
3962            .list_modules_with_nonce_verification()
3963            .map_err(|err| {
3964                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3965            })?;
3966        let entries = modules
3967            .into_iter()
3968            .filter(|(registration, _)| {
3969                module_id
3970                    .as_deref()
3971                    .map(|wanted| registration.manifest.module_id == wanted)
3972                    .unwrap_or(true)
3973            })
3974            .map(|(registration, nonce_verified)| {
3975                let not_ready = self.not_ready_reason(&registration);
3976                let roles = registration.manifest.provides;
3977                CatalogEntry::new(
3978                    registration.manifest.module_id,
3979                    roles,
3980                    registration.control_ops,
3981                )
3982                .with_ready(not_ready.is_none())
3983                .with_not_ready(not_ready)
3984                .with_module_version(Some(registration.manifest.module_version))
3985                .with_capabilities(registration.manifest.capabilities)
3986                .with_self_signals(registration.manifest.self_signals)
3987                .with_events(registration.manifest.events)
3988                .with_operation_delivery(registration.manifest.operation_delivery)
3989                .with_rich_ui(registration.manifest.rich_ui)
3990                .with_nonce_verified(nonce_verified)
3991            })
3992            .collect();
3993        let response = ClientControlResponse::CatalogList {
3994            generation,
3995            modules: entries,
3996            subc_ops: subc_ops(),
3997        };
3998        Ok(vec![control_response_body_frame(
3999            &frame,
4000            &response,
4001            "ClientControlResponse::CatalogList",
4002        )?])
4003    }
4004
4005    fn route_open_principal(
4006        &self,
4007        frame: &Frame,
4008        consumer_identity: Option<ConsumerIdentity>,
4009    ) -> Result<Result<Principal, Frame>, RouterError> {
4010        let Some(consumer_identity) = consumer_identity else {
4011            return Ok(Ok(Principal::Direct));
4012        };
4013
4014        if self.supervisor.spawned_consumer_authorized(
4015            &consumer_identity.module_id,
4016            &consumer_identity.launch_nonce,
4017        ) {
4018            return Ok(Ok(Principal::Reserved {
4019                module_id: consumer_identity.module_id,
4020            }));
4021        }
4022
4023        Ok(Err(control_error_frame(
4024            frame,
4025            "bad_consumer_identity",
4026            format!(
4027                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4028                consumer_identity.module_id
4029            ),
4030        )?))
4031    }
4032
4033    /// Ordinary `route.open` refusals go through here; admission and breaker
4034    /// refusals log separately with their capacity or breaker state. The daemon can
4035    /// attest which code it sent: without the event, a client's "the daemon
4036    /// refused me" and the daemon's own view could only be reconciled by
4037    /// argument. Malformed input (`invalid_project_root`) does not come here;
4038    /// rejecting a request that was never a valid open is not a refusal of one.
4039    fn route_open_refusal_frame(
4040        &self,
4041        ctx: &RouteCtx,
4042        frame: &Frame,
4043        module_id: &str,
4044        reason: &'static str,
4045        code: &'static str,
4046        message: impl Into<String>,
4047    ) -> Result<Frame, RouterError> {
4048        self.observe_route_open_refusal(ctx, module_id, reason, code);
4049        control_error_frame(frame, code, message.into())
4050    }
4051
4052    /// Refuse a `route.open` because the target module's bind-relay breaker is
4053    /// open, without attempting the relay.
4054    ///
4055    /// The wire code is `module_timeout`, which is the truth (the module has
4056    /// not been answering binds) and which both SDKs already classify as
4057    /// retryable with capped backoff. Reusing it is what keeps this change out
4058    /// of both SDKs; the daemon-side distinction lives in the counter key
4059    /// instead.
4060    ///
4061    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4062    /// While a breaker is open this fires on every open to that module, and the
4063    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4064    /// produced 261 lines about a single module inside 3000 lines of daemon
4065    /// log. The rare transitions are logged at warn/info instead and the volume
4066    /// is carried by the counter, so the evidence survives without the flood.
4067    /// The debug line keeps a per-refusal record reachable for whoever turns
4068    /// the level up.
4069    fn route_open_breaker_refusal_frame(
4070        &self,
4071        ctx: &RouteCtx,
4072        frame: &Frame,
4073        module_id: &str,
4074        consecutive_timeouts: u32,
4075        retry_in: Duration,
4076        probe_in_flight: bool,
4077    ) -> Result<Frame, RouterError> {
4078        self.counters
4079            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4080        debug!(
4081            target: "control",
4082            code = "module_timeout",
4083            module_id = ?module_id,
4084            connection_id = ctx.connection_id.get(),
4085            consecutive_timeouts,
4086            retry_in_ms = retry_in.as_millis() as u64,
4087            probe_in_flight,
4088            "route.open refused by open bind-relay breaker"
4089        );
4090        // Say what a caller can act on. An open bind-relay breaker means the
4091        // module timed out accepting several new routes in a row. The module
4092        // is still running and its established routes keep working; only new
4093        // route.open requests are refused until the cooldown ends and one
4094        // test route (the probe) gets through. A message that only counts
4095        // failed relays reads as "the module is down" to a worker that sees it.
4096        let detail = if probe_in_flight {
4097            "one test route is already being tried; retry once it settles".to_string()
4098        } else {
4099            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4100        };
4101        control_error_frame(
4102            frame,
4103            "module_timeout",
4104            format!(
4105                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4106                 timed out in a row); {detail}; its established routes are unaffected"
4107            ),
4108        )
4109    }
4110
4111    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4112    /// requester's bytes (an unknown target is whatever the client sent) and
4113    /// is Debug-formatted so control characters land in the log escaped
4114    /// rather than as terminal sequences for whoever tails it.
4115    ///
4116    /// `reason` names the check that refused, because one wire code has
4117    /// several senders: after a module registers, `target_unavailable` can
4118    /// come from a missing role, an inactive registration, a supervisor that
4119    /// has not marked the process live, a missing forwarding connection, or a
4120    /// failed relay, and a log that records only the code cannot say which of
4121    /// them fired. It is a static, daemon-chosen label per branch, so it is
4122    /// safe to print plainly and stays a closed set.
4123    fn observe_route_open_refusal(
4124        &self,
4125        ctx: &RouteCtx,
4126        module_id: &str,
4127        reason: &'static str,
4128        code: &'static str,
4129    ) {
4130        self.counters.increment_route_open_refused(code);
4131        info!(
4132            target: "control",
4133            code,
4134            reason,
4135            module_id = ?module_id,
4136            connection_id = ctx.connection_id.get(),
4137            "route.open refused"
4138        );
4139        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4140            self.route_outages.record_not_serving(module_id, reason);
4141        }
4142    }
4143
4144    /// Record an ACCEPTED route.open.
4145    ///
4146    /// Refusals have been logged and counted since the attestation work; accepts
4147    /// were invisible, so the daemon knew every principal it stamped and wrote
4148    /// none of them down. The party that attests the identity was the only party
4149    /// not recording it, which left a credential vault unable to name the sender
4150    /// of a call that reached it (claustrum #43) and left the launch-nonce
4151    /// concurrency question unanswerable from the outside.
4152    ///
4153    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4154    /// `code`/`module_id`/`connection_id` returns both directions of the same
4155    /// decision rather than two shapes a reader has to join by hand.
4156    ///
4157    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4158    /// and the difference carries information rather than being an
4159    /// inconsistency. This line is only reachable after a successful bind to a
4160    /// REGISTERED module, so the value has already passed HELLO validation
4161    /// including the path-hazard refusal and cannot contain control bytes. A
4162    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4163    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4164    ///
4165    /// Bare is also what every other daemon line already emits (`module
4166    /// registered`, `configured module supervised`). Shipping `?module_id` here
4167    /// made this instrument the only one in the file whose ids did not answer
4168    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4169    /// line whose whole purpose is being grepped beside its sibling.
4170    ///
4171    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4172    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4173    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4174    /// one are recorded identically. A test written against that harness passes
4175    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4176    /// green assertion that cannot fail. The same limit applies to the escaping
4177    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4178    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4179    /// Fencing either needs the real formatter, not the capture layer.
4180    ///
4181    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4182    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4183    /// to record. The ephemeral port would decay within minutes and answer only a
4184    /// live question. The identity question is instead answered by counting
4185    /// distinct live connections presenting one module's `consumer_identity` --
4186    /// "is anyone else holding this secret" rather than "is this the right
4187    /// process".
4188    fn observe_route_open_accept(
4189        &self,
4190        ctx: &RouteCtx,
4191        module_id: &str,
4192        principal: &str,
4193        scope: &str,
4194    ) {
4195        self.route_outages.record_accepted(module_id);
4196        self.counters.increment_route_open_accepted(principal);
4197        info!(
4198            target: "control",
4199            principal,
4200            module_id,
4201            connection_id = ctx.connection_id.get(),
4202            scope = %scope,
4203            "route.open accepted"
4204        );
4205    }
4206
4207    fn supervised_absent_route_open_refusal_frame(
4208        &self,
4209        ctx: &RouteCtx,
4210        frame: &Frame,
4211        module_id: &str,
4212        code: &'static str,
4213        status: &crate::supervise::ModuleStatus,
4214    ) -> Result<Frame, RouterError> {
4215        self.counters.increment_route_open_refused(code);
4216        info!(
4217            target: "control",
4218            code,
4219            reason = "supervised_not_registered",
4220            module_id = ?module_id,
4221            connection_id = ctx.connection_id.get(),
4222            state = %status.state,
4223            enabled = status.enabled,
4224            live = status.live,
4225            "route.open refused"
4226        );
4227        // A supervised module whose process has not registered is not
4228        // serving, whatever the reason; the supervisor knows this id, so it is
4229        // safe to track.
4230        self.route_outages
4231            .record_not_serving(module_id, "supervised_not_registered");
4232        let message = if let Some(pid) = status.waiting_for_previous_pid {
4233            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4234                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4235            })
4236        } else {
4237            format!(
4238                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4239                status.state, status.enabled, status.live
4240            )
4241        };
4242        control_error_frame(frame, code, message)
4243    }
4244
4245    async fn handle_route_open(
4246        &self,
4247        ctx: &RouteCtx,
4248        frame: Frame,
4249        request: RouteOpenRequest,
4250        timings: &mut RouteOpenTimings,
4251    ) -> Result<Vec<Frame>, RouterError> {
4252        let RouteOpenRequest {
4253            target,
4254            mut identity,
4255            consumer_identity,
4256            consumer_capabilities,
4257            role_versions,
4258            admission_facts,
4259            scope,
4260        } = request;
4261        let target_module_id = target_module_id(&target).to_string();
4262        timings.module_id.clone_from(&target_module_id);
4263        if self
4264            .registry
4265            .get_module_by_connection(ctx.connection_id)
4266            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4267            .is_some()
4268        {
4269            return Ok(vec![control_error_frame(
4270                &frame,
4271                "invalid_request",
4272                "module connections cannot open client routes",
4273            )?]);
4274        }
4275        debug!(
4276            connection_id = ctx.connection_id.get(),
4277            corr = frame.header.corr,
4278            module_id = %target_module_id,
4279            "handling route.open"
4280        );
4281
4282        // A malformed declaration is refused first, before anything about the
4283        // target is looked up: the same body would be refused against any
4284        // module, so the caller learns nothing by retrying or waiting. An empty
4285        // map declares nothing and travels as no field at all, so a provider
4286        // only ever sees a missing field or a non-empty one.
4287        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4288        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4289            self.observe_route_open_refusal(
4290                ctx,
4291                &target_module_id,
4292                "invalid_role_versions",
4293                error_codes::INVALID_REQUEST,
4294            );
4295            return Ok(vec![control_error_body_frame(
4296                &frame,
4297                ErrorBody {
4298                    code: error_codes::INVALID_REQUEST.to_string(),
4299                    message: error.to_string(),
4300                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4301                },
4302            )?]);
4303        }
4304
4305        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4306            if status.waiting_for_previous_pid.is_some() {
4307                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4308                    ctx,
4309                    &frame,
4310                    &target_module_id,
4311                    "module_warming",
4312                    &status,
4313                )?]);
4314            }
4315        }
4316
4317        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4318        // opposite. Below, a caller learns whether a module is unregistered,
4319        // supervised-but-down (with state/enabled/live), or registered without the
4320        // requested role. Elsewhere that is an enumeration leak: a probe learning
4321        // the shape of a fleet it cannot otherwise see.
4322        //
4323        // It is not one here, and the reason is the ACCESS MODEL rather than
4324        // anything about these errors. Reaching route.open requires the
4325        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4326        // connection file, so any caller who completes it already runs as this
4327        // user -- and can read subc.jsonc for the module list and `ck module
4328        // status` for live state. The reply discloses nothing the caller cannot
4329        // read more easily from disk, while the precision is load-bearing:
4330        // `unknown_module` is retryable and a missing role is not.
4331        //
4332        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4333        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4334        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4335        let Some(registration) = self
4336            .registry
4337            .get_module(&target_module_id)
4338            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4339        else {
4340            if let Some((status, warming)) =
4341                self.supervisor_status(&target_module_id, frame.header.corr)?
4342            {
4343                // BEFORE the two availability codes below, because for a module
4344                // that speaks no subc wire both of them are false comfort: they
4345                // say "not right now" and are retried, and this module will
4346                // never register no matter how long the caller waits. The
4347                // absence here is the declaration being honoured, not a module
4348                // that is late.
4349                if status.protocol == ModuleProtocol::None {
4350                    return Ok(vec![self.route_open_refusal_frame(
4351                        ctx,
4352                        &frame,
4353                        &target_module_id,
4354                        "protocol_none",
4355                        error_codes::MODULE_NO_PROTOCOL,
4356                        format!(
4357                            "module_id '{target_module_id}' is declared protocol: none; \
4358                             it speaks no subc wire and serves no routes"
4359                        ),
4360                    )?]);
4361                }
4362                let code = if warming {
4363                    "module_warming"
4364                } else {
4365                    "target_unavailable"
4366                };
4367                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4368                    ctx,
4369                    &frame,
4370                    &target_module_id,
4371                    code,
4372                    &status,
4373                )?]);
4374            }
4375            if let Some(removed_ago_ms) =
4376                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4377            {
4378                return Ok(vec![self.route_open_refusal_frame(
4379                    ctx,
4380                    &frame,
4381                    &target_module_id,
4382                    "removed",
4383                    error_codes::MODULE_REMOVED,
4384                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4385                )?]);
4386            }
4387            return Ok(vec![self.route_open_refusal_frame(
4388                ctx,
4389                &frame,
4390                &target_module_id,
4391                "not_registered",
4392                error_codes::UNKNOWN_MODULE,
4393                format!("module_id '{target_module_id}' is not registered"),
4394            )?]);
4395        };
4396
4397        // Best-effort only: registry readiness and forwarding reservation use
4398        // different locks, so a module can flip readiness between this read and
4399        // the relay. Modules must still tolerate an `on_bind` while not ready.
4400        if !registration.ready {
4401            self.counters
4402                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4403            info!(
4404                target: "control",
4405                code = error_codes::MODULE_WARMING,
4406                module_id = ?target_module_id,
4407                connection_id = ctx.connection_id.get(),
4408                reason = "declared_not_ready",
4409                "route.open refused"
4410            );
4411            // The module is registered but says it cannot take work, which is
4412            // an outage from the caller's side even though its process is up.
4413            self.route_outages
4414                .record_not_serving(&target_module_id, "declared_not_ready");
4415            return Ok(vec![control_error_body_frame(
4416                &frame,
4417                ErrorBody {
4418                    code: error_codes::MODULE_WARMING.to_string(),
4419                    message: format!(
4420                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4421                    ),
4422                    detail: Some(serde_json::json!({
4423                        "reason": "declared_not_ready"
4424                    })),
4425                },
4426            )?]);
4427        }
4428
4429        // Effective readiness, second half: a module that declares a capability
4430        // `need: required` is not routable while that capability has no
4431        // registered provider. It is enforced HERE, as a retryable routing
4432        // refusal, and deliberately not as spawn ordering or a boot block. The
4433        // module is still started and registered and can make its own calls;
4434        // spawn ordering is a promise that cannot be kept once a provider
4435        // crashes at runtime, and refusing to boot would stop the whole
4436        // machine, including the tools needed to fix its configuration.
4437        //
4438        // "Provided" is the evaluator's verdict, which counts a provider as
4439        // soon as it has REGISTERED, not once it is ready. Two modules that
4440        // require each other's capabilities are therefore both routable once
4441        // both register; counting readiness instead would deadlock them.
4442        //
4443        // Only new opens are refused. Routes already bound when a provider
4444        // goes away stay bound: nothing here tears them down, and the module
4445        // answers them as it can. Like the readiness read above this is
4446        // best-effort against a provider registering or leaving concurrently.
4447        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
4448            self.counters
4449                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4450            info!(
4451                target: "control",
4452                code = error_codes::MODULE_WARMING,
4453                module_id = ?target_module_id,
4454                connection_id = ctx.connection_id.get(),
4455                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4456                capability = %capability,
4457                "route.open refused"
4458            );
4459            return Ok(vec![control_error_body_frame(
4460                &frame,
4461                ErrorBody {
4462                    code: error_codes::MODULE_WARMING.to_string(),
4463                    message: format!(
4464                        "module_id '{target_module_id}' requires capability '{capability}', \
4465                         which no registered module provides; retry"
4466                    ),
4467                    detail: Some(serde_json::json!({
4468                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4469                        "capability": capability,
4470                    })),
4471                },
4472            )?]);
4473        }
4474
4475        if !target_has_required_role(&target, &registration.manifest.provides) {
4476            return Ok(vec![self.route_open_refusal_frame(
4477                ctx,
4478                &frame,
4479                &target_module_id,
4480                "role_not_provided",
4481                "target_unavailable",
4482                format!("module_id '{target_module_id}' does not provide the requested target"),
4483            )?]);
4484        }
4485
4486        if registration.state != ChannelState::Active {
4487            return Ok(vec![self.route_open_refusal_frame(
4488                ctx,
4489                &frame,
4490                &target_module_id,
4491                "registration_not_active",
4492                "target_unavailable",
4493                format!("module_id '{target_module_id}' is not active"),
4494            )?]);
4495        }
4496
4497        if self
4498            .forwarding
4499            .module_is_draining(&target_module_id)
4500            .map_err(RouterError::Forwarding)?
4501        {
4502            return Ok(vec![self.route_open_refusal_frame(
4503                ctx,
4504                &frame,
4505                &target_module_id,
4506                "reloading",
4507                "module_reloading",
4508                format!("module_id '{target_module_id}' is reloading"),
4509            )?]);
4510        }
4511
4512        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4513            process_liveness.process_live(&target_module_id) == Some(false)
4514        }) {
4515            // A module the supervisor is restarting or reloading can still hold
4516            // a registration: the old process before its connection closes, or
4517            // a new one that registered while the supervisor was draining. The
4518            // forwarding table does not see that as draining, but the consumer
4519            // should still be told to retry soon, exactly as for the drain
4520            // above, rather than that the target is unavailable.
4521            if process_liveness.process_replacing(&target_module_id) {
4522                return Ok(vec![self.route_open_refusal_frame(
4523                    ctx,
4524                    &frame,
4525                    &target_module_id,
4526                    "reloading",
4527                    "module_reloading",
4528                    format!("module_id '{target_module_id}' is reloading"),
4529                )?]);
4530            }
4531            return Ok(vec![self.route_open_refusal_frame(
4532                ctx,
4533                &frame,
4534                &target_module_id,
4535                "supervisor_not_live",
4536                "target_unavailable",
4537                format!("module_id '{target_module_id}' is not live"),
4538            )?]);
4539        }
4540
4541        if !self
4542            .forwarding
4543            .has_live_module_connection(&target_module_id)
4544            .map_err(RouterError::Forwarding)?
4545        {
4546            return Ok(vec![self.route_open_refusal_frame(
4547                ctx,
4548                &frame,
4549                &target_module_id,
4550                "no_forwarding_connection",
4551                "target_unavailable",
4552                format!("module_id '{target_module_id}' has no live forwarding connection"),
4553            )?]);
4554        }
4555
4556        if let Some(error) =
4557            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4558        {
4559            self.observe_route_open_refusal(
4560                ctx,
4561                &target_module_id,
4562                "op_not_allowed",
4563                "op_not_allowed",
4564            );
4565            return Ok(vec![error]);
4566        }
4567
4568        let principal = match self.route_open_principal(&frame, consumer_identity)? {
4569            Ok(principal) => principal,
4570            Err(error) => {
4571                self.observe_route_open_refusal(
4572                    ctx,
4573                    &target_module_id,
4574                    "bad_consumer_identity",
4575                    "bad_consumer_identity",
4576                );
4577                return Ok(vec![error]);
4578            }
4579        };
4580        timings.principal = principal_label(&principal);
4581
4582        // This is attested, control-plane policy for supervised module origins.
4583        // Keep it before route reservation and out of the opaque forwarding hot
4584        // path: data frames must never acquire a per-frame capability check.
4585        if let Principal::Reserved {
4586            module_id: opening_module_id,
4587        } = &principal
4588        {
4589            if let Some(opening_registration) = self
4590                .registry
4591                .get_module(opening_module_id)
4592                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4593            {
4594                if let Some(capability) =
4595                    denied_capability(&opening_registration.manifest, &registration.manifest)
4596                {
4597                    warn!(
4598                        opening_module_id,
4599                        target_module_id,
4600                        capability,
4601                        "refusing route.open because an attested capability deny edge matches"
4602                    );
4603                    return Ok(vec![self.route_open_refusal_frame(
4604                        ctx,
4605                        &frame,
4606                        &target_module_id,
4607                        "capability_deny_edge",
4608                        "capability_forbidden",
4609                        format!(
4610                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4611                        ),
4612                    )?]);
4613                }
4614            }
4615        }
4616
4617        if admission_facts.is_some() {
4618            let carrier_matches = matches!(
4619                &principal,
4620                Principal::Reserved { module_id }
4621                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4622            );
4623            if !carrier_matches {
4624                return Ok(vec![self.route_open_refusal_frame(
4625                    ctx,
4626                    &frame,
4627                    &target_module_id,
4628                    "admission_facts_carrier_not_permitted",
4629                    "admission_facts_not_permitted",
4630                    "admission facts may only be carried by the configured reserved module",
4631                )?]);
4632            }
4633
4634            let target_allowed = self
4635                .admission_facts_targets
4636                .as_ref()
4637                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4638            if !target_allowed {
4639                return Ok(vec![self.route_open_refusal_frame(
4640                    ctx,
4641                    &frame,
4642                    &target_module_id,
4643                    "admission_facts_target_not_listed",
4644                    "admission_facts_target_not_allowed",
4645                    format!(
4646                        "admission facts are not permitted for target module_id '{target_module_id}'"
4647                    ),
4648                )?]);
4649            }
4650
4651            // Keep the value opaque to subc. The downstream admission validator owns
4652            // schema and semantic checks; this daemon only enforces carrier authority
4653            // and the configured destination allowlist.
4654        }
4655
4656        // Scope admission, on the attested principal above and never on the
4657        // request body. The tag read here travels with the pending bind and is
4658        // compared with the published one at commit, so a sync between here
4659        // and the module's ack refuses the open instead of binding a stamp
4660        // that is no longer true.
4661        let (bound_scope, mut scope_stamp) = match &scope {
4662            None => (None, None),
4663            Some(selector) => {
4664                let owner_configured = match &selector.owner {
4665                    // A reserved owner counts as configured from before its
4666                    // process is spawned (see `SupervisorHandle::is_configured`).
4667                    // So an owner that has not synced its scopes yet is refused
4668                    // as retryable (`scope_not_synced`), not as one that will
4669                    // never sync.
4670                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4671                    _ => false,
4672                };
4673                let mut scopes = self
4674                    .scopes
4675                    .write()
4676                    .unwrap_or_else(|poisoned| poisoned.into_inner());
4677                let admitted = scopes
4678                    .admit(&principal, &target_module_id, selector, owner_configured)
4679                    .and_then(|admission| {
4680                        crate::scopes::check_target_flow_support(
4681                            &admission.stamp,
4682                            &target_module_id,
4683                            registration.manifest.capabilities.as_ref(),
4684                        )?;
4685                        crate::scopes::check_target_agent_run_support(
4686                            &admission.stamp,
4687                            &target_module_id,
4688                            registration.manifest.capabilities.as_ref(),
4689                        )?;
4690                        let bound = scopes.reserve_bind(
4691                            &admission,
4692                            &principal,
4693                            ctx.connection_id,
4694                            selector.hold_presence,
4695                        );
4696                        Ok((admission, bound))
4697                    });
4698                match admitted {
4699                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4700                    Err(refusal) => {
4701                        return Ok(vec![self.route_open_refusal_frame(
4702                            ctx,
4703                            &frame,
4704                            &target_module_id,
4705                            refusal.code,
4706                            refusal.code,
4707                            refusal.message,
4708                        )?]);
4709                    }
4710                }
4711            }
4712        };
4713
4714        // Bind admits a root that no longer exists on disk, because refusing here
4715        // closes the only exit from a paused run: cancel needs a bound route, and a
4716        // renamed or reclaimed directory makes that route unopenable forever. The
4717        // run itself is intact and still addressable by its recorded identity.
4718        //
4719        // This does NOT relax the rule the strict constructor protects. That rule is
4720        // that no root is ever aliased into NEW durable state -- a missing component
4721        // can reappear as a symlink elsewhere, which would move the identity and
4722        // split a session's history across two of them. The engine now refuses the
4723        // two operations that create such state (send and import) at admission,
4724        // which is a narrower way to hold the same invariant: reads and terminations
4725        // are admitted, writes are not. That refusal had to ship before this line
4726        // changed, or there is an interval where a send commits under a provisional
4727        // identity -- the exact failure the original policy existed to prevent.
4728        //
4729        // Resolution follows realpath rather than lexical cleanup: the longest
4730        // existing ancestor is canonicalized and the missing tail re-appended, so a
4731        // live root is unchanged and a vanished leaf keeps the identity it was
4732        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
4733        // same caller the moment the directory vanished, which strands the run more
4734        // quietly than refusing it.
4735        timings.enter(RouteOpenPhase::ProjectRoot);
4736        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4737            Ok(project_root) => project_root,
4738            Err(err) => {
4739                return Ok(vec![control_error_frame(
4740                    &frame,
4741                    "invalid_project_root",
4742                    err.to_string(),
4743                )?])
4744            }
4745        };
4746        timings.enter(RouteOpenPhase::Admission);
4747        identity.project_root = project_root.as_path().to_path_buf();
4748
4749        // Last gate before any relay work, and deliberately after the cheap
4750        // registry and availability checks above: those name a more precise
4751        // condition (unknown, removed, reloading) and a caller is better served
4752        // by the precise code than by this one.
4753        //
4754        // Everything below this point costs an egress permit, a reserved handle
4755        // pair and, if the module does not answer, the whole relay budget. The
4756        // reader no longer waits for that budget, so cap each target explicitly;
4757        // serial dispatch used to provide the accidental cap of one relay per
4758        // connection. Admission is a mutex-protected count and never waits.
4759        let _concurrency_guard = match self
4760            .route_bind_concurrency
4761            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4762        {
4763            Ok(guard) => guard,
4764            Err(in_flight) => {
4765                return Ok(vec![self.route_open_target_capacity_refusal(
4766                    ctx,
4767                    &frame,
4768                    &target_module_id,
4769                    in_flight,
4770                )?]);
4771            }
4772        };
4773
4774        // A module that has already burned the whole budget `threshold` times
4775        // in a row does not get to charge it again until a probe says it recovered.
4776        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4777            RouteBindAdmission::Admitted { guard, probe } => {
4778                if probe {
4779                    info!(
4780                        module_id = %target_module_id,
4781                        connection_id = ctx.connection_id.get(),
4782                        "route.bind breaker half-open: admitting one probe"
4783                    );
4784                }
4785                guard
4786            }
4787            RouteBindAdmission::Refused {
4788                consecutive_timeouts,
4789                retry_in,
4790                probe_in_flight,
4791            } => {
4792                return Ok(vec![self.route_open_breaker_refusal_frame(
4793                    ctx,
4794                    &frame,
4795                    &target_module_id,
4796                    consecutive_timeouts,
4797                    retry_in,
4798                    probe_in_flight,
4799                )?]);
4800            }
4801        };
4802
4803        // Resolve the per-module budget here so the wait matches the operator's
4804        // intent for this specific target. A per-module override in
4805        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
4806        // daemons) wins over the daemon-wide default.
4807        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4808        let relay_deadline = Instant::now() + route_bind_relay_timeout;
4809        timings.enter(RouteOpenPhase::EgressReservation);
4810        let pending = match self
4811            .forwarding
4812            .begin_route_bind_relay_for(
4813                ctx.connection_id,
4814                ctx.egress.clone(),
4815                response_version(&frame),
4816                frame.header.corr,
4817                &target_module_id,
4818                principal.clone(),
4819                bound_scope,
4820                Some(project_root),
4821                relay_deadline,
4822            )
4823            .await
4824        {
4825            Ok(pending) => pending,
4826            Err(err) => {
4827                return Ok(vec![self.route_open_refusal_frame(
4828                    ctx,
4829                    &frame,
4830                    &target_module_id,
4831                    "relay_reservation_failed",
4832                    forwarding_error_code(&err),
4833                    err.to_string(),
4834                )?])
4835            }
4836        };
4837        timings.enter(RouteOpenPhase::Admission);
4838        let crate::forwarding::PendingRouteBindRelay {
4839            endpoint,
4840            module_sink,
4841            negotiated_ver,
4842            client_channel,
4843            client_epoch,
4844            module_channel,
4845            module_epoch,
4846            corr: relay_corr,
4847            receiver,
4848        } = pending;
4849        let mut reservation =
4850            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4851
4852        // Reserving egress can wait while a module reconnects or a swap cuts
4853        // over. Check the relay's captured connection, not the earlier by-id
4854        // lookup: the original module's flow/run capabilities cannot authorize
4855        // its replacement. The captured sink stays bound to that connection.
4856        let relay_registration = self
4857            .registry
4858            .get_module_by_connection(endpoint.connection_id)
4859            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4860        let relay_capabilities = relay_registration
4861            .as_ref()
4862            .and_then(|registration| registration.manifest.capabilities.as_ref());
4863        if let Some(selector) = &scope {
4864            // Egress reservation may wait across a scope update. Recheck the
4865            // proof against the current record before sending the captured bind.
4866            if let Err(refusal) = self
4867                .scopes
4868                .read()
4869                .unwrap_or_else(|poisoned| poisoned.into_inner())
4870                .check_bind_proof(&principal, selector)
4871            {
4872                reservation.release_and_disarm();
4873                return Ok(vec![self.route_open_refusal_frame(
4874                    ctx,
4875                    &frame,
4876                    &target_module_id,
4877                    refusal.code,
4878                    refusal.code,
4879                    refusal.message,
4880                )?]);
4881            }
4882        }
4883        if let Some(stamp) = scope_stamp
4884            .as_ref()
4885            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4886        {
4887            if let Err(refusal) = crate::scopes::check_target_flow_support(
4888                stamp,
4889                &target_module_id,
4890                relay_capabilities,
4891            )
4892            .and_then(|()| {
4893                crate::scopes::check_target_agent_run_support(
4894                    stamp,
4895                    &target_module_id,
4896                    relay_capabilities,
4897                )
4898            }) {
4899                reservation.release_and_disarm();
4900                return Ok(vec![self.route_open_refusal_frame(
4901                    ctx,
4902                    &frame,
4903                    &target_module_id,
4904                    refusal.code,
4905                    refusal.code,
4906                    refusal.message,
4907                )?]);
4908            }
4909        }
4910
4911        // Older readers may reject unknown fields. Select optional fields from
4912        // the captured endpoint's declaration, including on unscoped binds.
4913        if !relay_capabilities.is_some_and(|caps| {
4914            caps.provides
4915                .iter()
4916                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
4917        }) {
4918            if let Some(stamp) = scope_stamp.as_mut() {
4919                stamp.workdir = None;
4920                stamp.project_id = None;
4921            }
4922        }
4923        if !relay_capabilities.is_some_and(|caps| {
4924            caps.provides
4925                .iter()
4926                .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4927        }) {
4928            if let Some(stamp) = scope_stamp.as_mut() {
4929                stamp.attributes.on_behalf_of_agent = None;
4930            }
4931        }
4932        let call_mode = if relay_capabilities
4933            .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4934        {
4935            Some(derive_call_mode(scope_stamp.as_ref()))
4936        } else {
4937            if let Some(stamp) = scope_stamp.as_mut() {
4938                stamp.expires_at_ms = None;
4939            }
4940            None
4941        };
4942
4943        debug!(
4944            connection_id = ctx.connection_id.get(),
4945            client_channel,
4946            client_epoch,
4947            module_channel,
4948            module_epoch,
4949            "reserved route handle pair"
4950        );
4951        // Rendered BEFORE the move into the relay, because the accept arm below
4952        // is where it is logged and the principal is gone by then.
4953        let route_principal_label = principal_label(&principal);
4954        let scope_label = scope_stamp
4955            .as_ref()
4956            .map_or_else(|| "none".to_string(), route_open_scope_label);
4957        let relay = ModuleControlRequest::RouteBind {
4958            route_channel: module_channel,
4959            epoch: module_epoch,
4960            target,
4961            identity,
4962            principal: Some(principal),
4963            consumer_capabilities,
4964            role_versions,
4965            admission_facts,
4966            scope: scope_stamp,
4967            call_mode,
4968        };
4969        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4970            RouterError::backend(
4971                0,
4972                frame.header.corr,
4973                format!("failed to encode route.bind request: {err}"),
4974            )
4975        })?;
4976        let relay_frame = Frame::build_with_version(
4977            negotiated_ver,
4978            FrameType::Request,
4979            control_flags(),
4980            0,
4981            0,
4982            relay_corr,
4983            relay_body,
4984        )
4985        .map_err(RouterError::FrameBuild)?;
4986
4987        timings.enter(RouteOpenPhase::BindSend);
4988        let sent = module_sink.send(relay_frame).await;
4989        timings.enter(RouteOpenPhase::Admission);
4990        if let Err(err) = sent {
4991            reservation.release_and_disarm();
4992            return Ok(vec![self.route_open_refusal_frame(
4993                ctx,
4994                &frame,
4995                &target_module_id,
4996                "relay_send_failed",
4997                "target_unavailable",
4998                err.to_string(),
4999            )?]);
5000        }
5001
5002        #[cfg(test)]
5003        if let Some(hold) = &self.route_bind_relay_sent_hold {
5004            // First wait: tell the test the bind is queued. Second wait: stay
5005            // here until the test has driven whatever it wants to happen
5006            // between queueing the bind and marking it enqueued.
5007            hold.wait().await;
5008            hold.wait().await;
5009        }
5010
5011        // A missing relay is not by itself an abandoned bind: the module may
5012        // already have acked it and the route may be committed. Only a relay
5013        // dropped without any module reply needs this GOODBYE; one a reply
5014        // settled has already had its GOODBYE decided by the reply path.
5015        if self
5016            .forwarding
5017            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5018            .map_err(RouterError::Forwarding)?
5019            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5020        {
5021            self.send_abandoned_route_bind_goodbye(
5022                &module_sink,
5023                negotiated_ver,
5024                module_channel,
5025                module_epoch,
5026            );
5027        }
5028
5029        timings.enter(RouteOpenPhase::BindAckWait);
5030        let outcome = timeout_at(relay_deadline, receiver).await;
5031        timings.enter(RouteOpenPhase::Admission);
5032        match outcome {
5033            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5034                reservation.disarm();
5035                if breaker.record_accepted() {
5036                    info!(
5037                        module_id = %target_module_id,
5038                        "route.bind breaker closed: the probe was accepted"
5039                    );
5040                }
5041                self.observe_route_open_accept(
5042                    ctx,
5043                    &target_module_id,
5044                    &route_principal_label,
5045                    &scope_label,
5046                );
5047                Ok(Vec::new())
5048            }
5049            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5050                reservation.release_and_disarm();
5051                // A module that says no in microseconds is healthy. Rejection
5052                // is a different condition with its own refusal and must not
5053                // move the breaker.
5054                breaker.record_inconclusive();
5055                // The daemon's own commit re-check refused the bind because the
5056                // scope ended or changed after admission. The module accepted;
5057                // counting it as a module rejection would blame the module.
5058                let scope_code = match body.code.as_str() {
5059                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5060                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5061                    _ => None,
5062                };
5063                if let Some(code) = scope_code {
5064                    self.observe_route_open_refusal(
5065                        ctx,
5066                        &target_module_id,
5067                        "scope_changed_before_commit",
5068                        code,
5069                    );
5070                    return Ok(vec![control_error_body_frame(&frame, body)?]);
5071                }
5072                self.counters
5073                    .increment_route_open_refused("module_rejected");
5074                info!(
5075                    target: "control",
5076                    code = "module_rejected",
5077                    module_code = ?body.code,
5078                    module_id = ?target_module_id,
5079                    connection_id = ctx.connection_id.get(),
5080                    "route.open refused"
5081                );
5082                Ok(vec![control_error_body_frame(&frame, body)?])
5083            }
5084            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5085                reservation.release_and_disarm();
5086                breaker.record_inconclusive();
5087                // Fires when the module's connection closes while a relayed
5088                // bind is pending -- typically a caller racing a module restart
5089                // whose bind was relayed BEFORE the drain mark went up. Logged
5090                // because the caller sees only its own error and the fleet has
5091                // already spent one diagnosis round unable to tell this arm
5092                // from a relay timeout without daemon-side evidence.
5093                tracing::warn!(
5094                    module_id = %target_module_id,
5095                    "route.bind relay abandoned: {message}"
5096                );
5097                Ok(vec![self.route_open_refusal_frame(
5098                    ctx,
5099                    &frame,
5100                    &target_module_id,
5101                    "relay_abandoned",
5102                    "target_unavailable",
5103                    message,
5104                )?])
5105            }
5106            Ok(Err(_)) => {
5107                reservation.release_and_disarm();
5108                breaker.record_inconclusive();
5109                Ok(vec![self.route_open_refusal_frame(
5110                    ctx,
5111                    &frame,
5112                    &target_module_id,
5113                    "relay_waiter_canceled",
5114                    "target_unavailable",
5115                    "route.bind relay waiter was canceled before the module responded",
5116                )?])
5117            }
5118            Err(_) => {
5119                reservation.release_and_disarm();
5120                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5121                // answer at all is the one condition a fast refusal can
5122                // usefully stand in for; every other arm already answered.
5123                if let Some(opened) = breaker.record_timeout(
5124                    self.route_bind_breaker_threshold,
5125                    self.route_bind_breaker_cooldown,
5126                ) {
5127                    warn!(
5128                        module_id = %target_module_id,
5129                        consecutive_timeouts = opened.consecutive_timeouts,
5130                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5131                        reopened_after_probe = opened.reopened_after_probe,
5132                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5133                    );
5134                }
5135                // The generous budget just burned to no answer: the module is
5136                // registered and its connection is up, but its bind handler sat
5137                // on the ack for the full budget (warm-on-bind, cold configure,
5138                // or a wedged handler). Every earlier unavailability shape
5139                // fast-refuses BEFORE the relay, so this arm firing means the
5140                // slowness is module-side -- log it so the per-module timeline
5141                // is reconstructable without client audit rows.
5142                tracing::warn!(
5143                    module_id = %target_module_id,
5144                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5145                    "route.bind relay timed out: module did not ack within budget"
5146                );
5147                Ok(vec![self.route_open_refusal_frame(
5148                    ctx,
5149                    &frame,
5150                    &target_module_id,
5151                    "relay_timed_out",
5152                    "module_timeout",
5153                    format!(
5154                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5155                        route_bind_relay_timeout
5156                    ),
5157                )?])
5158            }
5159        }
5160    }
5161
5162    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5163        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5164            snapshot: self.supervisor.spawn_snapshot(),
5165        };
5166        Ok(vec![control_response_body_frame(
5167            &frame,
5168            &response,
5169            "ClientControlResponse::SupervisorSpawnSnapshot",
5170        )?])
5171    }
5172
5173    fn handle_supervisor_spawn_subscribe(
5174        &self,
5175        ctx: &RouteCtx,
5176        frame: Frame,
5177        since: Option<SpawnCursor>,
5178    ) -> Result<Vec<Frame>, RouterError> {
5179        match self.supervisor.subscribe_spawns(
5180            ctx.connection_id,
5181            frame.header.corr,
5182            response_version(&frame),
5183            since,
5184            ctx.egress.clone(),
5185        ) {
5186            Ok(()) => Ok(Vec::new()),
5187            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5188                Ok(vec![control_error_body_frame(
5189                    &frame,
5190                    ErrorBody {
5191                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5192                        message: "spawn cursor belongs to a different daemon incarnation"
5193                            .to_string(),
5194                        detail: Some(serde_json::json!({
5195                            "current_daemon_incarnation": current
5196                        })),
5197                    },
5198                )?])
5199            }
5200            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5201                &frame,
5202                ErrorBody {
5203                    code: "spawn_cursor_too_old".to_string(),
5204                    message: "spawn cursor predates the retained event ring".to_string(),
5205                    detail: Some(serde_json::json!({
5206                        "oldest_retained_cursor": oldest
5207                    })),
5208                },
5209            )?]),
5210            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5211                0,
5212                frame.header.corr,
5213                format!("failed to open supervisor spawn subscription: {error}"),
5214            )),
5215        }
5216    }
5217
5218    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5219        let generation = self
5220            .registry
5221            .generation()
5222            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5223        let mut modules = Vec::new();
5224        for module in self.supervisor.list() {
5225            let status = module.status_for_control("list").map_err(|err| {
5226                RouterError::backend(
5227                    0,
5228                    frame.header.corr,
5229                    format!("failed to read supervisor status: {err}"),
5230                )
5231            })?;
5232            let (configured, _) = module.configuration().map_err(|err| {
5233                RouterError::backend(
5234                    0,
5235                    frame.header.corr,
5236                    format!("failed to read module configuration: {err}"),
5237                )
5238            })?;
5239            // Status and configuration snapshots release their locks before the image probe awaits.
5240            let image = module.running_image_agreement().await;
5241            // Read per request so the figure is current when the operator asks;
5242            // the daemon samples nothing in between.
5243            let resources = Some(module.child_resource_usage());
5244            let mut verdict =
5245                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5246            verdict.spec = Some(launch_spec_agreement(
5247                &configured,
5248                status.process_alive,
5249                status.launch_spec_digest.as_ref(),
5250            ));
5251            let pending_reload = Some(verdict);
5252            // Keep the retired policy field on the wire for one release so
5253            // existing status consumers still receive the platform policy.
5254            modules.push(
5255                SupervisorEntry::new(
5256                    status.module_id,
5257                    status.state.to_string(),
5258                    status.enabled,
5259                    status.live,
5260                    status.health.status,
5261                )
5262                .with_launch_nonce_env(Some(!cfg!(unix)))
5263                .with_launch_nonce_source(status.launch_nonce_source)
5264                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5265                .with_protocol(status.protocol)
5266                .with_pending_reload(pending_reload)
5267                .with_last_probe_ms(status.health.last_probe_ms)
5268                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5269                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5270                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5271                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5272                .with_restart_count(Some(status.restart_count))
5273                .with_max_restarts(Some(status.max_restarts))
5274                .with_lifetime_restarts(Some(status.lifetime_restarts))
5275                .with_spawn_generation(Some(status.spawn_generation))
5276                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5277                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5278                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5279                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5280                .with_resources(resources),
5281            );
5282        }
5283        let response = ClientControlResponse::SupervisorList {
5284            generation,
5285            modules,
5286        };
5287        Ok(vec![control_response_body_frame(
5288            &frame,
5289            &response,
5290            "ClientControlResponse::SupervisorList",
5291        )?])
5292    }
5293
5294    fn handle_supervisor_stderr_tail(
5295        &self,
5296        frame: Frame,
5297        module_id: String,
5298        max_lines: Option<u32>,
5299        max_bytes: Option<u32>,
5300    ) -> Result<Vec<Frame>, RouterError> {
5301        let Some(module) = self.supervisor.get(&module_id) else {
5302            return Ok(vec![control_error_frame(
5303                &frame,
5304                "unknown_module",
5305                format!("module_id '{module_id}' is not supervised"),
5306            )?]);
5307        };
5308
5309        let snapshot = module.stderr_tail(
5310            max_lines.map(|value| value as usize),
5311            max_bytes.map(|value| value as usize),
5312        );
5313
5314        let response = ClientControlResponse::SupervisorStderrTail {
5315            module_id,
5316            tail: StderrTail {
5317                capture: match snapshot.capture {
5318                    CaptureState::Captured => StderrCaptureState::Captured,
5319                    CaptureState::Incomplete { reason } => {
5320                        StderrCaptureState::Incomplete { reason }
5321                    }
5322                    CaptureState::NotCaptured { reason } => {
5323                        StderrCaptureState::NotCaptured { reason }
5324                    }
5325                },
5326                entries: snapshot
5327                    .entries
5328                    .into_iter()
5329                    .map(|entry| match entry {
5330                        TailEntry::Line {
5331                            text,
5332                            truncated,
5333                            at_ms,
5334                        } => StderrTailEntry::Line {
5335                            text,
5336                            truncated,
5337                            at_ms,
5338                        },
5339                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5340                    })
5341                    .collect(),
5342                dropped_lines: snapshot.dropped_lines,
5343            },
5344        };
5345        Ok(vec![control_response_body_frame(
5346            &frame,
5347            &response,
5348            "ClientControlResponse::SupervisorStderrTail",
5349        )?])
5350    }
5351
5352    async fn handle_supervisor_terminals(
5353        &self,
5354        frame: Frame,
5355        module_id: String,
5356    ) -> Result<Vec<Frame>, RouterError> {
5357        let Some(module) = self.supervisor.get(&module_id) else {
5358            return Ok(vec![control_error_frame(
5359                &frame,
5360                "unknown_module",
5361                format!("module_id '{module_id}' is not supervised"),
5362            )?]);
5363        };
5364
5365        // The journal read runs on a blocking thread: it can be megabytes of
5366        // file I/O and must not occupy a runtime worker.
5367        let terminals = module
5368            .read_durable_terminal_history()
5369            .await
5370            .map_err(|error| {
5371                RouterError::backend(
5372                    0,
5373                    frame.header.corr,
5374                    format!("failed to read terminal history: {error}"),
5375                )
5376            })?;
5377        let response = ClientControlResponse::SupervisorTerminals {
5378            module_id,
5379            terminals,
5380        };
5381        Ok(vec![control_response_body_frame(
5382            &frame,
5383            &response,
5384            "ClientControlResponse::SupervisorTerminals",
5385        )?])
5386    }
5387
5388    fn handle_supervisor_routes(
5389        &self,
5390        frame: Frame,
5391        module_id: Option<String>,
5392    ) -> Result<Vec<Frame>, RouterError> {
5393        let modules = self
5394            .forwarding
5395            .route_census(module_id.as_deref())
5396            .map_err(RouterError::Forwarding)?
5397            .into_iter()
5398            .map(|(module_id, routes)| SupervisorRouteModule {
5399                module_id,
5400                routes: routes
5401                    .into_iter()
5402                    .map(|route| SupervisorRoute {
5403                        consumer: match route.principal {
5404                            Principal::Reserved { module_id } => {
5405                                SupervisorRouteConsumer::Reserved { module_id }
5406                            }
5407                            Principal::Direct | Principal::Unverified => {
5408                                SupervisorRouteConsumer::Direct {
5409                                    connection_id: route.goodbye_target.connection_id.get(),
5410                                }
5411                            }
5412                        },
5413                        age_ms: Instant::now()
5414                            .saturating_duration_since(route.bound_at)
5415                            .as_millis()
5416                            .try_into()
5417                            .unwrap_or(u64::MAX),
5418                        draining: route.draining,
5419                        drain_reason: route.drain_reason,
5420                    })
5421                    .collect(),
5422            })
5423            .collect();
5424        let response = ClientControlResponse::SupervisorRoutes { modules };
5425        Ok(vec![control_response_body_frame(
5426            &frame,
5427            &response,
5428            "ClientControlResponse::SupervisorRoutes",
5429        )?])
5430    }
5431
5432    async fn handle_supervisor_provenance(
5433        &self,
5434        frame: Frame,
5435        module_id: Option<String>,
5436    ) -> Result<Vec<Frame>, RouterError> {
5437        let mut selected = if let Some(module_id) = module_id {
5438            let Some(module) = self.supervisor.get(&module_id) else {
5439                return Ok(vec![control_error_frame(
5440                    &frame,
5441                    "unknown_module",
5442                    format!("module_id '{module_id}' is not supervised"),
5443                )?]);
5444            };
5445            vec![module]
5446        } else {
5447            self.supervisor.list()
5448        };
5449
5450        let mut modules = Vec::with_capacity(selected.len());
5451        for module in selected.drain(..) {
5452            let (status, observed_image) = module
5453                .status_and_running_image_agreement()
5454                .await
5455                .map_err(|err| {
5456                    RouterError::backend(
5457                        0,
5458                        frame.header.corr,
5459                        format!("failed to read supervisor status: {err}"),
5460                    )
5461                })?;
5462            let module_declared = self
5463                .registry
5464                .get_module(&status.module_id)
5465                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5466                .and_then(|registration| registration.manifest.provenance)
5467                .map(|build| ModuleDeclaredProvenance::Reported { build })
5468                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5469            #[cfg(test)]
5470            let running_image = match &self.provenance_probe_override {
5471                Some(result) => result.clone(),
5472                None => observed_image,
5473            };
5474            #[cfg(not(test))]
5475            let running_image = observed_image;
5476            modules.push(SupervisorModuleProvenance {
5477                module_id: status.module_id,
5478                module_declared,
5479                daemon_observed: SupervisorObservedProcess {
5480                    launch_nonce_source: status.launch_nonce_source,
5481                    pid: status.pid,
5482                    spawned_at_ms: status.spawned_at_ms,
5483                    spawned_from: status.spawned_from,
5484                    running_image,
5485                },
5486            });
5487        }
5488        let daemon = SupervisorDaemonProvenance {
5489            daemon_build: self.daemon_provenance.build.clone(),
5490            daemon_observed: DaemonObservedProcess {
5491                pid: self.daemon_provenance.pid,
5492                started_at_ms: self
5493                    .daemon_provenance
5494                    .start_clock
5495                    .map(|clock| clock.started_at_ms())
5496                    .or(self.daemon_provenance.started_at_ms),
5497                running_image: self
5498                    .daemon_provenance
5499                    .probe
5500                    .observe(
5501                        self.daemon_provenance.pid,
5502                        self.daemon_provenance.executable_path.as_deref(),
5503                        self.daemon_provenance.executable_identity.clone(),
5504                        self.daemon_provenance.process_start_time,
5505                    )
5506                    .await,
5507            },
5508        };
5509        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5510        Ok(vec![control_response_body_frame(
5511            &frame,
5512            &response,
5513            "ClientControlResponse::SupervisorProvenance",
5514        )?])
5515    }
5516
5517    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5518        self.refresh_capability_requirements();
5519        let generation = self
5520            .registry
5521            .generation()
5522            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5523        let modules = self
5524            .supervisor
5525            .list()
5526            .into_iter()
5527            .map(|module| {
5528                let status = module.status_for_control("health").map_err(|err| {
5529                    RouterError::backend(
5530                        0,
5531                        frame.header.corr,
5532                        format!("failed to read supervisor health: {err}"),
5533                    )
5534                })?;
5535                let module_id = status.module_id;
5536                let capability_detail = self
5537                    .capability_evaluator
5538                    .required_problem_detail(&module_id);
5539                Ok(SupervisorHealthEntry {
5540                    module_id,
5541                    status: status.health.status,
5542                    detail: append_capability_problem_detail(
5543                        status.health.detail,
5544                        capability_detail,
5545                    ),
5546                    metrics: status.health.metrics,
5547                    consecutive_failures: status.health.consecutive_failures,
5548                    late_answer_count: status.health.late_answer_count,
5549                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5550                    last_action: status.health.last_action,
5551                    last_action_ms: status.health.last_action_ms,
5552                    last_probe_ms: status.health.last_probe_ms,
5553                })
5554            })
5555            .collect::<Result<Vec<_>, RouterError>>()?;
5556        let response = ClientControlResponse::SupervisorHealth {
5557            generation,
5558            modules,
5559        };
5560        Ok(vec![control_response_body_frame(
5561            &frame,
5562            &response,
5563            "ClientControlResponse::SupervisorHealth",
5564        )?])
5565    }
5566
5567    async fn handle_supervisor_restart(
5568        &self,
5569        frame: Frame,
5570        module_id: String,
5571        drain_timeout_ms: Option<u64>,
5572        requester: Principal,
5573    ) -> Result<Vec<Frame>, RouterError> {
5574        let operation_lock = self.supervisor.operation_lock();
5575        let _operation_guard = operation_lock.lock().await;
5576        let Some(module) = self.supervisor.get(&module_id) else {
5577            return Ok(vec![control_error_frame(
5578                &frame,
5579                "unknown_module",
5580                format!("module_id '{module_id}' is not supervised"),
5581            )?]);
5582        };
5583
5584        self.route_outages.mark_operator_action(&module_id);
5585        if let Err(err) = module
5586            .restart_requested(drain_timeout_ms, Some(requester))
5587            .await
5588        {
5589            self.route_outages
5590                .operator_action_ended_unrefused(&module_id);
5591            let (code, message) = match err {
5592                crate::supervise::SuperviseError::Disabled { .. } => {
5593                    ("module_disabled", err.to_string())
5594                }
5595                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5596                    ("swap_in_progress", err.to_string())
5597                }
5598                _ => (
5599                    "target_unavailable",
5600                    format!("failed to restart module_id '{module_id}': {err}"),
5601                ),
5602            };
5603            return Ok(vec![control_error_frame(&frame, code, message)?]);
5604        }
5605
5606        let response = ClientControlResponse::SupervisorAck {
5607            module_id,
5608            applied: true,
5609        };
5610        Ok(vec![control_response_body_frame(
5611            &frame,
5612            &response,
5613            "ClientControlResponse::SupervisorAck",
5614        )?])
5615    }
5616
5617    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
5618    /// when the old process has finished draining: a caller whose own lane
5619    /// rides the old process must get its reply before that drain waits on it.
5620    async fn handle_supervisor_swap(
5621        &self,
5622        frame: Frame,
5623        module_id: String,
5624        ready_timeout_ms: Option<u64>,
5625        requester: Principal,
5626    ) -> Result<Vec<Frame>, RouterError> {
5627        // The daemon-wide operation lock is held only to resolve the handle,
5628        // not across the swap. The swap can take its whole readiness budget,
5629        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
5630        // holding it here would park an operator's stop behind the swap it is
5631        // meant to abort. A rescan or stop that reaches the module during the
5632        // swap is served by the swap itself (see `supervise_swap`).
5633        let module = {
5634            let operation_lock = self.supervisor.operation_lock();
5635            let _operation_guard = operation_lock.lock().await;
5636            self.supervisor.get(&module_id)
5637        };
5638        let Some(module) = module else {
5639            return Ok(vec![control_error_frame(
5640                &frame,
5641                "unknown_module",
5642                format!("module_id '{module_id}' is not supervised"),
5643            )?]);
5644        };
5645
5646        self.route_outages.mark_operator_action(&module_id);
5647        if let Err(err) = module
5648            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5649            .await
5650        {
5651            self.route_outages
5652                .operator_action_ended_unrefused(&module_id);
5653            use crate::supervise::SuperviseError;
5654            let message = err.to_string();
5655            let error = match err {
5656                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5657                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5658                    code: "swap_refused".to_string(),
5659                    message,
5660                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5661                },
5662                SuperviseError::SwapFailed {
5663                    arm,
5664                    candidate_exit,
5665                    ..
5666                } => ErrorBody {
5667                    code: "swap_failed".to_string(),
5668                    message,
5669                    detail: Some(serde_json::json!({
5670                        "arm": arm.as_str(),
5671                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5672                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5673                    })),
5674                },
5675                _ => ErrorBody::new(
5676                    "target_unavailable",
5677                    format!("failed to swap module_id '{module_id}': {message}"),
5678                ),
5679            };
5680            return Ok(vec![control_error_body_frame(&frame, error)?]);
5681        }
5682        // A completed swap kept the incumbent serving until cutover, so it
5683        // usually opened no outage; a mark left behind would make the next,
5684        // unrelated outage read as requested.
5685        self.route_outages
5686            .operator_action_ended_unrefused(&module_id);
5687
5688        let response = ClientControlResponse::SupervisorAck {
5689            module_id,
5690            applied: true,
5691        };
5692        Ok(vec![control_response_body_frame(
5693            &frame,
5694            &response,
5695            "ClientControlResponse::SupervisorAck",
5696        )?])
5697    }
5698
5699    async fn handle_supervisor_reload(
5700        &self,
5701        frame: Frame,
5702        module_id: String,
5703        requester: Principal,
5704    ) -> Result<Vec<Frame>, RouterError> {
5705        let operation_lock = self.supervisor.operation_lock();
5706        let _operation_guard = operation_lock.lock().await;
5707        let Some(module) = self.supervisor.get(&module_id) else {
5708            return Ok(vec![control_error_frame(
5709                &frame,
5710                "unknown_module",
5711                format!("module_id '{module_id}' is not supervised"),
5712            )?]);
5713        };
5714
5715        self.route_outages.mark_operator_action(&module_id);
5716        if let Err(err) = module.reload_requested(Some(requester)).await {
5717            self.route_outages
5718                .operator_action_ended_unrefused(&module_id);
5719            let (code, message) = match err {
5720                crate::supervise::SuperviseError::Disabled { .. } => {
5721                    ("module_disabled", err.to_string())
5722                }
5723                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5724                    ("swap_in_progress", err.to_string())
5725                }
5726                _ => (
5727                    "reload_failed",
5728                    format!("failed to reload module_id '{module_id}': {err}"),
5729                ),
5730            };
5731            return Ok(vec![control_error_frame(&frame, code, message)?]);
5732        }
5733
5734        let response = ClientControlResponse::SupervisorAck {
5735            module_id,
5736            applied: true,
5737        };
5738        Ok(vec![control_response_body_frame(
5739            &frame,
5740            &response,
5741            "ClientControlResponse::SupervisorAck",
5742        )?])
5743    }
5744
5745    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
5746    /// the configuration comes from and whether to apply it; every mode then
5747    /// runs the same comparison, so a preview describes the apply it previews.
5748    async fn handle_supervisor_rescan(
5749        &self,
5750        frame: Frame,
5751        mode: RescanMode,
5752        requester: Principal,
5753    ) -> Result<Vec<Frame>, RouterError> {
5754        let preview = mode.previews();
5755        let Some(context) = self.rescan.clone() else {
5756            return Ok(vec![control_error_frame(
5757                &frame,
5758                "rescan_unavailable",
5759                "the daemon was not started with a reloadable config path".to_string(),
5760            )?]);
5761        };
5762
5763        let operation_lock = self.supervisor.operation_lock();
5764        let _operation_guard = operation_lock.lock().await;
5765        // Loaded under the operation lock, so no other rescan can change the
5766        // staged configuration between a checked apply's comparison and the
5767        // apply itself.
5768        let loaded = match self.load_rescan_config(&mode, &context) {
5769            Ok(config) => config,
5770            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5771        };
5772        // `load` reports a missing file as Ok(None), which is correct at boot
5773        // (no config, nothing to supervise) and catastrophic here: rescan treats
5774        // "not in the config" as "remove it", so an absent file would read as an
5775        // empty module list and retire the entire running fleet. An editor
5776        // writing via write-new-then-rename, or a half-finished edit, is enough
5777        // to open that window. Refuse instead: a config that cannot be read
5778        // carries no instruction to remove anything.
5779        let Some((config, activity_config)) = loaded else {
5780            return Ok(vec![control_error_frame(
5781                &frame,
5782                "invalid_daemon_config",
5783                format!(
5784                    "daemon config not found at {}; refusing to rescan (an absent config would \
5785                     retire every supervised module)",
5786                    context.config_path.display()
5787                ),
5788            )?]);
5789        };
5790        let (
5791            configured_port,
5792            storage_config,
5793            admission_facts_carrier_module_id,
5794            admission_facts_targets,
5795            scope_authority_owners,
5796            privileged_operator_prompt,
5797            modules,
5798            reserved_capabilities,
5799        ) = (
5800            config.port,
5801            config.storage,
5802            config.admission_facts_carrier_module_id,
5803            config.admission_facts_targets,
5804            config.scope_authority_owners,
5805            config.privileged_operator_prompt,
5806            config.modules,
5807            config.reserved_capabilities,
5808        );
5809
5810        // Collect the sections rescan cannot apply, so the REPLY carries them.
5811        //
5812        // The warning below has always been correct and has always gone only to
5813        // the journal -- addressed to whoever reads logs, while the person who
5814        // just edited the config is looking at the CLI. Naming each section
5815        // individually rather than setting a flag: "something outside modules
5816        // changed" sends the operator back to diffing their own file, which is
5817        // the work this is meant to save.
5818        let mut restart_required = Vec::new();
5819        for section in RestartRequiredSection::ALL {
5820            let changed = match section {
5821                RestartRequiredSection::Port => configured_port != context.configured_port,
5822                RestartRequiredSection::Storage => storage_config != context.storage_config,
5823                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5824                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5825                }
5826                RestartRequiredSection::AdmissionFactsTargets => {
5827                    admission_facts_targets != context.admission_facts_targets
5828                }
5829                RestartRequiredSection::ScopeAuthorityOwners => {
5830                    scope_authority_owners != context.scope_authority_owners
5831                }
5832                RestartRequiredSection::PrivilegedOperatorPrompt => {
5833                    privileged_operator_prompt != context.privileged_operator_prompt
5834                }
5835            };
5836            if changed {
5837                restart_required.push(section.label().to_string());
5838            }
5839        }
5840        if !restart_required.is_empty() {
5841            warn!(
5842                config_path = %context.config_path.display(),
5843                sections = %restart_required.join(", "),
5844                "daemon config changed outside the modules section; restart the daemon to apply those changes"
5845            );
5846        }
5847
5848        for configured in &modules {
5849            if let Err(err) = validate_spec(&configured.module_spec()) {
5850                return Ok(vec![control_error_frame(
5851                    &frame,
5852                    "invalid_daemon_config",
5853                    format!("supervisor rescan rejected daemon config: {err}"),
5854                )?]);
5855            }
5856        }
5857
5858        let configured_capabilities = modules
5859            .iter()
5860            .map(|module| (module.module_id.clone(), module.enabled))
5861            .collect::<Vec<_>>();
5862        let preview_capability_warnings = if preview {
5863            let (_, registrations) = self.runtime_capability_snapshot()?;
5864            let current_modules = self
5865                .supervisor
5866                .list()
5867                .into_iter()
5868                .map(|module| module.module_id().to_string())
5869                .collect::<BTreeSet<_>>();
5870            let resulting_modules = configured_capabilities.clone();
5871            let removed = current_modules
5872                .into_iter()
5873                .filter(|module_id| {
5874                    !resulting_modules
5875                        .iter()
5876                        .any(|(configured_id, _)| configured_id == module_id)
5877                })
5878                .collect::<Vec<_>>();
5879            // A candidate preview also names providers the candidate disables.
5880            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
5881            let disabled = match &mode {
5882                RescanMode::Candidate(_) => resulting_modules
5883                    .iter()
5884                    .filter(|(_, enabled)| !enabled)
5885                    .map(|(module_id, _)| module_id.clone())
5886                    .collect::<Vec<_>>(),
5887                _ => Vec::new(),
5888            };
5889            self.capability_evaluator.preview_warnings(
5890                resulting_modules,
5891                &removed,
5892                &disabled,
5893                &registrations,
5894            )
5895        } else {
5896            Vec::new()
5897        };
5898        let reconciled = match &mode {
5899            RescanMode::Candidate(candidate) => self
5900                .preview_candidate(&context, modules, candidate)
5901                .map(|result| (result, Vec::new())),
5902            _ => {
5903                self.reconcile_supervised_modules(
5904                    &context.supervisor,
5905                    modules,
5906                    preview,
5907                    requester.clone(),
5908                )
5909                .await
5910            }
5911        };
5912        let (result, changed) = match reconciled {
5913            Ok(result) => result,
5914            Err(message) => {
5915                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5916            }
5917        };
5918        if !preview {
5919            if let Some(ledger) = &self.activity {
5920                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
5921                ledger.config_changed(&result.added, &result.removed, &changed, requester);
5922            }
5923            if let Err(code) = self.configure_activity(activity_config).await {
5924                warn!(
5925                    code,
5926                    "activity retention could not be applied during rescan"
5927                );
5928            }
5929            self.capability_evaluator
5930                .configure(configured_capabilities, reserved_capabilities);
5931            self.capability_evaluator.wake_deadline_loop();
5932            self.refresh_capability_requirements();
5933        }
5934        let mut result = result;
5935        result.restart_required = restart_required;
5936        result.capability_warnings = preview_capability_warnings;
5937        let response = ClientControlResponse::SupervisorRescan { result };
5938        Ok(vec![control_response_body_frame(
5939            &frame,
5940            &response,
5941            "ClientControlResponse::SupervisorRescan",
5942        )?])
5943    }
5944
5945    async fn handle_supervisor_release_reserved(
5946        &self,
5947        frame: Frame,
5948        module_id: String,
5949    ) -> Result<Vec<Frame>, RouterError> {
5950        let Some(context) = self.rescan.clone() else {
5951            return Ok(vec![control_error_frame(
5952                &frame,
5953                "release_unavailable",
5954                "reserved-id release requires a daemon started with a reloadable config path",
5955            )?]);
5956        };
5957        let operation_lock = self.supervisor.operation_lock();
5958        let _operation_guard = operation_lock.lock().await;
5959        let loaded = match crate::daemon_config::load(&context.config_path) {
5960            Ok(Some(config)) => config,
5961            Ok(None) => {
5962                return Ok(vec![control_error_frame(
5963                    &frame,
5964                    "invalid_daemon_config",
5965                    format!(
5966                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5967                        context.config_path.display()
5968                    ),
5969                )?])
5970            }
5971            Err(err) => {
5972                return Ok(vec![control_error_frame(
5973                    &frame,
5974                    "invalid_daemon_config",
5975                    format!("unable to verify reserved-id release against daemon config: {err}"),
5976                )?])
5977            }
5978        };
5979        if loaded
5980            .modules
5981            .iter()
5982            .any(|configured| configured.module_id == module_id)
5983        {
5984            return Ok(vec![control_error_frame(
5985                &frame,
5986                "reserved_module_configured",
5987                format!(
5988                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5989                ),
5990            )?]);
5991        }
5992        if !self.supervisor.release_retained_reserved_gate(&module_id) {
5993            return Ok(vec![control_error_frame(
5994                &frame,
5995                "reserved_gate_not_retained",
5996                format!(
5997                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5998                ),
5999            )?]);
6000        }
6001
6002        let response = ClientControlResponse::SupervisorAck {
6003            module_id,
6004            applied: true,
6005        };
6006        Ok(vec![control_response_body_frame(
6007            &frame,
6008            &response,
6009            "ClientControlResponse::SupervisorAck",
6010        )?])
6011    }
6012
6013    /// Compare the configured module set with the supervised one, changing
6014    /// nothing. The staged spec each supervised module holds is what the
6015    /// configured entry is compared with.
6016    fn plan_rescan(
6017        &self,
6018        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6019    ) -> Result<RescanPlan, String> {
6020        let mut current = BTreeMap::new();
6021        for module in self.supervisor.list() {
6022            let (spec, health) = module.configuration().map_err(|err| {
6023                format!(
6024                    "failed to read configuration for module_id '{}': {err}",
6025                    module.module_id()
6026                )
6027            })?;
6028            let enabled = module
6029                .status()
6030                .map_err(|err| {
6031                    format!(
6032                        "failed to read status for module_id '{}': {err}",
6033                        module.module_id()
6034                    )
6035                })?
6036                .enabled;
6037            current.insert(
6038                module.module_id().to_string(),
6039                (module, spec, health, enabled),
6040            );
6041        }
6042        let configured = configured_modules
6043            .into_iter()
6044            .map(|module| (module.module_id.clone(), module))
6045            .collect::<BTreeMap<_, _>>();
6046
6047        let added = configured
6048            .keys()
6049            .filter(|module_id| !current.contains_key(*module_id))
6050            .cloned()
6051            .collect::<Vec<_>>();
6052        let removed = current
6053            .keys()
6054            .filter(|module_id| !configured.contains_key(*module_id))
6055            .cloned()
6056            .collect::<Vec<_>>();
6057        let mut changed_pending_reload = Vec::new();
6058        let mut configuration_changes = BTreeSet::new();
6059        let mut enabled_changes = BTreeSet::new();
6060        let mut unchanged = 0_u32;
6061
6062        for (module_id, configured_module) in &configured {
6063            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6064            else {
6065                continue;
6066            };
6067            // Compare the whole launch spec so a future launch field cannot
6068            // accidentally become a live-only policy change. Health is stored
6069            // separately and applies live without replacing the process.
6070            let launch_changed = *current_spec != configured_module.module_spec();
6071            let configuration_changed =
6072                launch_changed || *current_health != configured_module.health;
6073            let enabled_changed = *current_enabled != configured_module.enabled;
6074            if configuration_changed {
6075                configuration_changes.insert(module_id.clone());
6076            }
6077            if launch_changed {
6078                changed_pending_reload.push(module_id.clone());
6079            }
6080            if enabled_changed {
6081                enabled_changes.insert(module_id.clone());
6082            }
6083            if !configuration_changed && !enabled_changed {
6084                unchanged = unchanged.saturating_add(1);
6085            }
6086        }
6087        Ok(RescanPlan {
6088            current,
6089            configured,
6090            added,
6091            removed,
6092            changed_pending_reload,
6093            configuration_changes,
6094            enabled_changes,
6095            unchanged,
6096        })
6097    }
6098
6099    /// Reconcile the running module set against the configured one.
6100    ///
6101    /// With `preview` set, the diff is computed and returned WITHOUT applying any
6102    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
6103    /// deliberately shares this function with the executing path rather than
6104    /// computing the same diff somewhere else -- two implementations of one
6105    /// decision agree until they do not, and the whole value of a preview is that
6106    /// it describes the operation that will actually run.
6107    async fn reconcile_supervised_modules(
6108        &self,
6109        supervisor: &Supervisor,
6110        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6111        preview: bool,
6112        requester: Principal,
6113    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6114        let plan = self.plan_rescan(configured_modules)?;
6115
6116        // Everything above this point is pure computation over two snapshots.
6117        // Everything below MUTATES. The preview returns here so the boundary is a
6118        // single early return rather than a condition repeated at each mutation
6119        // site, where one missed guard would apply part of a change the caller was
6120        // told would not happen.
6121        if preview {
6122            return Ok((plan.preview_result(), Vec::new()));
6123        }
6124        let changed = plan
6125            .configuration_changes
6126            .union(&plan.enabled_changes)
6127            .cloned()
6128            .collect::<Vec<_>>();
6129        let RescanPlan {
6130            current,
6131            configured,
6132            added,
6133            removed,
6134            changed_pending_reload,
6135            configuration_changes,
6136            enabled_changes,
6137            unchanged,
6138        } = plan;
6139        // The staged generation moves before the first change rather than
6140        // after the last, so a rescan that fails partway still invalidates
6141        // every earlier `supervisor.rescan_checked` preview: part of its diff
6142        // may have run.
6143        if !(added.is_empty()
6144            && removed.is_empty()
6145            && configuration_changes.is_empty()
6146            && enabled_changes.is_empty())
6147        {
6148            if let Some(context) = &self.rescan {
6149                context
6150                    .generation
6151                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6152            }
6153        }
6154
6155        for module_id in &removed {
6156            let module = &current
6157                .get(module_id)
6158                .expect("removed module came from current supervisor state")
6159                .0;
6160            module
6161                .retire_requested(Some(requester.clone()))
6162                .await
6163                .map_err(|err| {
6164                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
6165                })?;
6166            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
6167            //
6168            // `handle_route_open` resolves an absent module in three steps:
6169            // registry, then supervisor status, then tombstone. Retiring first
6170            // opens a window where ALL THREE ARE ABSENT -- the registry entry
6171            // went with the teardown above, the supervisor entry went with
6172            // `retire`, and the tombstone does not exist yet -- so a route.open
6173            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
6174            // it") for a module that was deliberately removed and whose caller
6175            // should get `module_removed` (TERMINAL, carrying a removal age).
6176            //
6177            // Writing the tombstone first closes it: during the window the
6178            // supervisor entry still answers, so the caller gets
6179            // `target_unavailable` -- retryable, and TRUE, because the module
6180            // is mid-teardown. After both statements it is `module_removed`.
6181            // No instant remains where a removed module reads as one that
6182            // never existed.
6183            //
6184            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6185            // the absence of a test beside a fix invites deletion: these are
6186            // two sync statements with no await between them, so reaching the
6187            // window needs a second worker thread to land exactly between them
6188            // and there is no hook to force it. MEASURED: the 25 daemon_config
6189            // tests pass identically with the old order and the new one, so
6190            // the existing suite cannot see this and a green run is not
6191            // evidence either way. What the suite does hold is the
6192            // post-condition -- a removed module answers `module_removed` --
6193            // which this preserves.
6194            //
6195            // Found by an Athena panel reading the shipped tree against a
6196            // design note (2026-09-19), as the one concrete instance of that
6197            // note's class that survived contact with source. Direction is
6198            // benign: retryable where terminal was intended, never the reverse.
6199            self.supervisor.record_rescan_removal(module_id);
6200            self.supervisor.retire(module_id);
6201            self.route_outages.forget(module_id);
6202        }
6203
6204        for module_id in configured.keys() {
6205            let Some((module, _, _, _)) = current.get(module_id) else {
6206                continue;
6207            };
6208            let configured_module = configured
6209                .get(module_id)
6210                .expect("configured module id came from configured map");
6211            if configuration_changes.contains(module_id) {
6212                module
6213                    .update_configuration(
6214                        configured_module.module_spec(),
6215                        configured_module.health.clone(),
6216                        configured_module.drain_timeout_ms,
6217                    )
6218                    .await
6219                    .map_err(|err| {
6220                        format!(
6221                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6222                        )
6223                    })?;
6224            }
6225            if enabled_changes.contains(module_id) {
6226                // A rescan that starts or stops a module applies an operator's
6227                // edit to the config, so the resulting outage was asked for.
6228                self.route_outages.mark_operator_action(module_id);
6229                module
6230                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6231                    .await
6232                    .map_err(|err| {
6233                        self.route_outages.operator_action_ended_unrefused(module_id);
6234                        format!(
6235                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6236                            configured_module.enabled
6237                        )
6238                    })?;
6239            }
6240        }
6241
6242        for module_id in &added {
6243            let configured_module = configured
6244                .get(module_id)
6245                .expect("added module id came from configured map");
6246            supervisor
6247                .supervise_configured_requested(
6248                    configured_module.module_spec(),
6249                    configured_module.enabled,
6250                    configured_module.health.clone(),
6251                    configured_module.drain_timeout_ms,
6252                    configured_module.restart,
6253                    Some(requester.clone()),
6254                )
6255                .map_err(|err| {
6256                    format!("failed to add module_id '{module_id}' during rescan: {err}")
6257                })?;
6258        }
6259
6260        Ok((
6261            SupervisorRescanResult {
6262                added,
6263                removed,
6264                changed_pending_reload,
6265                enabled_changes: enabled_changes.iter().cloned().collect(),
6266                unchanged,
6267                preview: false,
6268                // Filled by the caller, which is the only layer that can see the
6269                // previous config to diff against.
6270                restart_required: Vec::new(),
6271                capability_warnings: Vec::new(),
6272                module_actions: Vec::new(),
6273                earlier_pending: Vec::new(),
6274                expect: None,
6275            },
6276            changed,
6277        ))
6278    }
6279
6280    async fn handle_supervisor_set_enabled(
6281        &self,
6282        frame: Frame,
6283        module_id: String,
6284        enabled: bool,
6285        requester: Principal,
6286    ) -> Result<Vec<Frame>, RouterError> {
6287        let operation_lock = self.supervisor.operation_lock();
6288        let _operation_guard = operation_lock.lock().await;
6289        let Some(module) = self.supervisor.get(&module_id) else {
6290            return Ok(vec![control_error_frame(
6291                &frame,
6292                "unknown_module",
6293                format!("module_id '{module_id}' is not supervised"),
6294            )?]);
6295        };
6296
6297        // Enabling counts as well as disabling: a module an operator starts
6298        // is refused until it registers, and that wait was asked for.
6299        self.route_outages.mark_operator_action(&module_id);
6300        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6301            Ok(applied) => applied,
6302            Err(err) => {
6303                self.route_outages
6304                    .operator_action_ended_unrefused(&module_id);
6305                return Ok(vec![control_error_frame(
6306                    &frame,
6307                    "target_unavailable",
6308                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6309                )?]);
6310            }
6311        };
6312        if !applied {
6313            // Already in the requested state: nothing was made unavailable,
6314            // so the mark must not outlive this request.
6315            self.route_outages
6316                .operator_action_ended_unrefused(&module_id);
6317        }
6318
6319        self.capability_evaluator.wake_deadline_loop();
6320        self.refresh_capability_requirements();
6321        let response = ClientControlResponse::SupervisorAck { module_id, applied };
6322        Ok(vec![control_response_body_frame(
6323            &frame,
6324            &response,
6325            "ClientControlResponse::SupervisorAck",
6326        )?])
6327    }
6328
6329    async fn handle_supervisor_health_probe(
6330        &self,
6331        frame: Frame,
6332        module_id: String,
6333    ) -> Result<Vec<Frame>, RouterError> {
6334        self.refresh_capability_requirements();
6335        let Some(registration) = self
6336            .registry
6337            .get_module(&module_id)
6338            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6339        else {
6340            return Ok(vec![control_error_frame(
6341                &frame,
6342                "unknown_module",
6343                format!("module_id '{module_id}' is not registered"),
6344            )?]);
6345        };
6346
6347        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
6348        // named for it. Making `module_registration_grants_op` return false
6349        // unconditionally reddens five tests, and every one is named for something
6350        // else -- capability relay, probe/bind demultiplexing, supervision-only
6351        // probing. They exercise a successful advertisement check on the way to their
6352        // own subject.
6353        //
6354        // Real protection, fragile in a specific way: narrowing any of those tests to
6355        // focus on its stated subject would silently remove coverage nobody knows
6356        // they are carrying. Recorded here rather than as a sixth test, because the
6357        // useful fact is WHICH tests hold the guard up -- a new test would add
6358        // coverage without telling the next person what the existing ones quietly do.
6359        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6360        {
6361            return Ok(vec![control_error_frame(
6362                &frame,
6363                "health_not_advertised",
6364                format!("module_id '{module_id}' did not advertise health.check"),
6365            )?]);
6366        }
6367
6368        let deadline = Instant::now() + self.health_probe_timeout;
6369        let pending = match self.forwarding.begin_module_control_rpc_for(
6370            &module_id,
6371            MODULE_CONTROL_OP_HEALTH_CHECK,
6372            deadline,
6373        ) {
6374            Ok(pending) => pending,
6375            Err(err) => {
6376                return Ok(vec![control_error_frame(
6377                    &frame,
6378                    forwarding_error_code(&err),
6379                    err.to_string(),
6380                )?])
6381            }
6382        };
6383
6384        let PendingModuleControlRpc {
6385            endpoint,
6386            module_sink,
6387            negotiated_ver,
6388            corr: probe_corr,
6389            receiver,
6390        } = pending;
6391        let mut guard =
6392            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6393        let probe_body =
6394            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6395                RouterError::backend(
6396                    0,
6397                    frame.header.corr,
6398                    format!("failed to encode health.check request: {err}"),
6399                )
6400            })?;
6401        let probe_frame = Frame::build_with_version(
6402            negotiated_ver,
6403            FrameType::Request,
6404            control_flags(),
6405            0,
6406            0,
6407            probe_corr,
6408            probe_body,
6409        )
6410        .map_err(RouterError::FrameBuild)?;
6411
6412        // Queue admission and the reply share one deadline, so a module that
6413        // stops reading cannot park the requesting connection's reader forever.
6414        match timeout_at(deadline, module_sink.send(probe_frame)).await {
6415            Ok(Ok(())) => {}
6416            Ok(Err(err)) => {
6417                return Ok(vec![control_error_frame(
6418                    &frame,
6419                    "target_unavailable",
6420                    err.to_string(),
6421                )?]);
6422            }
6423            Err(_) => {
6424                return Ok(vec![control_error_frame(
6425                    &frame,
6426                    "module_timeout",
6427                    format!(
6428                        "module_id '{module_id}' did not answer health.check within {:?}",
6429                        self.health_probe_timeout
6430                    ),
6431                )?]);
6432            }
6433        }
6434
6435        match timeout_at(deadline, receiver).await {
6436            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6437                guard.disarm();
6438                let Some(report) = response.health_report() else {
6439                    return Ok(vec![control_error_frame(
6440                        &frame,
6441                        "invalid_control_body",
6442                        "health.check RPC returned a non-health response",
6443                    )?]);
6444                };
6445                // Metrics go out whole here. The supervisor's cached snapshot
6446                // caps this blob (see truncate_health_metrics), and this path
6447                // exists precisely to answer without that cap -- so applying it
6448                // here would leave no way to see what the cached view drops.
6449                let HealthReport {
6450                    status,
6451                    detail,
6452                    metrics,
6453                } = report;
6454                let capability_detail = self
6455                    .capability_evaluator
6456                    .required_problem_detail(&module_id);
6457                let response = ClientControlResponse::SupervisorHealthProbe {
6458                    module_id,
6459                    status,
6460                    detail: append_capability_problem_detail(detail, capability_detail),
6461                    metrics,
6462                };
6463                Ok(vec![control_response_body_frame(
6464                    &frame,
6465                    &response,
6466                    "ClientControlResponse::SupervisorHealthProbe",
6467                )?])
6468            }
6469            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6470                guard.disarm();
6471                Ok(vec![control_error_body_frame(&frame, body)?])
6472            }
6473            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6474                guard.disarm();
6475                Ok(vec![control_error_frame(
6476                    &frame,
6477                    "target_unavailable",
6478                    message,
6479                )?])
6480            }
6481            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6482                guard.disarm();
6483                Ok(vec![control_error_frame(
6484                    &frame,
6485                    "invalid_control_body",
6486                    message,
6487                )?])
6488            }
6489            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6490                guard.disarm();
6491                Ok(vec![control_error_frame(
6492                    &frame,
6493                    "invalid_control_body",
6494                    format!("expected module-control op '{expected}', got '{actual}'"),
6495                )?])
6496            }
6497            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6498                guard.disarm();
6499                Ok(vec![control_error_frame(
6500                    &frame,
6501                    "module_timeout",
6502                    format!(
6503                        "module_id '{module_id}' answered health.check after {:?}",
6504                        self.health_probe_timeout
6505                    ),
6506                )?])
6507            }
6508            Ok(Err(_)) => Ok(vec![control_error_frame(
6509                &frame,
6510                "target_unavailable",
6511                "health.check waiter was canceled before the module responded",
6512            )?]),
6513            Err(_) => Ok(vec![control_error_frame(
6514                &frame,
6515                "module_timeout",
6516                format!(
6517                    "module_id '{module_id}' did not answer health.check within {:?}",
6518                    self.health_probe_timeout
6519                ),
6520            )?]),
6521        }
6522    }
6523
6524    fn supervisor_status(
6525        &self,
6526        module_id: &str,
6527        corr: u64,
6528    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6529        self.supervisor
6530            .get(module_id)
6531            .map(|module| {
6532                let warming = module.is_warming_for_control("status").map_err(|err| {
6533                    RouterError::backend(
6534                        0,
6535                        corr,
6536                        format!(
6537                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
6538                        ),
6539                    )
6540                })?;
6541                module.status_for_control("status").map_err(|err| {
6542                    RouterError::backend(
6543                        0,
6544                        corr,
6545                        format!(
6546                            "failed to read supervisor status for module_id '{module_id}': {err}"
6547                        ),
6548                    )
6549                }).map(|status| (status, warming))
6550            })
6551            .transpose()
6552    }
6553
6554    fn guard_module_control_op(
6555        &self,
6556        frame: &Frame,
6557        module_id: &str,
6558        op: &str,
6559    ) -> Result<Option<Frame>, RouterError> {
6560        if self.module_grants_op(module_id, op, frame.header.corr)? {
6561            return Ok(None);
6562        }
6563
6564        Ok(Some(control_error_frame(
6565            frame,
6566            "op_not_allowed",
6567            format!("module_id '{module_id}' did not grant control op '{op}'"),
6568        )?))
6569    }
6570
6571    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6572        let Some(registration) = self
6573            .registry
6574            .get_module(module_id)
6575            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6576        else {
6577            return Ok(false);
6578        };
6579        Ok(module_registration_grants_op(&registration.control_ops, op))
6580    }
6581
6582    fn handle_status_update(
6583        &self,
6584        endpoint: ModuleEndpointId,
6585        frame: Frame,
6586    ) -> Result<Vec<Frame>, RouterError> {
6587        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6588            Ok(update) => update,
6589            Err(err) => {
6590                // Forward-compat: a newer module may push a channel-0 op this subc
6591                // version doesn't know. The control contract says unknown push ops
6592                // are IGNORED, never answered with an error. Only a malformed body
6593                // for an op we DO know is a real error worth surfacing.
6594                if is_known_module_push_op(&frame.body) {
6595                    return Ok(vec![control_error_frame(
6596                        &frame,
6597                        "invalid_control_body",
6598                        format!("malformed module control push body: {err}"),
6599                    )?]);
6600                }
6601                return Ok(Vec::new());
6602            }
6603        };
6604
6605        match update {
6606            ModuleControlPush::RouteStatus {
6607                route_channel,
6608                route_epoch,
6609                status,
6610            } => {
6611                self.forwarding
6612                    .cache_status(endpoint, route_channel, route_epoch, status)
6613                    .map_err(RouterError::Forwarding)?;
6614            }
6615        }
6616        Ok(Vec::new())
6617    }
6618
6619    fn handle_route_poll(
6620        &self,
6621        ctx: &RouteCtx,
6622        frame: Frame,
6623        route_channel: u16,
6624        route_epoch: u32,
6625        kind: PollKind,
6626    ) -> Result<Vec<Frame>, RouterError> {
6627        let snapshot = self
6628            .forwarding
6629            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6630            .map_err(RouterError::Forwarding)?;
6631        let response = match (kind, snapshot) {
6632            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6633                ClientControlResponse::RoutePoll {
6634                    route_channel,
6635                    route_epoch,
6636                    status,
6637                    live: None,
6638                }
6639            }
6640            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6641                route_channel,
6642                route_epoch,
6643                status: None,
6644                live: None,
6645            },
6646            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6647                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
6648                // is what makes reporting `true` correct rather than a
6649                // confident guess. `process_live` returns None only when the
6650                // module id has no supervisor snapshot at all -- an
6651                // externally-started module the daemon did not spawn -- and
6652                // for those the supervisor has no opinion to offer, ever. It
6653                // is never None for a supervised module in an unknown state:
6654                // a supervised module always has a snapshot, and the answer
6655                // comes from `state == Running && process_alive`.
6656                //
6657                // The route is Bound, so the module completed a HELLO on a
6658                // live connection; "the process this route points at is
6659                // running" is therefore attested by the binding rather than
6660                // assumed. Reporting `false` for an unsupervised module would
6661                // be the actual lie -- it would tell a client its healthy
6662                // route is dead because the daemon does not manage the
6663                // process.
6664                //
6665                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
6666                // module whose liveness is genuinely unknown, e.g. a snapshot
6667                // that has not been populated yet -- THIS DEFAULT BECOMES
6668                // WRONG and must split: unsupervised stays true, unknown
6669                // becomes null so the client can tell the two apart. The
6670                // response field is already `Option<bool>`, so the wire can
6671                // carry that distinction today.
6672                let live = self
6673                    .process_liveness
6674                    .as_ref()
6675                    .and_then(|source| source.process_live(&module_id))
6676                    .unwrap_or(true);
6677                ClientControlResponse::RoutePoll {
6678                    route_channel,
6679                    route_epoch,
6680                    status: None,
6681                    live: Some(live),
6682                }
6683            }
6684            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6685                route_channel,
6686                route_epoch,
6687                status: None,
6688                live: Some(false),
6689            },
6690        };
6691
6692        Ok(vec![control_response_body_frame(
6693            &frame,
6694            &response,
6695            "ClientControlResponse::RoutePoll",
6696        )?])
6697    }
6698
6699    pub(crate) fn observe_module_control_completion(
6700        &self,
6701        completion: ModuleControlRpcCompletion,
6702    ) -> bool {
6703        match completion {
6704            ModuleControlRpcCompletion::Unknown => false,
6705            ModuleControlRpcCompletion::Settled => true,
6706            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6707                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6708                info!(
6709                    module_id = %module_id,
6710                    latency_ms,
6711                    "late health.check answer proves the module is alive"
6712                );
6713                match self
6714                    .supervisor
6715                    .record_late_health_answer(&module_id, latency_ms)
6716                {
6717                    Ok(true) => {}
6718                    Ok(false) => debug!(
6719                        module_id = %module_id,
6720                        latency_ms,
6721                        "late health.check answer has no active supervisor snapshot"
6722                    ),
6723                    Err(err) => warn!(
6724                        module_id = %module_id,
6725                        latency_ms,
6726                        error = %err,
6727                        "failed to record late health.check answer"
6728                    ),
6729                }
6730                true
6731            }
6732        }
6733    }
6734
6735    /// Decide whether a failure while settling a relayed `route.bind` belongs to
6736    /// the module connection whose frame is being handled, or to the client that
6737    /// relay was opened for.
6738    ///
6739    /// This runs on the MODULE connection's frame handler, where returning `Err`
6740    /// ends that connection -- and a module connection carries every client's
6741    /// routes to that module, so ending it costs the whole fleet its tools.
6742    /// `ConnectionClosing` carries the id of the connection that is closing, and
6743    /// when that id is a CLIENT's, the condition is entirely about that one
6744    /// client's route.open. A client-scoped condition has no authority over a
6745    /// shared module connection, so it is logged and the single relay is dropped:
6746    /// the client is going away, and `complete_pending_relay` already removed the
6747    /// relay before failing, so there is nothing left to settle. Anything that
6748    /// relay still reserved is released by that client's own connection teardown,
6749    /// which is already under way -- that is what "closing" means.
6750    ///
6751    /// Every other failure is a statement about THIS connection and stays fatal:
6752    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
6753    /// id in `ConnectionClosing` all mean this connection cannot keep serving
6754    /// frames correctly.
6755    fn refuse_to_end_module_connection_for_a_client(
6756        &self,
6757        module_connection_id: ConnectionId,
6758        corr: u64,
6759        err: ForwardingError,
6760    ) -> Result<(), RouterError> {
6761        if let ForwardingError::ConnectionClosing { connection_id } = err {
6762            if connection_id != module_connection_id {
6763                warn!(
6764                    module_connection_id = module_connection_id.get(),
6765                    client_connection_id = connection_id.get(),
6766                    corr,
6767                    "dropping a route.bind response for a closing client; the module connection keeps serving"
6768                );
6769                return Ok(());
6770            }
6771        }
6772        Err(RouterError::Forwarding(err))
6773    }
6774
6775    fn handle_module_relay_response(
6776        &self,
6777        connection_id: ConnectionId,
6778        frame: Frame,
6779    ) -> Result<Vec<Frame>, RouterError> {
6780        let mut secondary_error = None;
6781        let outcome = match frame.header.ty {
6782            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6783                Ok(probe) if probe.op == "route.bind" => {
6784                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6785                        Ok(ModuleControlResponse::RouteBindAck {}) => {
6786                            RouteBindRelayOutcome::Accepted
6787                        }
6788                        Ok(other) => {
6789                            let message =
6790                                format!("route.bind response carried unexpected body: {other:?}");
6791                            secondary_error = Some(control_error_frame(
6792                                &frame,
6793                                "invalid_control_body",
6794                                message.clone(),
6795                            )?);
6796                            RouteBindRelayOutcome::ModuleGone(message)
6797                        }
6798                        Err(err) => {
6799                            let message = format!("malformed route.bind response body: {err}");
6800                            secondary_error = Some(control_error_frame(
6801                                &frame,
6802                                "invalid_control_body",
6803                                message.clone(),
6804                            )?);
6805                            RouteBindRelayOutcome::ModuleGone(message)
6806                        }
6807                    }
6808                }
6809                Ok(probe) => {
6810                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6811                    {
6812                        Ok(response) => ModuleControlRpcOutcome::Response(response),
6813                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6814                            "malformed {} response body: {err}",
6815                            probe.op
6816                        )),
6817                    };
6818                    let completion = self
6819                        .forwarding
6820                        .complete_module_control_rpc(
6821                            connection_id,
6822                            frame.header.corr,
6823                            Some(&probe.op),
6824                            outcome,
6825                        )
6826                        .map_err(RouterError::Forwarding)?;
6827                    if !self.observe_module_control_completion(completion) {
6828                        debug!(
6829                            connection_id = connection_id.get(),
6830                            corr = frame.header.corr,
6831                            op = %probe.op,
6832                            "dropping late or unknown module-control RPC response"
6833                        );
6834                    }
6835                    return Ok(Vec::new());
6836                }
6837                Err(err) => {
6838                    if let Some(expected_op) = self
6839                        .forwarding
6840                        .pending_module_control_op(connection_id, frame.header.corr)
6841                        .map_err(RouterError::Forwarding)?
6842                    {
6843                        let completion = self
6844                            .forwarding
6845                            .complete_module_control_rpc(
6846                                connection_id,
6847                                frame.header.corr,
6848                                None,
6849                                ModuleControlRpcOutcome::MalformedResponse(format!(
6850                                    "malformed {expected_op} response body: {err}"
6851                                )),
6852                            )
6853                            .map_err(RouterError::Forwarding)?;
6854                        if !self.observe_module_control_completion(completion) {
6855                            debug!(
6856                                connection_id = connection_id.get(),
6857                                corr = frame.header.corr,
6858                                "dropping late malformed module-control RPC response"
6859                            );
6860                        }
6861                        return Ok(Vec::new());
6862                    }
6863                    let message = format!("malformed route.bind response body: {err}");
6864                    secondary_error = Some(control_error_frame(
6865                        &frame,
6866                        "invalid_control_body",
6867                        message.clone(),
6868                    )?);
6869                    RouteBindRelayOutcome::ModuleGone(message)
6870                }
6871            },
6872            FrameType::Error => {
6873                if self
6874                    .forwarding
6875                    .pending_module_control_op(connection_id, frame.header.corr)
6876                    .map_err(RouterError::Forwarding)?
6877                    .is_some()
6878                {
6879                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6880                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6881                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6882                            "malformed module-control ERROR body: {err}"
6883                        )),
6884                    };
6885                    let completion = self
6886                        .forwarding
6887                        .complete_module_control_rpc(
6888                            connection_id,
6889                            frame.header.corr,
6890                            None,
6891                            outcome,
6892                        )
6893                        .map_err(RouterError::Forwarding)?;
6894                    if !self.observe_module_control_completion(completion) {
6895                        debug!(
6896                            connection_id = connection_id.get(),
6897                            corr = frame.header.corr,
6898                            "dropping late or unknown module-control RPC error"
6899                        );
6900                    }
6901                    return Ok(Vec::new());
6902                }
6903                match serde_json::from_slice::<ErrorBody>(&frame.body) {
6904                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
6905                    Err(err) => {
6906                        let message = format!("malformed route.bind ERROR body: {err}");
6907                        secondary_error = Some(control_error_frame(
6908                            &frame,
6909                            "invalid_control_body",
6910                            message.clone(),
6911                        )?);
6912                        RouteBindRelayOutcome::ModuleGone(message)
6913                    }
6914                }
6915            }
6916            ty => {
6917                return Ok(vec![control_error_frame(
6918                    &frame,
6919                    "unsupported_control_frame",
6920                    format!("unsupported module channel-0 frame {ty:?}"),
6921                )?])
6922            }
6923        };
6924
6925        let settled =
6926            self.forwarding
6927                .complete_pending_relay(connection_id, frame.header.corr, outcome);
6928        let completion = match settled {
6929            Ok(completion) => completion,
6930            Err(err) => {
6931                self.refuse_to_end_module_connection_for_a_client(
6932                    connection_id,
6933                    frame.header.corr,
6934                    err,
6935                )?;
6936                return Ok(secondary_error.into_iter().collect());
6937            }
6938        };
6939        if let Some(target) = completion.abandoned.as_ref() {
6940            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6941        }
6942        if !completion.settled {
6943            debug!(
6944                connection_id = connection_id.get(),
6945                corr = frame.header.corr,
6946                frame_type = ?frame.header.ty,
6947                "dropping late or unknown route.bind relay response"
6948            );
6949        }
6950        Ok(secondary_error.into_iter().collect())
6951    }
6952
6953    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6954        debug!(connection_id = connection_id.get(), "handling GOODBYE");
6955        // GOODBYE ends the connection's logical session even when its socket
6956        // stays open. Use disconnect teardown so verdicts, client notices and
6957        // scope authority are released at the same lifecycle boundary.
6958        self.cleanup_connection_with_end_reason(
6959            connection_id,
6960            RegistrationEndReason::ExplicitGoodbye,
6961        )
6962        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6963        Ok(Vec::new())
6964    }
6965}
6966
6967impl Default for ControlHandler {
6968    fn default() -> Self {
6969        Self::new(Arc::new(Registry::default()))
6970    }
6971}
6972
6973impl crate::supervise::SwapPromotionObserver for ControlHandler {
6974    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6975        self.apply_registration_capabilities(registration);
6976    }
6977}
6978
6979fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6980    CapabilityRequirementStatus {
6981        consumer: status.consumer,
6982        capability: status.capability,
6983        need: match status.need {
6984            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6985            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6986        },
6987        verdict: status.verdict.as_str().to_string(),
6988        episode_seq: status.episode_seq,
6989        config_satisfiable: status.config_satisfiable,
6990        runtime_available: status.runtime_available,
6991        detail: status.detail,
6992    }
6993}
6994
6995fn append_capability_problem_detail(
6996    detail: Option<String>,
6997    capability_detail: Option<String>,
6998) -> Option<String> {
6999    match (detail, capability_detail) {
7000        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7001        (Some(detail), None) => Some(detail),
7002        (None, Some(capability_detail)) => Some(capability_detail),
7003        (None, None) => None,
7004    }
7005}
7006
7007fn subc_ops() -> Vec<String> {
7008    SUBC_CONTROL_OPS
7009        .iter()
7010        .map(|op| (*op).to_string())
7011        .collect()
7012}
7013
7014fn module_subc_ops() -> Vec<String> {
7015    SUBC_CONTROL_OPS
7016        .iter()
7017        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7018        .map(|op| (*op).to_string())
7019        .collect()
7020}
7021
7022#[cfg(test)]
7023fn module_baseline_control_ops() -> Vec<String> {
7024    MODULE_BASELINE_CONTROL_OPS
7025        .iter()
7026        .map(|op| (*op).to_string())
7027        .collect()
7028}
7029
7030fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7031    let mut seen = HashSet::new();
7032    let mut effective = Vec::new();
7033    for op in MODULE_BASELINE_CONTROL_OPS {
7034        if seen.insert((*op).to_string()) {
7035            effective.push((*op).to_string());
7036        }
7037    }
7038    for op in declared.unwrap_or_default() {
7039        if seen.insert(op.clone()) {
7040            effective.push(op);
7041        }
7042    }
7043    effective
7044}
7045
7046fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7047    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7048}
7049
7050fn target_module_id(target: &RouteTarget) -> &str {
7051    match target {
7052        RouteTarget::ToolProvider { module_id }
7053        | RouteTarget::ManagementSurface { module_id }
7054        | RouteTarget::InternalService { module_id, .. } => module_id,
7055    }
7056}
7057
7058fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7059    roles.iter().any(|role| match (target, role) {
7060        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7061        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7062        (
7063            RouteTarget::InternalService { service_id, .. },
7064            ProviderRole::InternalService {
7065                service_id: provided,
7066                ..
7067            },
7068        ) => service_id == provided,
7069        _ => false,
7070    })
7071}
7072
7073fn is_routable_role(role: &ProviderRole) -> bool {
7074    matches!(
7075        role,
7076        ProviderRole::ToolProvider { .. }
7077            | ProviderRole::ManagementSurface { .. }
7078            | ProviderRole::InternalService { .. }
7079    )
7080}
7081
7082#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7083enum ControlRequestBodyError {
7084    UnknownOp,
7085    InvalidBody,
7086}
7087
7088#[derive(Debug, Deserialize)]
7089struct ControlOpProbe {
7090    op: String,
7091}
7092
7093/// Channel-0 push ops this subc version understands. A push whose `op` is not in
7094/// this set is treated as a forward-compat unknown and ignored rather than errored.
7095const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7096
7097fn is_known_module_push_op(body: &[u8]) -> bool {
7098    serde_json::from_slice::<ControlOpProbe>(body)
7099        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7100        .unwrap_or(false)
7101}
7102
7103fn is_known_module_request_op(body: &[u8]) -> bool {
7104    serde_json::from_slice::<ControlOpProbe>(body)
7105        .map(|probe| is_module_to_subc_op(&probe.op))
7106        .unwrap_or(false)
7107}
7108
7109fn is_module_to_subc_op(op: &str) -> bool {
7110    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7111}
7112
7113fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7114    debug!(
7115        op = %op,
7116        connection_id = connection_id.get(),
7117        corr,
7118        "control dispatch"
7119    );
7120}
7121
7122fn log_slow_control_dispatch(
7123    dispatch_started_at: Option<StdInstant>,
7124    op: &'static str,
7125    connection_id: ConnectionId,
7126    corr: u64,
7127    route_open_timings: Option<&RouteOpenTimings>,
7128) {
7129    let Some(dispatch_started_at) = dispatch_started_at else {
7130        return;
7131    };
7132    // Use the same monotonic clock as the phase timers and relay deadline.
7133    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7134    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7135        if let Some(timings) = route_open_timings {
7136            warn!(
7137                op = %op,
7138                connection_id = connection_id.get(),
7139                corr,
7140                elapsed_ms = elapsed.as_millis() as u64,
7141                module_id = ?timings.module_id,
7142                principal = %timings.principal,
7143                admission_ms = timings.admission.as_millis() as u64,
7144                project_root_ms = timings.project_root.as_millis() as u64,
7145                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7146                bind_send_ms = timings.bind_send.as_millis() as u64,
7147                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7148                "slow control dispatch"
7149            );
7150            return;
7151        }
7152        warn!(
7153            op = %op,
7154            connection_id = connection_id.get(),
7155            corr,
7156            elapsed_ms = elapsed.as_millis() as u64,
7157            "slow control dispatch"
7158        );
7159    }
7160}
7161
7162/// A test stand-in for the OS caller lookup, given the recorded endpoints.
7163#[cfg(test)]
7164type TestCallerLookup =
7165    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7166
7167/// Every new client operation needs an explicit privilege decision here.
7168fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7169    match request {
7170        ClientControlRequest::SupervisorRestart { module_id, .. } => {
7171            Some(PrivilegedOperation::Restart { module_id })
7172        }
7173        ClientControlRequest::SupervisorSwap { module_id, .. } => {
7174            Some(PrivilegedOperation::Swap { module_id })
7175        }
7176        ClientControlRequest::SupervisorReload { module_id } => {
7177            Some(PrivilegedOperation::Reload { module_id })
7178        }
7179        ClientControlRequest::SupervisorRescan { preview } => {
7180            if *preview {
7181                None
7182            } else {
7183                Some(PrivilegedOperation::Rescan)
7184            }
7185        }
7186        // Only the well-formed apply asks for approval. A preview changes
7187        // nothing, and any other combination is refused before it touches
7188        // anything, so prompting the operator for it would ask about a request
7189        // that cannot run.
7190        ClientControlRequest::SupervisorRescanChecked {
7191            preview,
7192            candidate,
7193            expect,
7194        } => (!*preview && candidate.is_none() && expect.is_some())
7195            .then_some(PrivilegedOperation::Rescan),
7196        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7197            Some(PrivilegedOperation::SetEnabled {
7198                module_id,
7199                enabled: *enabled,
7200            })
7201        }
7202        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7203            Some(PrivilegedOperation::ReleaseReserved { module_id })
7204        }
7205        ClientControlRequest::ServerDescribe {}
7206        | ClientControlRequest::CatalogList { .. }
7207        | ClientControlRequest::RouteOpen { .. }
7208        | ClientControlRequest::RoutePoll { .. }
7209        | ClientControlRequest::SupervisorList {}
7210        | ClientControlRequest::SupervisorHealth {}
7211        | ClientControlRequest::SupervisorHealthProbe { .. }
7212        | ClientControlRequest::SupervisorRoutes { .. }
7213        | ClientControlRequest::SupervisorProvenance { .. }
7214        | ClientControlRequest::SupervisorStderrTail { .. }
7215        | ClientControlRequest::SupervisorTerminals { .. }
7216        | ClientControlRequest::SupervisorSpawnSnapshot {}
7217        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7218    }
7219}
7220
7221fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7222    match request {
7223        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7224        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7225        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7226        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7227        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7228        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7229        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7230        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7231        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7232        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7233        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7234        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7235        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7236        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7237        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7238        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7239        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7240        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7241        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7242        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7243    }
7244}
7245
7246/// The `scope=` field of the `route.open accepted` line, from the stamp that
7247/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
7248/// so a ref holding control characters cannot forge a log line, and cut to 64
7249/// characters with a trailing `…` to bound the line. The bind proof and the
7250/// scope's attributes never appear.
7251fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7252    use subc_protocol::scope::ScopeKind;
7253    const MAX_REF_CHARS: usize = 64;
7254    let kind = match stamp.kind {
7255        ScopeKind::Head => "head",
7256        ScopeKind::Worker => "worker",
7257        ScopeKind::Ephemeral => "ephemeral",
7258    };
7259    let mut scope_ref: String = stamp.scope_ref.chars().take(MAX_REF_CHARS).collect();
7260    if stamp.scope_ref.chars().nth(MAX_REF_CHARS).is_some() {
7261        scope_ref.push('…');
7262    }
7263    format!(
7264        "{kind}/{}/{scope_ref:?}@{}",
7265        principal_label(&stamp.owner),
7266        stamp.scope_epoch
7267    )
7268}
7269
7270fn principal_label(principal: &Principal) -> String {
7271    match principal {
7272        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7273        Principal::Direct => "direct".to_string(),
7274        other => format!("{other:?}"),
7275    }
7276}
7277
7278fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7279    match request {
7280        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7281        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7282        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7283        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7284        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7285    }
7286}
7287
7288fn parse_client_control_request(
7289    body: &[u8],
7290) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7291    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7292        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7293            Ok(probe)
7294                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7295                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7296            {
7297                ControlRequestBodyError::InvalidBody
7298            }
7299            Ok(_) => ControlRequestBodyError::UnknownOp,
7300            Err(_) => ControlRequestBodyError::InvalidBody,
7301        };
7302        (err, classification)
7303    })
7304}
7305
7306fn parse_module_control_request_from_module(
7307    body: &[u8],
7308) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7309    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7310        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7311            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7312            Ok(_) => ControlRequestBodyError::UnknownOp,
7313            Err(_) => ControlRequestBodyError::InvalidBody,
7314        };
7315        (err, classification)
7316    })
7317}
7318
7319#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7320enum ProviderRoleKind {
7321    ToolProvider,
7322    PipelineStage,
7323    ManagementSurface,
7324    InternalService,
7325}
7326
7327fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7328    match role {
7329        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7330        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7331        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7332        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7333    }
7334}
7335
7336fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7337    roles.iter().map(provider_role_kind).collect()
7338}
7339
7340/// Most refused scope records named individually in the log per sync; the
7341/// `refused` count on the accepted line is always complete.
7342const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7343
7344/// Per-outcome counts of one accepted `scope.sync`, for its log line.
7345#[derive(Debug, Default, PartialEq, Eq)]
7346struct ScopeOutcomeCounts {
7347    created: usize,
7348    replaced: usize,
7349    updated: usize,
7350    unchanged: usize,
7351    refused: usize,
7352}
7353
7354impl ScopeOutcomeCounts {
7355    fn of(results: &[ScopeRecordResult]) -> Self {
7356        let mut counts = Self::default();
7357        for result in results {
7358            let slot = match result.outcome {
7359                ScopeRecordOutcome::Created => &mut counts.created,
7360                ScopeRecordOutcome::Replaced => &mut counts.replaced,
7361                ScopeRecordOutcome::Updated => &mut counts.updated,
7362                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7363                ScopeRecordOutcome::Refused => &mut counts.refused,
7364            };
7365            *slot += 1;
7366        }
7367        counts
7368    }
7369}
7370
7371#[cfg(test)]
7372mod scope_outcome_count_tests {
7373    use super::*;
7374
7375    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7376        ScopeRecordResult {
7377            scope_ref: "r".to_string(),
7378            scope_epoch: 1,
7379            outcome,
7380            code: None,
7381            message: None,
7382            version: None,
7383            parent_state: None,
7384        }
7385    }
7386
7387    /// Each outcome lands in its own count, so a refused record can never be
7388    /// hidden inside the total the log already printed.
7389    #[test]
7390    fn every_outcome_is_counted_in_its_own_field() {
7391        let results = [
7392            result(ScopeRecordOutcome::Created),
7393            result(ScopeRecordOutcome::Created),
7394            result(ScopeRecordOutcome::Replaced),
7395            result(ScopeRecordOutcome::Updated),
7396            result(ScopeRecordOutcome::Unchanged),
7397            result(ScopeRecordOutcome::Refused),
7398            result(ScopeRecordOutcome::Refused),
7399            result(ScopeRecordOutcome::Refused),
7400        ];
7401        assert_eq!(
7402            ScopeOutcomeCounts::of(&results),
7403            ScopeOutcomeCounts {
7404                created: 2,
7405                replaced: 1,
7406                updated: 1,
7407                unchanged: 1,
7408                refused: 3,
7409            }
7410        );
7411    }
7412}
7413
7414/// Return whether a catalog change can create a newly violating live route.
7415/// Removing an attested claim is intentionally excluded: it makes fewer routes
7416/// forbidden and therefore must leave the existing route census untouched.
7417fn capability_census_trigger(
7418    old: Option<&CapabilityDeclarations>,
7419    new: Option<&CapabilityDeclarations>,
7420) -> bool {
7421    let old_provides = old
7422        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7423        .unwrap_or_default();
7424    let old_denies = old
7425        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7426        .unwrap_or_default();
7427    let new = new.cloned().unwrap_or(CapabilityDeclarations {
7428        provides: Vec::new(),
7429        requires: Vec::new(),
7430        must_never_reach: Vec::new(),
7431    });
7432
7433    new.provides
7434        .iter()
7435        .any(|capability| !old_provides.contains(capability))
7436        || new
7437            .must_never_reach
7438            .iter()
7439            .any(|capability| !old_denies.contains(capability))
7440}
7441
7442/// Find the first capability an attested opener denies that an attested target
7443/// claims. Both manifests are live registry records, never cached or client data.
7444fn denied_capability<'a>(
7445    opening_manifest: &'a ModuleManifest,
7446    target_manifest: &ModuleManifest,
7447) -> Option<&'a str> {
7448    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7449    let target_capabilities = target_manifest.capabilities.as_ref()?;
7450    opening_capabilities
7451        .must_never_reach
7452        .iter()
7453        .find(|denied| {
7454            target_capabilities
7455                .provides
7456                .iter()
7457                .any(|provided| provided == *denied)
7458        })
7459        .map(String::as_str)
7460}
7461
7462fn catalog_update_frozen_field_message(
7463    registered: &ModuleManifest,
7464    provides: &[ProviderRole],
7465) -> Option<String> {
7466    let old_has_provides = !registered.provides.is_empty();
7467    let new_has_provides = !provides.is_empty();
7468    if old_has_provides != new_has_provides {
7469        return Some(format!(
7470            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7471            registered.module_id
7472        ));
7473    }
7474
7475    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
7476        return Some(format!(
7477            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7478            registered.module_id
7479        ));
7480    }
7481
7482    let registered_concurrency = manifest_concurrency(registered);
7483    let mut candidate = registered.clone();
7484    candidate.provides = provides.to_vec();
7485    let candidate_concurrency = manifest_concurrency(&candidate);
7486    if candidate_concurrency != registered_concurrency {
7487        return Some(format!(
7488            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7489            registered.module_id, registered_concurrency, candidate_concurrency
7490        ));
7491    }
7492
7493    // control_ops live beside the manifest in the HELLO body, not inside
7494    // ModuleManifest, so a provides-only catalog.update cannot change them.
7495    None
7496}
7497
7498fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7499    manifest.provides.iter().any(is_routable_role)
7500}
7501
7502/// Returns the routable-provider concurrency subc should enforce for this manifest.
7503///
7504/// ToolProvider and ManagementSurface store their delivery concurrency directly.
7505/// InternalService has no role-specific concurrency field, so it retains the
7506/// existing ModuleManaged default for backward compatibility.
7507fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7508    manifest
7509        .provides
7510        .iter()
7511        .find_map(|provider| match provider {
7512            ProviderRole::ToolProvider { concurrency, .. }
7513            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7514            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7515        })
7516        .unwrap_or(Concurrency::ModuleManaged)
7517}
7518
7519/// True when the manifest carries a ManagementSurface role whose concurrency
7520/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
7521/// bytes because the typed manifest deliberately erases that distinction: the
7522/// default exists for wire compatibility, and this probe exists so the default
7523/// stays observable. Any parse irregularity returns false -- the caller only
7524/// logs, and a malformed body already failed registration upstream.
7525fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7526    let has_management_surface = manifest
7527        .provides
7528        .iter()
7529        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7530    if !has_management_surface {
7531        return false;
7532    }
7533    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7534        return false;
7535    };
7536    let Some(provides) = raw
7537        .get("manifest")
7538        .and_then(|manifest| manifest.get("provides"))
7539        .and_then(serde_json::Value::as_array)
7540    else {
7541        return false;
7542    };
7543    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
7544    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
7545    // against the management_surface_manifest_without_concurrency golden, not
7546    // recalled (the externally-tagged guess was this function's first bug).
7547    provides.iter().any(|role| {
7548        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7549            && role.get("concurrency").is_none()
7550    })
7551}
7552
7553fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7554    if peer_version != PROTOCOL_VERSION {
7555        return Err(format!(
7556            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7557        ));
7558    }
7559    Ok(PROTOCOL_VERSION)
7560}
7561
7562fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7563    Frame::build_with_version(
7564        response_version(frame),
7565        FrameType::Pong,
7566        frame.header.flags,
7567        0,
7568        0,
7569        frame.header.corr,
7570        Vec::new(),
7571    )
7572    .map_err(RouterError::FrameBuild)
7573}
7574
7575pub(crate) fn control_error_frame(
7576    frame: &Frame,
7577    code: &'static str,
7578    message: impl Into<String>,
7579) -> Result<Frame, RouterError> {
7580    control_error_body_frame(
7581        frame,
7582        ErrorBody {
7583            code: code.to_string(),
7584            message: message.into(),
7585            detail: None,
7586        },
7587    )
7588}
7589
7590fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7591    let body = serde_json::to_vec(&error).map_err(|err| {
7592        RouterError::backend(
7593            0,
7594            frame.header.corr,
7595            format!("failed to encode control ERROR: {err}"),
7596        )
7597    })?;
7598
7599    Frame::build_with_version(
7600        response_version(frame),
7601        FrameType::Error,
7602        control_flags(),
7603        0,
7604        0,
7605        frame.header.corr,
7606        body,
7607    )
7608    .map_err(RouterError::FrameBuild)
7609}
7610
7611fn control_response_body_frame<T: Serialize>(
7612    frame: &Frame,
7613    reply: &T,
7614    label: &'static str,
7615) -> Result<Frame, RouterError> {
7616    let body = serde_json::to_vec(reply).map_err(|err| {
7617        RouterError::backend(
7618            0,
7619            frame.header.corr,
7620            format!("failed to encode {label}: {err}"),
7621        )
7622    })?;
7623
7624    Frame::build_with_version(
7625        response_version(frame),
7626        FrameType::Response,
7627        control_flags(),
7628        0,
7629        0,
7630        frame.header.corr,
7631        body,
7632    )
7633    .map_err(RouterError::FrameBuild)
7634}
7635
7636/// Map a forwarding failure to the wire code a client sees.
7637///
7638/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
7639/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
7640/// chosen here decides whether a caller retries or gives up.
7641///
7642/// That makes attribution the load-bearing property, not merely having a code. A
7643/// permanent fault published as a retryable one produces a fleet-wide retry storm
7644/// against something that can never recover; a transient fault published as
7645/// permanent gives up on work that would have succeeded. Both look correct in a
7646/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
7647/// the mapping per variant rather than merely asserting that some code exists.
7648///
7649/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
7650/// two codes on the same side of the boundary passes it. Measured rather than
7651/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
7652/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
7653/// a test named for something else that happens to assert the string.
7654///
7655/// That accidental coverage is deliberately left alone rather than promoted to a
7656/// named test, because it guards a property this function does not promise.
7657/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
7658/// specific code within a class, so identity is free to change and only the
7659/// partition is a contract. Splitting it out would assert a guarantee nothing
7660/// depends on — and a suite that promises more than the code does is the harder
7661/// thing to correct later, because the next reader cannot tell which assertions
7662/// are load-bearing.
7663///
7664/// Pin identity here the moment a consumer branches on a specific code.
7665fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7666    match err {
7667        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7668        ForwardingError::NoModuleConnection => "target_unavailable",
7669        ForwardingError::ModuleReloading { .. } => "module_reloading",
7670        ForwardingError::ClientRouteChannelExhausted { .. }
7671        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7672        ForwardingError::StaleModuleEndpoint
7673        | ForwardingError::UnknownReservation { .. }
7674        | ForwardingError::ConnectionClosing { .. }
7675        | ForwardingError::ClientEgressClosed { .. }
7676        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7677        // Only a swap candidate's registration can produce this, and it means
7678        // exactly what a second active HELLO for a live id means.
7679        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7680        ForwardingError::RelayCorrelationExhausted
7681        | ForwardingError::RouteOpenBuild(_)
7682        | ForwardingError::Poisoned => "forwarding_error",
7683    }
7684}
7685
7686fn response_version(frame: &Frame) -> u8 {
7687    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7688        frame.header.ver
7689    } else {
7690        PROTOCOL_VERSION
7691    }
7692}
7693
7694fn control_flags() -> Flags {
7695    Flags::new(false, Priority::Passive, false)
7696}
7697
7698/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
7699/// channel. The target is the module (a client never saw the route), so this
7700/// takes the module path: delivered late rather than dropped when the module's
7701/// queue is momentarily full, and never closing its connection.
7702fn send_goodbye_target_best_effort(
7703    counters: &DaemonCounters,
7704    target: &GoodbyeTarget,
7705    context: &'static str,
7706) {
7707    let Ok(frame) = Frame::build_with_version(
7708        target.negotiated_ver,
7709        FrameType::Goodbye,
7710        control_flags(),
7711        target.channel,
7712        target.epoch,
7713        0,
7714        Vec::new(),
7715    ) else {
7716        return;
7717    };
7718    crate::forwarding::send_module_route_goodbye(
7719        counters,
7720        &target.sink,
7721        frame,
7722        target.module_id.as_deref(),
7723        context,
7724    );
7725}
7726
7727pub(crate) fn send_route_control_pushes(
7728    forwarding: &ForwardingTable,
7729    routes: Vec<EndpointRoute>,
7730    push: ClientControlPush,
7731) {
7732    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7733    for route in routes {
7734        let target = route.goodbye_target;
7735        if let Some((existing, channels)) = targets
7736            .iter_mut()
7737            .find(|(existing, _)| existing.connection_id == target.connection_id)
7738        {
7739            debug_assert_eq!(
7740                existing.negotiated_ver, target.negotiated_ver,
7741                "one connection cannot negotiate multiple frame versions"
7742            );
7743            if !channels.contains(&target.channel) {
7744                channels.push(target.channel);
7745            }
7746            continue;
7747        }
7748        let channel = target.channel;
7749        targets.push((target, vec![channel]));
7750    }
7751    for (target, mut channels) in targets {
7752        channels.sort_unstable();
7753        let mut push = push.clone();
7754        match &mut push {
7755            ClientControlPush::RouteClosing {
7756                channels: covered, ..
7757            }
7758            | ClientControlPush::RouteClosed {
7759                channels: covered, ..
7760            } => *covered = channels,
7761        }
7762        let body = match serde_json::to_vec(&push) {
7763            Ok(body) => body,
7764            Err(err) => {
7765                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7766                continue;
7767            }
7768        };
7769        let frame = match Frame::build_with_version(
7770            target.negotiated_ver,
7771            FrameType::Push,
7772            control_flags(),
7773            0,
7774            0,
7775            0,
7776            body.clone(),
7777        ) {
7778            Ok(frame) => frame,
7779            Err(err) => {
7780                warn!(
7781                    route_channel = target.channel,
7782                    error = %err,
7783                    "failed to build route lifecycle control PUSH frame"
7784                );
7785                continue;
7786            }
7787        };
7788        if let Err(err) = target.sink.try_send(frame) {
7789            if target.close_on_delivery_failure() {
7790                warn!(
7791                    target_connection_id = target.connection_id.get(),
7792                    route_channel = target.channel,
7793                    error = %err,
7794                    "route lifecycle control PUSH was not delivered to client; closing target connection"
7795                );
7796                let _ = forwarding.escalate_client_delivery_failure(
7797                    target.connection_id,
7798                    target.channel,
7799                    target.epoch,
7800                    CloseReason::new(
7801                        "route_lifecycle_push_delivery_failed",
7802                        format!(
7803                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7804                            target.channel
7805                        ),
7806                    ),
7807                    crate::forwarding::UndeliveredFrame {
7808                        module_id: target.module_id.as_deref(),
7809                        sink: &target.sink,
7810                    },
7811                );
7812            }
7813        }
7814    }
7815}
7816
7817#[cfg(test)]
7818mod tests {
7819    mod activity_read {
7820        include!("activity_read_tests.rs");
7821    }
7822
7823    mod lifecycle_activity {
7824        include!("control_lifecycle_tests.rs");
7825    }
7826    #[cfg(unix)]
7827    #[tokio::test]
7828    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7829        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7830        let path = dir.join("subc.jsonc");
7831        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7832            "program":"/bin/sleep","args":["60"],"protocol":"none",
7833            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7834        }}}).to_string()).unwrap();
7835        let mut configured = crate::daemon_config::load(&path)
7836            .unwrap()
7837            .unwrap()
7838            .modules
7839            .pop()
7840            .unwrap();
7841        let registry = std::sync::Arc::new(crate::Registry::default());
7842        let handle = crate::SupervisorHandle::new();
7843        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7844            .with_handle(handle.clone());
7845        let module = supervisor
7846            .supervise_configured_with_health(
7847                configured.module_spec(),
7848                true,
7849                configured.health.clone(),
7850                None,
7851                configured.restart,
7852            )
7853            .unwrap();
7854        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7855        let before = module.status().unwrap().pid;
7856        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7857        configured.health.cadence = std::time::Duration::from_secs(3600);
7858        let health_only = handler
7859            .reconcile_supervised_modules(
7860                &supervisor,
7861                vec![configured.clone()],
7862                false,
7863                Principal::Direct,
7864            )
7865            .await
7866            .unwrap();
7867        assert!(
7868            health_only.0.changed_pending_reload.is_empty(),
7869            "health policy is already applied live"
7870        );
7871        assert_eq!(module.status().unwrap().pid, before);
7872        assert_eq!(
7873            module.configuration().unwrap().1.http,
7874            configured.health.http
7875        );
7876        configured.args = vec!["61".into()];
7877        let launch = handler
7878            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7879            .await
7880            .unwrap();
7881        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
7882        assert_eq!(
7883            module.status().unwrap().pid,
7884            before,
7885            "a launch edit is stored until reload"
7886        );
7887        module.drain().await.unwrap();
7888    }
7889    use cortexkit_test_support::ScratchDir;
7890    use std::{
7891        collections::BTreeMap,
7892        fmt,
7893        path::PathBuf,
7894        sync::{Arc, Mutex},
7895        time::Duration,
7896    };
7897
7898    use serde_json::{json, Value};
7899    use subc_protocol::{
7900        manifest::{
7901            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7902            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7903        },
7904        session::HealthStatus,
7905        FrameType,
7906    };
7907
7908    use super::*;
7909    use crate::{
7910        forwarding::{DataRoute, DataRouteState},
7911        registry::ChannelState,
7912        router::FrameSink,
7913        stderr_tail::DEFAULT_MAX_LINE_BYTES,
7914        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7915        RouteCtx, Router,
7916    };
7917    use tokio::{
7918        sync::mpsc,
7919        time::{sleep, Instant},
7920    };
7921    use tracing::{
7922        field::{Field, Visit},
7923        Event, Subscriber,
7924    };
7925    use tracing_subscriber::{layer::Context, prelude::*, Layer};
7926
7927    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
7928    ///
7929    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
7930    /// is only populated for `tests/*.rs` integration test binaries -- this file
7931    /// compiles as part of the library target, which gets neither. This test's
7932    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
7933    /// and the sibling binary lives two directories up at
7934    /// `<target-dir>/<profile>/fake-aft-stub`.
7935    ///
7936    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
7937    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
7938    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
7939    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
7940    /// `NotFound`, which reads as a broken test rather than an unbuilt
7941    /// dependency -- so state the cause and the remedy instead. Deliberately a
7942    /// panic and not a silent skip: a test that quietly passes when it could not
7943    /// run is worse than one that fails, because it reports health it never
7944    /// verified.
7945    fn fake_aft_stub_path() -> PathBuf {
7946        let mut path = std::env::current_exe().expect("current_exe available in tests");
7947        path.pop(); // .../deps/
7948        path.pop(); // .../<profile>/
7949        path.push(if cfg!(windows) {
7950            "fake-aft-stub.exe"
7951        } else {
7952            "fake-aft-stub"
7953        });
7954        assert!(
7955            path.exists(),
7956            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7957             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7958            path.display()
7959        );
7960        path
7961    }
7962
7963    /// Whether clients retry `code` in place: the predicate itself, never a copy
7964    /// of its set. A copied list breaks silently when a code is added to or
7965    /// removed from the real one, and a stale copy here would let exactly the
7966    /// failure this test exists to catch pass.
7967    fn client_retries(code: &str) -> bool {
7968        subc_protocol::error_codes::is_retryable_route_open(code)
7969    }
7970
7971    /// A code is not a label — clients branch on it, so publishing the wrong KIND
7972    /// of failure is worse than publishing none. A permanent fault dressed as
7973    /// retryable makes every client in the fleet retry forever against something
7974    /// that cannot recover; a transient fault dressed as permanent abandons work
7975    /// that would have succeeded.
7976    ///
7977    /// Asserting "a code exists" cannot catch either, because the string is free
7978    /// to say anything. This enumerates every variant and pins which side of the
7979    /// retry boundary it lands on, so a new variant must be classified here
7980    /// deliberately rather than inheriting whichever arm it was appended to.
7981    #[test]
7982    fn retryability_of_forwarding_codes_matches_the_failure() {
7983        // Transient by nature: the target is booting, reloading, or its endpoint
7984        // was swapped mid-flight. Retrying is how these resolve.
7985        let transient = [
7986            ForwardingError::NoModuleConnection,
7987            ForwardingError::ModuleReloading {
7988                module_id: "m".into(),
7989            },
7990            ForwardingError::StaleModuleEndpoint,
7991            ForwardingError::UnknownReservation {
7992                client_channel: 1,
7993                module_channel: 1,
7994            },
7995            ForwardingError::ConnectionClosing {
7996                connection_id: ConnectionId::new(1),
7997            },
7998            ForwardingError::ClientEgressClosed {
7999                connection_id: ConnectionId::new(1),
8000            },
8001            ForwardingError::ModuleEgressUnavailable {
8002                connection_id: ConnectionId::new(1),
8003            },
8004        ];
8005        for err in transient {
8006            let code = forwarding_error_code(&err);
8007            assert!(
8008                client_retries(code),
8009                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8010            );
8011        }
8012
8013        // Not fixed by retrying. Channel and correlation exhaustion need the
8014        // caller to close routes, and a poisoned lock is a daemon that cannot
8015        // recover at all — the worst thing to advertise as retryable, since every
8016        // client would storm a daemon that will never answer.
8017        let permanent = [
8018            ForwardingError::ConnectionRoleConflict {
8019                connection_id: ConnectionId::new(1),
8020            },
8021            ForwardingError::ClientRouteChannelExhausted {
8022                connection_id: ConnectionId::new(1),
8023            },
8024            ForwardingError::ModuleRouteChannelExhausted {
8025                endpoint: ModuleEndpointId {
8026                    connection_id: ConnectionId::new(1),
8027                    generation: 1,
8028                },
8029            },
8030            ForwardingError::RelayCorrelationExhausted,
8031            ForwardingError::RouteOpenBuild("x".into()),
8032            ForwardingError::Poisoned,
8033        ];
8034        for err in permanent {
8035            let code = forwarding_error_code(&err);
8036            assert!(
8037                !client_retries(code),
8038                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8039            );
8040        }
8041    }
8042
8043    /// The principal is the daemon's answer to "who is calling", and modules
8044    /// branch on it: aft gates bash on it, cerebellum gates browser control,
8045    /// plexus gates connector invocation. So a stamp is an authorization input in
8046    /// another process, not a label — and both possible answers SUCCEED, which is
8047    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
8048    /// first-party capability to something that never proved it; a supervised one
8049    /// stamped `Direct` silently strips a module of capability it is entitled to.
8050    ///
8051    /// Neither shows up in a test that only checks the bind succeeded. Before this
8052    /// test the only coverage was accidental —
8053    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
8054    /// stamped principal on its way past, so narrowing that wire-shape test to its
8055    /// stated subject would have deleted the last assertion on this value. It
8056    /// still asserts the stamp, which is now redundancy rather than the only
8057    /// guard: both fail under the same mutation, and this one names the reason.
8058    /// SCOPE: this handler's supervisor has spawned nothing, so
8059    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
8060    /// is unreachable here. Both assertions below are refusals, and a mutant that
8061    /// refuses everything would satisfy them.
8062    ///
8063    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
8064    /// spawns a supervised consumer, reads its live nonce, and asserts the module
8065    /// observed `principal.kind == "reserved"` carrying that module_id — verified
8066    /// at source rather than assumed, since a citation is a claim about another
8067    /// file and ages like one. Recorded because a harness that structurally
8068    /// cannot reach an arm reports "none" for that arm identically to one that
8069    /// covers it and found nothing.
8070    #[tokio::test]
8071    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8072        let handler = ControlHandler::default();
8073        let frame =
8074            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8075
8076        // Absent consumer_identity is the ordinary case: a human at a terminal, or
8077        // any process holding the connection file. Nothing was proved, so nothing
8078        // may be granted beyond the unattested floor.
8079        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8080        assert_eq!(
8081            stamped,
8082            Principal::Direct,
8083            "a caller that proved nothing must not be stamped as a supervised module"
8084        );
8085
8086        // A claimed module_id with a nonce no supervised child was given is a
8087        // forgery attempt, not a weaker caller: it must be REFUSED rather than
8088        // quietly demoted to Direct, or an impersonation attempt looks identical
8089        // to an ordinary unattested connection.
8090        let forged = handler
8091            .route_open_principal(
8092                &frame,
8093                Some(ConsumerIdentity {
8094                    module_id: "aft".to_string(),
8095                    launch_nonce: "not-a-real-nonce".to_string(),
8096                }),
8097            )
8098            .unwrap();
8099        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8100        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8101    }
8102
8103    /// The test above hands `route_open_principal` an identity it built itself,
8104    /// which proves the stamping rule and nothing about where the identity comes
8105    /// from. The real producer is a wire body, and the two are joined by a serde
8106    /// field name that nothing else asserts.
8107    ///
8108    /// That join fails quietly in one specific way: an unrecognised key is simply
8109    /// absent after parsing, so a renamed or misspelled `consumer_identity`
8110    /// yields `None` and every supervised module silently drops to `Direct`.
8111    /// Capability-wise that is the safe direction, but it surfaces far from its
8112    /// cause — as a module mysteriously refused bash — and it would pass every
8113    /// test that builds its own input.
8114    ///
8115    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
8116    /// would break every client the moment the daemon gains a field, trading a
8117    /// quiet demotion for a hard refusal on additive change. Asserting the join
8118    /// instead means a rename breaks a test here rather than the fleet.
8119    #[test]
8120    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8121        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8122        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8123        let ClientControlRequest::RouteOpen {
8124            consumer_identity, ..
8125        } = parsed
8126        else {
8127            panic!("route.open body must parse as RouteOpen");
8128        };
8129        assert_eq!(
8130            consumer_identity,
8131            Some(ConsumerIdentity {
8132                module_id: "aft".to_string(),
8133                launch_nonce: "n".to_string(),
8134            }),
8135            "the wire field name must reach the value route_open_principal reads"
8136        );
8137    }
8138
8139    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8140        ModuleManifest::builder(module_id, "0.1.0")
8141            .protocol_ver(protocol_ver)
8142            .provides(vec![ProviderRole::ToolProvider {
8143                tools: vec![Tool {
8144                    name: "read".to_string(),
8145                    description: None,
8146                    execution_mode: ExecutionMode::Pure,
8147                    schema: json!({"type": "object"}),
8148                }],
8149                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8150                concurrency: Concurrency::ModuleManaged,
8151                emits_push: true,
8152                sub_supervises: true,
8153            }])
8154            .build()
8155    }
8156
8157    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8158        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8159    }
8160
8161    fn hello_frame_with_control_ops(
8162        module_id: &str,
8163        protocol_ver: u8,
8164        corr: u64,
8165        control_ops: Option<Vec<String>>,
8166    ) -> Frame {
8167        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8168    }
8169
8170    fn hello_frame_with_nonce(
8171        module_id: &str,
8172        protocol_ver: u8,
8173        corr: u64,
8174        launch_nonce: Option<&str>,
8175    ) -> Frame {
8176        hello_frame_full(
8177            module_id,
8178            protocol_ver,
8179            corr,
8180            None,
8181            launch_nonce.map(ToOwned::to_owned),
8182        )
8183    }
8184
8185    fn hello_frame_full(
8186        module_id: &str,
8187        protocol_ver: u8,
8188        corr: u64,
8189        control_ops: Option<Vec<String>>,
8190        launch_nonce: Option<String>,
8191    ) -> Frame {
8192        let body = serde_json::to_vec(&ModuleHelloBody {
8193            manifest: manifest(module_id, protocol_ver),
8194            protocol_ver,
8195            control_ops,
8196            launch_nonce,
8197        })
8198        .unwrap();
8199        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8200    }
8201
8202    fn hello_frame_with_rich_ui(
8203        module_id: &str,
8204        protocol_ver: u8,
8205        corr: u64,
8206        launch_nonce: Option<&str>,
8207        rich_ui: Value,
8208    ) -> Frame {
8209        let mut body = serde_json::to_value(ModuleHelloBody {
8210            manifest: manifest(module_id, protocol_ver),
8211            protocol_ver,
8212            control_ops: None,
8213            launch_nonce: launch_nonce.map(str::to_string),
8214        })
8215        .unwrap();
8216        body["manifest"]["rich_ui"] = rich_ui;
8217        Frame::build(
8218            FrameType::Hello,
8219            control_flags(),
8220            0,
8221            0,
8222            corr,
8223            serde_json::to_vec(&body).unwrap(),
8224        )
8225        .unwrap()
8226    }
8227
8228    fn non_routable_hello_frame_with_control_ops(
8229        module_id: &str,
8230        corr: u64,
8231        control_ops: Option<Vec<String>>,
8232    ) -> Frame {
8233        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8234        manifest.provides.clear();
8235        let body = serde_json::to_vec(&ModuleHelloBody {
8236            manifest,
8237            protocol_ver: PROTOCOL_VERSION,
8238            control_ops,
8239            launch_nonce: None,
8240        })
8241        .unwrap();
8242        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8243    }
8244
8245    fn capability_grammar_hello_frame(
8246        capabilities: Value,
8247        runtime_computed: Option<Value>,
8248        corr: u64,
8249    ) -> Frame {
8250        let mut body = serde_json::to_value(ModuleHelloBody {
8251            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8252            protocol_ver: PROTOCOL_VERSION,
8253            control_ops: None,
8254            launch_nonce: None,
8255        })
8256        .expect("HELLO body serializes");
8257        body["manifest"]["capabilities"] = capabilities;
8258        if let Some(runtime_computed) = runtime_computed {
8259            body["runtime_computed"] = runtime_computed;
8260        }
8261        Frame::build(
8262            FrameType::Hello,
8263            control_flags(),
8264            0,
8265            0,
8266            corr,
8267            serde_json::to_vec(&body).expect("HELLO body reserializes"),
8268        )
8269        .expect("HELLO frame builds")
8270    }
8271
8272    fn channel_request(channel: u16, corr: u64) -> Frame {
8273        Frame::build(
8274            FrameType::Request,
8275            Flags::new(true, Priority::Interactive, false),
8276            channel,
8277            0,
8278            corr,
8279            b"opaque".to_vec(),
8280        )
8281        .unwrap()
8282    }
8283
8284    fn route_ctx(
8285        connection_id: ConnectionId,
8286    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8287        let (tx, rx) = mpsc::channel(8);
8288        (
8289            RouteCtx {
8290                connection_id,
8291                egress: FrameSink::new(tx),
8292            },
8293            rx,
8294        )
8295    }
8296
8297    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8298        serde_json::from_slice(&frame.body).unwrap()
8299    }
8300
8301    /// Register a module over a connection that has a sink and return the
8302    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
8303    /// module's own sink rather than returning it as a reply, so the ack is
8304    /// taken off `rx` here and whatever the test reads next is what followed it.
8305    async fn hello_via_sink(
8306        handler: &ControlHandler,
8307        ctx: &RouteCtx,
8308        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8309        hello: Frame,
8310    ) -> Frame {
8311        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8312        assert!(
8313            replies.is_empty(),
8314            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8315        );
8316        let ack = rx
8317            .try_recv()
8318            .expect("HELLO_ACK is queued on the module sink")
8319            .frame;
8320        assert_eq!(ack.header.ty, FrameType::HelloAck);
8321        ack
8322    }
8323
8324    fn parse_error(frame: &Frame) -> Value {
8325        serde_json::from_slice(&frame.body).unwrap()
8326    }
8327
8328    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8329        serde_json::from_slice(&frame.body).unwrap()
8330    }
8331
8332    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8333        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8334            route_channel,
8335            route_epoch: 0,
8336            kind,
8337        })
8338        .unwrap();
8339        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8340    }
8341
8342    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8343        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8344            module_id: module_id.to_string(),
8345        })
8346        .unwrap();
8347        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8348    }
8349
8350    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8351        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8352    }
8353
8354    fn route_open_frame_with_consumer_capabilities(
8355        corr: u64,
8356        module_id: &str,
8357        project_root: ScratchDir,
8358        consumer_capabilities: Option<Vec<String>>,
8359    ) -> Frame {
8360        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8361            target: RouteTarget::ToolProvider {
8362                module_id: module_id.to_string(),
8363            },
8364            identity: BindIdentity::new(
8365                project_root.path().to_path_buf(),
8366                "unit".to_string(),
8367                "session".to_string(),
8368            ),
8369            consumer_identity: None,
8370            consumer_capabilities,
8371            role_versions: None,
8372            admission_facts: None,
8373            scope: None,
8374        })
8375        .unwrap();
8376        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8377    }
8378
8379    fn route_open_frame_with_role_versions(
8380        corr: u64,
8381        module_id: &str,
8382        project_root: ScratchDir,
8383        role_versions: Option<BTreeMap<String, String>>,
8384    ) -> Frame {
8385        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8386            target: RouteTarget::ToolProvider {
8387                module_id: module_id.to_string(),
8388            },
8389            identity: BindIdentity::new(
8390                project_root.path().to_path_buf(),
8391                "unit".to_string(),
8392                format!("session-{corr}"),
8393            ),
8394            consumer_identity: None,
8395            consumer_capabilities: None,
8396            role_versions,
8397            admission_facts: None,
8398            scope: None,
8399        })
8400        .unwrap();
8401        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8402    }
8403
8404    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8405        entries
8406            .iter()
8407            .map(|(role, version)| (role.to_string(), version.to_string()))
8408            .collect()
8409    }
8410
8411    fn route_open_frame_with_admission_facts(
8412        corr: u64,
8413        module_id: &str,
8414        project_root: ScratchDir,
8415        consumer_identity: Option<subc_control::ConsumerIdentity>,
8416        facts: Option<Value>,
8417    ) -> Frame {
8418        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8419            target: RouteTarget::ToolProvider {
8420                module_id: module_id.to_string(),
8421            },
8422            identity: BindIdentity::new(
8423                project_root.path().to_path_buf(),
8424                "unit".to_string(),
8425                format!("session-{corr}"),
8426            ),
8427            consumer_identity,
8428            consumer_capabilities: None,
8429            role_versions: None,
8430            admission_facts: facts,
8431            scope: None,
8432        })
8433        .unwrap();
8434        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8435    }
8436
8437    #[derive(Clone, Default)]
8438    struct EventCapture {
8439        events: Arc<Mutex<Vec<CapturedEvent>>>,
8440    }
8441
8442    #[derive(Clone, Debug)]
8443    struct CapturedEvent {
8444        target: String,
8445        level: tracing::Level,
8446        fields: BTreeMap<String, String>,
8447    }
8448
8449    impl EventCapture {
8450        fn events(&self) -> Vec<CapturedEvent> {
8451            self.events.lock().unwrap().clone()
8452        }
8453    }
8454
8455    impl<S> Layer<S> for EventCapture
8456    where
8457        S: Subscriber,
8458    {
8459        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8460            let mut visitor = EventFieldVisitor::default();
8461            event.record(&mut visitor);
8462            self.events.lock().unwrap().push(CapturedEvent {
8463                target: event.metadata().target().to_string(),
8464                level: *event.metadata().level(),
8465                fields: visitor.fields,
8466            });
8467        }
8468    }
8469
8470    #[derive(Default)]
8471    struct EventFieldVisitor {
8472        fields: BTreeMap<String, String>,
8473    }
8474
8475    impl Visit for EventFieldVisitor {
8476        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8477            self.fields
8478                .insert(field.name().to_string(), format!("{value:?}"));
8479        }
8480    }
8481
8482    fn health_response(corr: u64, status: HealthStatus) -> Frame {
8483        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8484            status,
8485            detail: Some("warming".to_string()),
8486            metrics: Some(json!({"queue_depth": 3})),
8487        })
8488        .unwrap();
8489        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8490    }
8491
8492    fn route_bind_ack(corr: u64) -> Frame {
8493        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8494        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8495    }
8496
8497    fn unique_project_root(label: &str) -> ScratchDir {
8498        ScratchDir::new(label)
8499    }
8500
8501    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8502        match parse_route_poll(frame) {
8503            ClientControlResponse::RoutePoll {
8504                status: None,
8505                live: Some(live),
8506                ..
8507            } => assert_eq!(live, expected_live),
8508            other => panic!("unexpected route.poll response: {other:?}"),
8509        }
8510    }
8511
8512    fn bind_liveness_route(
8513        registry: &Registry,
8514        forwarding: &ForwardingTable,
8515        module_id: &str,
8516    ) -> (RouteCtx, u16, u32) {
8517        let module_connection = ConnectionId::new(101);
8518        let client_connection = ConnectionId::new(202);
8519        let registration = registry
8520            .register_with_control_ops(
8521                manifest(module_id, PROTOCOL_VERSION),
8522                PROTOCOL_VERSION,
8523                module_connection,
8524                module_baseline_control_ops(),
8525            )
8526            .unwrap();
8527        let (module_tx, _module_rx) = mpsc::channel(8);
8528        let endpoint = forwarding
8529            .register_module_connection(
8530                module_connection,
8531                module_id.to_string(),
8532                PROTOCOL_VERSION,
8533                manifest_concurrency(&registration.manifest),
8534                FrameSink::new(module_tx),
8535            )
8536            .unwrap();
8537        let (client_ctx, _client_rx) = route_ctx(client_connection);
8538        let pending = forwarding
8539            .begin_route_bind_relay_for_test(
8540                client_connection,
8541                client_ctx.egress.clone(),
8542                1,
8543                module_id,
8544            )
8545            .unwrap();
8546        assert_eq!(pending.endpoint, endpoint);
8547        let route_channel = pending.client_channel;
8548        let route_epoch = pending.client_epoch;
8549        forwarding
8550            .complete_pending_relay(
8551                module_connection,
8552                pending.corr,
8553                RouteBindRelayOutcome::Accepted,
8554            )
8555            .unwrap();
8556        (client_ctx, route_channel, route_epoch)
8557    }
8558
8559    struct FakeProcessLiveness {
8560        live: Option<bool>,
8561    }
8562
8563    impl ModuleProcessLiveness for FakeProcessLiveness {
8564        fn process_live(&self, _module_id: &str) -> Option<bool> {
8565            self.live
8566        }
8567    }
8568
8569    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8570    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8571    {
8572        let registry = Arc::new(Registry::default());
8573        let supervisor_handle = SupervisorHandle::new();
8574        let supervisor = Supervisor::new_for_test(
8575            Arc::clone(&registry),
8576            RestartPolicy::new(1, Duration::from_millis(10)),
8577        )
8578        .with_handle(supervisor_handle.clone());
8579        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8580        let module = supervisor
8581            .spawn(ModuleSpec {
8582                module_id: "stderr-tail-wire".to_string(),
8583                program: fake_aft_stub_path(),
8584                args: Vec::new(),
8585                env: vec![
8586                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8587                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8588                ],
8589                reserved: false,
8590                reserved_prefixes: Vec::new(),
8591                protocol: ModuleProtocol::Subc,
8592                overlap: Default::default(),
8593            })
8594            .unwrap();
8595
8596        let deadline = Instant::now() + Duration::from_secs(5);
8597        loop {
8598            let tail = module.stderr_tail(None, None);
8599            if tail
8600                .entries
8601                .iter()
8602                .any(|entry| matches!(entry, TailEntry::ProcessStart))
8603                && tail.entries.iter().any(|entry| {
8604                    matches!(
8605                        entry,
8606                        TailEntry::Line {
8607                            truncated: true,
8608                            ..
8609                        }
8610                    )
8611                })
8612            {
8613                break;
8614            }
8615            assert!(
8616                Instant::now() < deadline,
8617                "module did not produce a truncated line and restart boundary: {tail:?}"
8618            );
8619            sleep(Duration::from_millis(10)).await;
8620        }
8621
8622        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8623        let request = ClientControlRequest::SupervisorStderrTail {
8624            module_id: "stderr-tail-wire".to_string(),
8625            max_lines: None,
8626            max_bytes: None,
8627        };
8628        let frame = Frame::build(
8629            FrameType::Request,
8630            control_flags(),
8631            0,
8632            0,
8633            1,
8634            serde_json::to_vec(&request).unwrap(),
8635        )
8636        .unwrap();
8637        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8638        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8639        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8640            serde_json::from_slice(&responses[0].body).unwrap()
8641        else {
8642            panic!("expected supervisor.stderr_tail response");
8643        };
8644
8645        assert!(
8646            tail.entries
8647                .iter()
8648                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8649            "the control response lost the restart boundary"
8650        );
8651        let Some(StderrTailEntry::Line {
8652            text,
8653            truncated,
8654            at_ms,
8655        }) = tail.entries.iter().find(|entry| {
8656            matches!(
8657                entry,
8658                StderrTailEntry::Line {
8659                    truncated: true,
8660                    ..
8661                }
8662            )
8663        })
8664        else {
8665            panic!("the control response lost the truncated line");
8666        };
8667        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8668        assert!(*truncated);
8669        assert!(
8670            at_ms.is_some(),
8671            "the control response lost the line's capture time"
8672        );
8673    }
8674
8675    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
8676    /// read done on the worker thread would stall every other task until it
8677    /// finished; the read must run off the worker so this test's own task keeps
8678    /// running while the read is paused.
8679    #[tokio::test(flavor = "current_thread")]
8680    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8681        let dir = ScratchDir::new("terminals-off-worker");
8682        let journal_path = dir.join("terminals.jsonl");
8683        let registry = Arc::new(Registry::default());
8684        let supervisor_handle = SupervisorHandle::new();
8685        let supervisor =
8686            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8687                .with_handle(supervisor_handle.clone())
8688                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8689        let module = supervisor
8690            .spawn(ModuleSpec {
8691                module_id: "terminal-off-worker".to_string(),
8692                program: fake_aft_stub_path(),
8693                args: Vec::new(),
8694                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8695                reserved: false,
8696                reserved_prefixes: Vec::new(),
8697                protocol: ModuleProtocol::Subc,
8698                overlap: Default::default(),
8699            })
8700            .unwrap();
8701        let deadline = Instant::now() + Duration::from_secs(5);
8702        while module.terminal_history().entries.len() != 2 {
8703            assert!(Instant::now() < deadline, "module did not record two exits");
8704            sleep(Duration::from_millis(10)).await;
8705        }
8706
8707        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8708        let handler =
8709            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
8710        let frame = Frame::build(
8711            FrameType::Request,
8712            control_flags(),
8713            0,
8714            0,
8715            1,
8716            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8717                module_id: "terminal-off-worker".to_string(),
8718            })
8719            .unwrap(),
8720        )
8721        .unwrap();
8722        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8723        let spawned_at = std::time::Instant::now();
8724        let read = tokio::spawn({
8725            let handler = Arc::clone(&handler);
8726            async move { handler.handle_control_frame(&ctx, frame).await }
8727        });
8728        // Waiting for the pause from a blocking thread keeps this task pending,
8729        // so the runtime's single worker is free to run the read task.
8730        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8731            .await
8732            .unwrap()
8733            .expect("the history read reached its pause");
8734        let elapsed = spawned_at.elapsed();
8735        assert!(
8736            elapsed < Duration::from_secs(2) && !read.is_finished(),
8737            "this task could not run while the history read was paused \
8738             (resumed after {elapsed:?}, read finished: {})",
8739            read.is_finished()
8740        );
8741
8742        drop(release);
8743        let responses = read.await.unwrap().unwrap();
8744        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8745        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8746            panic!("expected supervisor.terminals response");
8747        };
8748        assert_eq!(terminals.entries.len(), 2);
8749        assert_eq!(terminals.journal_skipped_lines, 0);
8750        assert_eq!(terminals.journal_read_errors, 0);
8751    }
8752
8753    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8754    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8755        let registry = Arc::new(Registry::default());
8756        let supervisor_handle = SupervisorHandle::new();
8757        let supervisor =
8758            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8759                .with_handle(supervisor_handle.clone());
8760        let module = supervisor
8761            .spawn(ModuleSpec {
8762                module_id: "terminal-golden".to_string(),
8763                program: fake_aft_stub_path(),
8764                args: Vec::new(),
8765                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8766                reserved: false,
8767                reserved_prefixes: Vec::new(),
8768                protocol: ModuleProtocol::Subc,
8769                overlap: Default::default(),
8770            })
8771            .unwrap();
8772
8773        let deadline = Instant::now() + Duration::from_secs(5);
8774        while module.terminal_history().entries.len() != 2 {
8775            assert!(
8776                Instant::now() < deadline,
8777                "module did not retain two terminal exits: {:?}",
8778                module.terminal_history()
8779            );
8780            sleep(Duration::from_millis(10)).await;
8781        }
8782
8783        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8784        let request = ClientControlRequest::SupervisorTerminals {
8785            module_id: "terminal-golden".to_string(),
8786        };
8787        let frame = Frame::build(
8788            FrameType::Request,
8789            control_flags(),
8790            0,
8791            0,
8792            1,
8793            serde_json::to_vec(&request).unwrap(),
8794        )
8795        .unwrap();
8796        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8797        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8798        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8799        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8800            panic!("expected supervisor.terminals response");
8801        };
8802        assert_eq!(terminals.entries.len(), 2);
8803        assert_eq!(terminals.dropped, 0);
8804
8805        let mut rendered = serde_json::to_value(response).unwrap();
8806        // Wall-clock fields are the observation contract, but not stable fixture
8807        // bytes; normalize only them after the real handler has shaped the response.
8808        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8809        for (index, entry) in rendered["entries"]
8810            .as_array_mut()
8811            .expect("terminal response entries array")
8812            .iter_mut()
8813            .enumerate()
8814        {
8815            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8816        }
8817
8818        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8819            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8820        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8821        if std::env::var_os("UPDATE_GOLDEN").is_some() {
8822            std::fs::write(&golden_path, &serialized).unwrap();
8823        }
8824        let expected: Value =
8825            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8826        assert_eq!(rendered, expected);
8827    }
8828
8829    #[test]
8830    fn hello_registers_manifest_and_returns_ack() {
8831        let registry = Arc::new(Registry::default());
8832        let handler = ControlHandler::new(Arc::clone(&registry));
8833        let conn = ConnectionId::new(1);
8834
8835        let responses = handler
8836            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8837            .unwrap();
8838
8839        assert_eq!(responses.len(), 1);
8840        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8841        assert_eq!(responses[0].header.channel, 0);
8842        assert_eq!(responses[0].header.corr, 7);
8843        let ack = parse_ack(&responses[0]);
8844        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8845        assert!(ack
8846            .subc_capabilities
8847            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8848        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8849        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8850        assert!(ack
8851            .subc_ops
8852            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8853        assert!(ack
8854            .subc_ops
8855            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8856
8857        let registration = registry.get_module("aft").unwrap().unwrap();
8858        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8859        assert_eq!(registration.state, ChannelState::Active);
8860        assert_eq!(registration.connection_id, conn);
8861        assert_eq!(registration.control_ops, module_baseline_control_ops());
8862    }
8863
8864    #[test]
8865    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8866        let invalid_identifiers = [
8867            ("case_change", "credentials-Provider/v1"),
8868            ("leading_zero", "credentials-provider/v01"),
8869            ("trailing_hyphen", "credentials-provider-/v1"),
8870            ("consecutive_hyphens", "credentials--provider/v1"),
8871            ("uppercase", "Credentials-provider/v1"),
8872            ("missing_v", "credentials-provider/1"),
8873            ("whitespace", "credentials provider/v1"),
8874            ("zero_version", "credentials-provider/v0"),
8875            ("out_of_range_version", "credentials-provider/v4294967296"),
8876            (
8877                "overlength_name",
8878                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8879            ),
8880        ];
8881        let mut cases = invalid_identifiers
8882            .into_iter()
8883            .map(|(name, identifier)| {
8884                (
8885                    format!("identifier_{name}"),
8886                    "capabilities.provides[0]".to_string(),
8887                    identifier.to_string(),
8888                    json!({ "provides": [identifier] }),
8889                    None,
8890                )
8891            })
8892            .collect::<Vec<_>>();
8893        cases.extend([
8894            (
8895                "unknown_need".to_string(),
8896                "capabilities.requires[0].need".to_string(),
8897                "deferred".to_string(),
8898                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8899                None,
8900            ),
8901            (
8902                "duplicate_provides".to_string(),
8903                "capabilities.provides[1]".to_string(),
8904                "credentials-provider/v1".to_string(),
8905                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8906                None,
8907            ),
8908            (
8909                "duplicate_must_never_reach".to_string(),
8910                "capabilities.must_never_reach[1]".to_string(),
8911                "credentials-provider/v1".to_string(),
8912                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8913                None,
8914            ),
8915            (
8916                "duplicate_requires_same_need".to_string(),
8917                "capabilities.requires[1]".to_string(),
8918                "credentials-provider/v1".to_string(),
8919                json!({ "requires": [
8920                    { "capability": "credentials-provider/v1", "need": "required" },
8921                    { "capability": "credentials-provider/v1", "need": "required" }
8922                ] }),
8923                None,
8924            ),
8925            (
8926                "duplicate_requires_conflicting_need".to_string(),
8927                "capabilities.requires[1]".to_string(),
8928                "credentials-provider/v1".to_string(),
8929                json!({ "requires": [
8930                    { "capability": "credentials-provider/v1", "need": "required" },
8931                    { "capability": "credentials-provider/v1", "need": "optional" }
8932                ] }),
8933                None,
8934            ),
8935            (
8936                "capabilities_root_pointer".to_string(),
8937                "runtime_computed[0]".to_string(),
8938                "/capabilities".to_string(),
8939                json!({}),
8940                Some(json!(["/capabilities"])),
8941            ),
8942            (
8943                "capabilities_descendant_pointer".to_string(),
8944                "runtime_computed[0]".to_string(),
8945                "/capabilities/provides".to_string(),
8946                json!({}),
8947                Some(json!(["/capabilities/provides"])),
8948            ),
8949            (
8950                "malformed_pointer_without_leading_slash".to_string(),
8951                "runtime_computed[0]".to_string(),
8952                "capabilities".to_string(),
8953                json!({}),
8954                Some(json!(["capabilities"])),
8955            ),
8956            (
8957                "malformed_pointer_escape".to_string(),
8958                "runtime_computed[0]".to_string(),
8959                "/roles/~2/tools".to_string(),
8960                json!({}),
8961                Some(json!(["/roles/~2/tools"])),
8962            ),
8963            (
8964                "unknown_capabilities_field".to_string(),
8965                "capabilities.future".to_string(),
8966                "<array>".to_string(),
8967                json!({ "future": [] }),
8968                None,
8969            ),
8970        ]);
8971
8972        for (index, (name, field, value, capabilities, runtime_computed)) in
8973            cases.into_iter().enumerate()
8974        {
8975            let registry = Arc::new(Registry::default());
8976            let handler = ControlHandler::new(Arc::clone(&registry));
8977            let response = handler
8978                .handle_control(
8979                    ConnectionId::new((index + 1) as u64),
8980                    capability_grammar_hello_frame(
8981                        capabilities,
8982                        runtime_computed,
8983                        index as u64 + 1,
8984                    ),
8985                )
8986                .expect("invalid HELLO returns a refusal");
8987
8988            assert_eq!(response.len(), 1, "{name} must emit one refusal");
8989            let error = parse_error(&response[0]);
8990            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8991            let message = error["message"]
8992                .as_str()
8993                .expect("error message is a string");
8994            assert!(
8995                message.contains(&field),
8996                "{name}: field missing from {message}"
8997            );
8998            assert!(
8999                message.contains(&value),
9000                "{name}: value missing from {message}"
9001            );
9002            assert_eq!(
9003                registry
9004                    .active_registration_count()
9005                    .expect("registry reads"),
9006                0,
9007                "{name}: refused HELLO must not create a catalog entry"
9008            );
9009        }
9010    }
9011
9012    #[test]
9013    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9014        let registry = Arc::new(Registry::default());
9015        let handler = ControlHandler::new(Arc::clone(&registry));
9016        let capabilities = json!({
9017            "provides": ["credentials-provider/v1"],
9018            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9019            "must_never_reach": ["federation-transport/v1"]
9020        });
9021        let response = handler
9022            .handle_control(
9023                ConnectionId::new(99),
9024                capability_grammar_hello_frame(
9025                    capabilities.clone(),
9026                    Some(json!(["/roles/0/tools"])),
9027                    99,
9028                ),
9029            )
9030            .expect("valid HELLO registers");
9031        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9032
9033        let request = Frame::build(
9034            FrameType::Request,
9035            control_flags(),
9036            0,
9037            0,
9038            100,
9039            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9040                .expect("catalog request serializes"),
9041        )
9042        .expect("catalog request frame builds");
9043        let response = handler
9044            .handle_catalog_list(request, None)
9045            .expect("catalog list succeeds");
9046        let ClientControlResponse::CatalogList { modules, .. } =
9047            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9048        else {
9049            panic!("catalog request must return catalog.list");
9050        };
9051        assert_eq!(modules.len(), 1);
9052        assert_eq!(
9053            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9054            capabilities
9055        );
9056    }
9057
9058    #[test]
9059    fn catalog_list_mirrors_management_operation_description() {
9060        let registry = Arc::new(Registry::default());
9061        let handler = ControlHandler::new(Arc::clone(&registry));
9062        let description = "List managed records and return their identifiers and metadata.";
9063        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9064        manifest.provides = vec![ProviderRole::ManagementSurface {
9065            operations: vec![ManagementOperation {
9066                name: "records.list".to_string(),
9067                kind: ManagementOperationKind::Query,
9068                description: Some(description.to_string()),
9069            }],
9070            config_schema: json!({"type": "object"}),
9071            observability: vec![ObservabilitySurface {
9072                name: "records.stats".to_string(),
9073                kind: ObservabilityKind::Snapshot,
9074            }],
9075            identity_scope: vec![IdentityScope::Project],
9076            concurrency: Concurrency::ModuleManaged,
9077        }];
9078        registry
9079            .register_with_control_ops(
9080                manifest,
9081                PROTOCOL_VERSION,
9082                ConnectionId::new(99),
9083                Vec::new(),
9084            )
9085            .expect("described management manifest registers");
9086
9087        let request = Frame::build(
9088            FrameType::Request,
9089            control_flags(),
9090            0,
9091            0,
9092            100,
9093            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9094                .expect("catalog request serializes"),
9095        )
9096        .expect("catalog request frame builds");
9097        let response = handler
9098            .handle_catalog_list(request, None)
9099            .expect("catalog list succeeds");
9100        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9101        assert_eq!(
9102            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9103            "catalog.list must preserve the declared operation description verbatim"
9104        );
9105    }
9106
9107    #[test]
9108    fn catalog_list_mirrors_registered_operation_delivery() {
9109        let registry = Arc::new(Registry::default());
9110        let handler = ControlHandler::new(Arc::clone(&registry));
9111        let delivery =
9112            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9113        let mut body = serde_json::to_value(ModuleHelloBody {
9114            manifest: manifest("watch-management", PROTOCOL_VERSION),
9115            protocol_ver: PROTOCOL_VERSION,
9116            control_ops: None,
9117            launch_nonce: None,
9118        })
9119        .unwrap();
9120        body["manifest"]["provides"] = json!([{
9121            "role": "management_surface",
9122            "operations": [
9123                {"name": "status", "kind": "query"},
9124                {"name": "future", "kind": "query"},
9125                {"name": "reset", "kind": "mutate"}
9126            ],
9127            "config_schema": {}, "observability": [], "identity_scope": []
9128        }]);
9129        body["manifest"]["operation_delivery"] = delivery.clone();
9130        let hello = Frame::build(
9131            FrameType::Hello,
9132            control_flags(),
9133            0,
9134            0,
9135            99,
9136            serde_json::to_vec(&body).unwrap(),
9137        )
9138        .unwrap();
9139        let response = handler
9140            .handle_control(ConnectionId::new(99), hello)
9141            .unwrap();
9142        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9143
9144        let (_, registrations) = registry.list_modules().unwrap();
9145        let registered = &registrations[0].manifest;
9146        assert!(registered.is_watch_operation("status"));
9147        assert!(!registered.is_watch_operation("future"));
9148        assert!(!registered.is_watch_operation("reset"));
9149        assert!(!registered.is_watch_operation("missing"));
9150
9151        let request = Frame::build(
9152            FrameType::Request,
9153            control_flags(),
9154            0,
9155            0,
9156            100,
9157            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9158        )
9159        .unwrap();
9160        let response = handler.handle_catalog_list(request, None).unwrap();
9161        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9162        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9163        let ClientControlResponse::CatalogList { modules, .. } =
9164            serde_json::from_slice(&response[0].body).unwrap()
9165        else {
9166            panic!("expected catalog.list");
9167        };
9168        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9169    }
9170
9171    #[test]
9172    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9173        let registry = Arc::new(Registry::default());
9174        let supervisor = SupervisorHandle::new();
9175        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9176        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9177        let descriptor = json!({
9178            "schema": "ckios.richui.manifest/1",
9179            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9180            "bytes": 4096,
9181            "chunks": 4
9182        });
9183        let connection = ConnectionId::new(201);
9184        let hello = handler
9185            .handle_control(
9186                connection,
9187                hello_frame_with_rich_ui(
9188                    "vault",
9189                    PROTOCOL_VERSION,
9190                    201,
9191                    Some("the-real-nonce"),
9192                    descriptor.clone(),
9193                ),
9194            )
9195            .unwrap();
9196        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9197
9198        let request =
9199            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9200        let provides = registry
9201            .get_module_by_connection(connection)
9202            .unwrap()
9203            .unwrap()
9204            .manifest
9205            .provides;
9206        handler
9207            .handle_catalog_update(connection, request, provides, None, Some(false))
9208            .unwrap();
9209
9210        let request = Frame::build(
9211            FrameType::Request,
9212            control_flags(),
9213            0,
9214            0,
9215            203,
9216            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9217        )
9218        .unwrap();
9219        let response = handler.handle_catalog_list(request, None).unwrap();
9220        let ClientControlResponse::CatalogList { modules, .. } =
9221            serde_json::from_slice(&response[0].body).unwrap()
9222        else {
9223            panic!("expected catalog.list");
9224        };
9225        assert_eq!(
9226            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9227            descriptor
9228        );
9229        assert!(modules[0].nonce_verified);
9230    }
9231
9232    #[test]
9233    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9234        let registry = Arc::new(Registry::default());
9235        let handler = ControlHandler::new(registry.clone());
9236        let descriptor = json!({
9237            "schema": "ckios.richui.manifest/1",
9238            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9239            "bytes": 128,
9240            "chunks": 2
9241        });
9242        let malformed = [
9243            ("schema", json!("")),
9244            ("schema", json!("a".repeat(65))),
9245            ("schema", json!("Bad schema")),
9246            (
9247                "manifest_digest",
9248                json!(format!("sha256:{}", "A".repeat(64))),
9249            ),
9250            ("manifest_digest", json!("sha256:abc")),
9251            ("bytes", json!(0)),
9252            ("bytes", json!(16_777_217)),
9253            ("bytes", json!("128")),
9254            ("chunks", json!(0)),
9255            ("chunks", json!(1025)),
9256            ("chunks", json!("2")),
9257        ];
9258        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9259            let mut invalid = descriptor.clone();
9260            invalid[field] = invalid_value;
9261            let response = handler
9262                .handle_control(
9263                    ConnectionId::new(210 + index as u64),
9264                    hello_frame_with_rich_ui(
9265                        &format!("bad-rich-ui-{index}"),
9266                        PROTOCOL_VERSION,
9267                        210 + index as u64,
9268                        None,
9269                        invalid,
9270                    ),
9271                )
9272                .unwrap();
9273            assert_eq!(response[0].header.ty, FrameType::Error);
9274            let error = parse_error(&response[0]);
9275            assert_eq!(error["code"], "invalid_manifest");
9276            assert!(error["message"]
9277                .as_str()
9278                .unwrap()
9279                .contains(&format!("rich_ui.{field}")));
9280            assert!(registry
9281                .get_module(&format!("bad-rich-ui-{index}"))
9282                .unwrap()
9283                .is_none());
9284        }
9285    }
9286
9287    #[test]
9288    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9289        let registry = Arc::new(Registry::default());
9290        let handler = ControlHandler::new(Arc::clone(&registry));
9291        let connection = ConnectionId::new(219);
9292        registry
9293            .register_with_control_ops(
9294                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9295                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9296                        "bad schema",
9297                        format!("sha256:{}", "a".repeat(64)),
9298                        128,
9299                        2,
9300                    )))
9301                    .build(),
9302                PROTOCOL_VERSION,
9303                connection,
9304                Vec::new(),
9305            )
9306            .unwrap();
9307        let request =
9308            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9309        let response = handler
9310            .handle_catalog_update(connection, request, Vec::new(), None, None)
9311            .unwrap();
9312        assert_eq!(response[0].header.ty, FrameType::Error);
9313        let error = parse_error(&response[0]);
9314        assert_eq!(error["code"], "invalid_manifest");
9315        assert!(error["message"]
9316            .as_str()
9317            .unwrap()
9318            .contains("rich_ui.schema"));
9319    }
9320
9321    #[test]
9322    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9323        let registry = Arc::new(Registry::default());
9324        let supervisor = SupervisorHandle::new();
9325        supervisor.open_swap("provider", "candidate-nonce".to_string());
9326        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9327        let connection = ConnectionId::new(218);
9328        let hello = handler
9329            .handle_control(
9330                connection,
9331                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9332            )
9333            .unwrap();
9334        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9335        assert!(registry.list_modules().unwrap().1.is_empty());
9336
9337        registry.promote_candidate("provider").unwrap().unwrap();
9338        let request = Frame::build(
9339            FrameType::Request,
9340            control_flags(),
9341            0,
9342            0,
9343            219,
9344            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9345        )
9346        .unwrap();
9347        let response = handler.handle_catalog_list(request, None).unwrap();
9348        let ClientControlResponse::CatalogList { modules, .. } =
9349            serde_json::from_slice(&response[0].body).unwrap()
9350        else {
9351            panic!("expected catalog.list");
9352        };
9353        assert_eq!(modules.len(), 1);
9354        assert!(modules[0].nonce_verified);
9355    }
9356
9357    #[test]
9358    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9359        let registry = Arc::new(Registry::default());
9360        let handler = ControlHandler::new(Arc::clone(&registry));
9361        let hello = Frame::build(
9362            FrameType::Hello,
9363            control_flags(),
9364            0,
9365            0,
9366            220,
9367            serde_json::to_vec(&ModuleHelloBody {
9368                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9369                protocol_ver: PROTOCOL_VERSION,
9370                control_ops: None,
9371                launch_nonce: Some("unverified".to_string()),
9372            })
9373            .unwrap(),
9374        )
9375        .unwrap();
9376        assert_eq!(
9377            handler
9378                .handle_control(ConnectionId::new(220), hello)
9379                .unwrap()[0]
9380                .header
9381                .ty,
9382            FrameType::HelloAck
9383        );
9384        let request = Frame::build(
9385            FrameType::Request,
9386            control_flags(),
9387            0,
9388            0,
9389            221,
9390            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9391        )
9392        .unwrap();
9393        let response = handler.handle_catalog_list(request, None).unwrap();
9394        let ClientControlResponse::CatalogList { modules, .. } =
9395            serde_json::from_slice(&response[0].body).unwrap()
9396        else {
9397            panic!("expected catalog.list");
9398        };
9399        assert_eq!(
9400            serde_json::to_vec(&modules[0]).unwrap(),
9401            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9402        );
9403        assert!(!modules[0].nonce_verified);
9404    }
9405
9406    #[test]
9407    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9408        let registry = Arc::new(Registry::default());
9409        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9410            [("vault".to_string(), true), ("squatter".to_string(), true)],
9411            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9412        );
9413        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9414        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9415            provides: vec!["credentials-provider/v1".to_string()],
9416            requires: Vec::new(),
9417            must_never_reach: Vec::new(),
9418        });
9419        let frame = Frame::build(
9420            FrameType::Hello,
9421            control_flags(),
9422            0,
9423            0,
9424            77,
9425            serde_json::to_vec(&ModuleHelloBody {
9426                manifest: squatter,
9427                protocol_ver: PROTOCOL_VERSION,
9428                control_ops: None,
9429                launch_nonce: None,
9430            })
9431            .expect("HELLO serializes"),
9432        )
9433        .expect("HELLO frame builds");
9434        let response = handler
9435            .handle_control(ConnectionId::new(77), frame)
9436            .expect("reserved claim receives a typed refusal");
9437        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9438        assert_eq!(
9439            registry
9440                .active_registration_count()
9441                .expect("registry reads"),
9442            0,
9443            "a reserved capability refusal must not leave a catalog entry"
9444        );
9445    }
9446
9447    #[test]
9448    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9449        for settlement in ["timeout", "inconclusive", "drop"] {
9450            let breakers = RouteBindBreakers::default();
9451            let RouteBindAdmission::Admitted {
9452                guard: mut old,
9453                probe: false,
9454            } = breakers.admit("prov")
9455            else {
9456                panic!("ordinary relay admitted")
9457            };
9458            let RouteBindAdmission::Admitted {
9459                guard: mut opener, ..
9460            } = breakers.admit("prov")
9461            else {
9462                panic!("second relay admitted")
9463            };
9464            assert!(
9465                !opener
9466                    .record_timeout(1, Duration::ZERO)
9467                    .unwrap()
9468                    .reopened_after_probe
9469            );
9470            let RouteBindAdmission::Admitted {
9471                guard: mut probe,
9472                probe: true,
9473            } = breakers.admit("prov")
9474            else {
9475                panic!("one cooldown probe admitted")
9476            };
9477            match settlement {
9478                "timeout" => assert!(
9479                    !old.record_timeout(1, Duration::ZERO)
9480                        .unwrap()
9481                        .reopened_after_probe
9482                ),
9483                "inconclusive" => old.record_inconclusive(),
9484                "drop" => drop(old),
9485                _ => unreachable!(),
9486            }
9487            assert!(
9488                matches!(
9489                    breakers.admit("prov"),
9490                    RouteBindAdmission::Refused {
9491                        probe_in_flight: true,
9492                        ..
9493                    }
9494                ),
9495                "{settlement} of a pre-open relay cannot release the real probe"
9496            );
9497            assert!(
9498                probe
9499                    .record_timeout(1, Duration::ZERO)
9500                    .unwrap()
9501                    .reopened_after_probe
9502            );
9503            assert!(matches!(
9504                breakers.admit("prov"),
9505                RouteBindAdmission::Admitted { probe: true, .. }
9506            ));
9507        }
9508        let breakers = RouteBindBreakers::default();
9509        let admit = || match breakers.admit("prov") {
9510            RouteBindAdmission::Admitted { guard, .. } => guard,
9511            _ => panic!("relay admitted"),
9512        };
9513        admit().record_timeout(1, Duration::ZERO);
9514        let mut old_probe = admit();
9515        breakers.reset_for_new_module_connection("prov");
9516        admit().record_timeout(1, Duration::ZERO);
9517        let _new_probe = admit();
9518        old_probe.record_inconclusive();
9519        assert!(matches!(
9520            breakers.admit("prov"),
9521            RouteBindAdmission::Refused {
9522                probe_in_flight: true,
9523                ..
9524            }
9525        ));
9526    }
9527
9528    #[tokio::test]
9529    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9530        for candidate in [false, true] {
9531            let registry = Arc::new(Registry::default());
9532            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9533                [("vault".to_string(), true), ("squatter".to_string(), true)],
9534                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9535            );
9536            let conn = ConnectionId::new(77);
9537            let (ctx, mut rx) = route_ctx(conn);
9538            let initial = capability_manifest("squatter", &[], &[]);
9539            if candidate {
9540                registry
9541                    .register_candidate_with_control_ops(
9542                        initial.clone(),
9543                        PROTOCOL_VERSION,
9544                        conn,
9545                        module_baseline_control_ops(),
9546                    )
9547                    .unwrap();
9548                handler
9549                    .forwarding
9550                    .register_candidate_module_connection(
9551                        conn,
9552                        "squatter".to_string(),
9553                        PROTOCOL_VERSION,
9554                        manifest_concurrency(&initial),
9555                        ctx.egress.clone(),
9556                    )
9557                    .unwrap();
9558            } else {
9559                hello_via_sink(
9560                    &handler,
9561                    &ctx,
9562                    &mut rx,
9563                    hello_frame_with_manifest(initial.clone(), 1),
9564                )
9565                .await;
9566            }
9567            let response = handler
9568                .handle_control_frame(
9569                    &ctx,
9570                    catalog_update_with_capabilities_frame(
9571                        2,
9572                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
9573                            .capabilities
9574                            .unwrap(),
9575                    ),
9576                )
9577                .await
9578                .unwrap();
9579            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9580            assert_eq!(
9581                registry
9582                    .get_module_by_connection(conn)
9583                    .unwrap()
9584                    .unwrap()
9585                    .manifest,
9586                initial
9587            );
9588        }
9589    }
9590
9591    #[test]
9592    fn server_describe_surfaces_required_capability_verdict_fields() {
9593        let registry = Arc::new(Registry::default());
9594        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9595            [
9596                ("consumer".to_string(), true),
9597                ("provider".to_string(), false),
9598            ],
9599            BTreeMap::new(),
9600        );
9601        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9602        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9603            provides: Vec::new(),
9604            requires: vec![subc_protocol::manifest::CapabilityRequirement {
9605                capability: "credentials-provider/v1".to_string(),
9606                need: subc_protocol::manifest::CapabilityNeed::Required,
9607            }],
9608            must_never_reach: Vec::new(),
9609        });
9610        let hello = Frame::build(
9611            FrameType::Hello,
9612            control_flags(),
9613            0,
9614            0,
9615            78,
9616            serde_json::to_vec(&ModuleHelloBody {
9617                manifest: consumer,
9618                protocol_ver: PROTOCOL_VERSION,
9619                control_ops: None,
9620                launch_nonce: None,
9621            })
9622            .expect("HELLO serializes"),
9623        )
9624        .expect("HELLO frame builds");
9625        handler
9626            .handle_control(ConnectionId::new(78), hello)
9627            .expect("consumer registers");
9628        let describe = Frame::build(
9629            FrameType::Request,
9630            control_flags(),
9631            0,
9632            0,
9633            79,
9634            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9635                .expect("request serializes"),
9636        )
9637        .expect("describe frame builds");
9638        let response = handler
9639            .handle_server_describe(describe)
9640            .expect("server.describe succeeds");
9641        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9642        let requirement = &rendered["capability_requirements"][0];
9643        assert_eq!(requirement["consumer"], "consumer");
9644        assert_eq!(requirement["verdict"], "never_provided");
9645        assert_eq!(requirement["episode_seq"], 1);
9646        assert_eq!(requirement["config_satisfiable"], false);
9647        assert_eq!(requirement["runtime_available"], false);
9648        assert!(requirement["detail"]
9649            .as_str()
9650            .expect("detail string")
9651            .contains("credentials-provider/v1"));
9652    }
9653
9654    /// Register a consumer that requires `project-identity/v1` and the provider
9655    /// of it, optionally mark daemon shutdown as begun, then deregister the
9656    /// provider as its connection closes. Returns the `capability.requirement`
9657    /// events that deregistration logged and the verdict the evaluator then
9658    /// holds for the requirement (the one `route.open` refusals read).
9659    fn deregister_required_provider(
9660        daemon_shutdown_begun: bool,
9661    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
9662        const CAPABILITY: &str = "project-identity/v1";
9663        let registry = Arc::new(Registry::default());
9664        let daemon_shutdown = DaemonShutdownFlag::default();
9665        let handler = ControlHandler::new(Arc::clone(&registry))
9666            .with_capability_config(
9667                [
9668                    ("consumer".to_string(), true),
9669                    ("provider".to_string(), true),
9670                ],
9671                BTreeMap::new(),
9672            )
9673            .with_daemon_shutdown_flag(daemon_shutdown.clone());
9674        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9675        consumer.capabilities = Some(CapabilityDeclarations {
9676            provides: Vec::new(),
9677            requires: vec![subc_protocol::manifest::CapabilityRequirement {
9678                capability: CAPABILITY.to_string(),
9679                need: subc_protocol::manifest::CapabilityNeed::Required,
9680            }],
9681            must_never_reach: Vec::new(),
9682        });
9683        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
9684        let provider_conn = ConnectionId::new(182);
9685        for (conn, manifest, corr) in [
9686            (ConnectionId::new(181), consumer, 181),
9687            (provider_conn, provider, 182),
9688        ] {
9689            let hello = handler
9690                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
9691                .expect("module registers");
9692            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9693        }
9694        assert_eq!(
9695            handler.capability_evaluator.verdict("consumer", CAPABILITY),
9696            Some(CapabilityVerdict::Provided),
9697            "the registered provider must satisfy the requirement first"
9698        );
9699        if daemon_shutdown_begun {
9700            daemon_shutdown.set_for_test();
9701        }
9702
9703        let capture = EventCapture::default();
9704        let _subscriber =
9705            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9706        handler
9707            .cleanup_connection(provider_conn)
9708            .expect("provider deregisters");
9709        let events = capture
9710            .events()
9711            .into_iter()
9712            .filter(|event| {
9713                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
9714            })
9715            .collect();
9716        (
9717            events,
9718            handler.capability_evaluator.verdict("consumer", CAPABILITY),
9719        )
9720    }
9721
9722    #[test]
9723    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
9724        let (events, verdict) = deregister_required_provider(true);
9725        assert_eq!(
9726            verdict,
9727            Some(CapabilityVerdict::NeverProvided),
9728            "shutdown must not change the verdict route.open refusals read"
9729        );
9730        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9731        let event = &events[0];
9732        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
9733        assert_eq!(
9734            event.fields.get("cause").map(String::as_str),
9735            Some("\"daemon_shutdown\""),
9736            "{event:?}"
9737        );
9738        assert_eq!(
9739            event.fields.get("verdict").map(String::as_str),
9740            Some("\"never_provided\""),
9741            "{event:?}"
9742        );
9743    }
9744
9745    #[test]
9746    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
9747        let (events, verdict) = deregister_required_provider(false);
9748        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
9749        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9750        let event = &events[0];
9751        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
9752        assert_eq!(event.fields.get("cause"), None, "{event:?}");
9753        assert_eq!(
9754            event.fields.get("verdict").map(String::as_str),
9755            Some("\"never_provided\""),
9756            "{event:?}"
9757        );
9758    }
9759
9760    #[test]
9761    fn catalog_list_omits_capabilities_for_legacy_manifest() {
9762        let registry = Arc::new(Registry::default());
9763        let handler = ControlHandler::new(Arc::clone(&registry));
9764        let hello = handler
9765            .handle_control(
9766                ConnectionId::new(101),
9767                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9768            )
9769            .expect("legacy HELLO registers");
9770        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9771
9772        let request = Frame::build(
9773            FrameType::Request,
9774            control_flags(),
9775            0,
9776            0,
9777            102,
9778            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9779                .expect("catalog request serializes"),
9780        )
9781        .expect("catalog request frame builds");
9782        let response = handler
9783            .handle_catalog_list(request, None)
9784            .expect("catalog list succeeds");
9785        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9786        assert!(
9787            body["modules"][0].get("capabilities").is_none(),
9788            "legacy manifest must retain an absent capabilities field on catalog.list"
9789        );
9790    }
9791
9792    #[test]
9793    fn hello_ack_omits_storage_when_no_storage_config() {
9794        let registry = Arc::new(Registry::default());
9795        let handler = ControlHandler::new(Arc::clone(&registry));
9796        let responses = handler
9797            .handle_control(
9798                ConnectionId::new(1),
9799                hello_frame("aft", PROTOCOL_VERSION, 7),
9800            )
9801            .unwrap();
9802        let ack = parse_ack(&responses[0]);
9803        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9804        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9805    }
9806
9807    #[tokio::test]
9808    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9809        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9810        let registry = Arc::new(Registry::default());
9811        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
9812        let responses = handler
9813            .handle_control(
9814                ConnectionId::new(1),
9815                hello_frame("aft", PROTOCOL_VERSION, 7),
9816            )
9817            .unwrap();
9818        let ack = parse_ack(&responses[0]);
9819        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9820
9821        let described = handler
9822            .handle_control_frame(
9823                &route_ctx(ConnectionId::new(2)).0,
9824                Frame::build(
9825                    FrameType::Request,
9826                    control_flags(),
9827                    0,
9828                    0,
9829                    9,
9830                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9831                )
9832                .unwrap(),
9833            )
9834            .await
9835            .unwrap();
9836        let ClientControlResponse::ServerDescribe { machine_id, .. } =
9837            serde_json::from_slice(&described[0].body).unwrap()
9838        else {
9839            panic!("server.describe answered with another shape");
9840        };
9841        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9842    }
9843
9844    #[test]
9845    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9846        // With a central sqlite storage policy, each registering module gets its
9847        // own resolved descriptor in HELLO_ACK, keyed by its module id.
9848        let registry = Arc::new(Registry::default());
9849        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
9850            crate::daemon_config::StorageConfig::Sqlite {
9851                data_home: std::path::PathBuf::from("/data"),
9852            },
9853        ));
9854
9855        let responses = handler
9856            .handle_control(
9857                ConnectionId::new(1),
9858                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9859            )
9860            .unwrap();
9861        let ack = parse_ack(&responses[0]);
9862        assert_eq!(
9863            ack.storage,
9864            Some(serde_json::json!({
9865                "module_id": "alfonso-routing",
9866                "storage_namespace": "default",
9867                "isolation": { "kind": "module" },
9868                "backend": {
9869                    "backend": "sqlite",
9870                    "path": "/data/cortexkit/alfonso-routing/store.db"
9871                }
9872            })),
9873            "the delivered descriptor is the module's own sqlite store path"
9874        );
9875    }
9876
9877    #[test]
9878    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9879        let registry = Arc::new(Registry::default());
9880        let handler = ControlHandler::new(Arc::clone(&registry));
9881        let conn = ConnectionId::new(1);
9882        let responses = handler
9883            .handle_control(
9884                conn,
9885                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9886            )
9887            .unwrap();
9888        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9889        let registration = registry.get_module("aft").unwrap().unwrap();
9890        assert_eq!(registration.control_ops, module_baseline_control_ops());
9891
9892        let frame =
9893            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9894        assert!(handler
9895            .guard_module_control_op(&frame, "aft", "route.bind")
9896            .unwrap()
9897            .is_none());
9898        let error = handler
9899            .guard_module_control_op(&frame, "aft", "test.synthetic")
9900            .unwrap()
9901            .expect("synthetic ungranted op should be rejected");
9902        assert_eq!(error.header.ty, FrameType::Error);
9903        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9904    }
9905
9906    #[test]
9907    fn hello_control_ops_some_adds_optional_grants() {
9908        let registry = Arc::new(Registry::default());
9909        let handler = ControlHandler::new(Arc::clone(&registry));
9910        handler
9911            .handle_control(
9912                ConnectionId::new(1),
9913                hello_frame_with_control_ops(
9914                    "aft",
9915                    PROTOCOL_VERSION,
9916                    7,
9917                    Some(vec![
9918                        "future.synthetic".to_string(),
9919                        "route.bind".to_string(),
9920                    ]),
9921                ),
9922            )
9923            .unwrap();
9924        let registration = registry.get_module("aft").unwrap().unwrap();
9925        assert_eq!(
9926            registration.control_ops,
9927            vec![
9928                "route.bind".to_string(),
9929                "route.status".to_string(),
9930                "future.synthetic".to_string(),
9931            ]
9932        );
9933        let frame =
9934            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9935        assert!(handler
9936            .guard_module_control_op(&frame, "aft", "future.synthetic")
9937            .unwrap()
9938            .is_none());
9939    }
9940
9941    #[tokio::test]
9942    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9943        let registry = Arc::new(Registry::default());
9944        let forwarding = Arc::new(ForwardingTable::default());
9945        let handler =
9946            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9947        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9948        hello_via_sink(
9949            &handler,
9950            &module_ctx,
9951            &mut module_rx,
9952            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9953        )
9954        .await;
9955
9956        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9957        let responses = handler
9958            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9959            .await
9960            .unwrap();
9961        assert_eq!(responses.len(), 1);
9962        assert_eq!(responses[0].header.ty, FrameType::Error);
9963        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9964        assert!(module_rx.try_recv().is_err());
9965    }
9966
9967    #[tokio::test]
9968    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9969        let registry = Arc::new(Registry::default());
9970        let forwarding = Arc::new(ForwardingTable::default());
9971        let handler =
9972            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9973        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9974        hello_via_sink(
9975            &handler,
9976            &module_ctx,
9977            &mut module_rx,
9978            hello_frame_with_control_ops(
9979                "aft",
9980                PROTOCOL_VERSION,
9981                7,
9982                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9983            ),
9984        )
9985        .await;
9986
9987        let project_root = unique_project_root("demux");
9988        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9989        let route_handler = handler.clone();
9990        let route_task = tokio::spawn(async move {
9991            route_handler
9992                .handle_control_frame(
9993                    &route_client_ctx,
9994                    route_open_frame(100, "aft", project_root),
9995                )
9996                .await
9997                .unwrap()
9998        });
9999        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10000            .await
10001            .unwrap()
10002            .unwrap();
10003        assert!(matches!(
10004            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10005            ModuleControlRequest::RouteBind { .. }
10006        ));
10007
10008        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10009        let health_handler = handler.clone();
10010        let health_task = tokio::spawn(async move {
10011            health_handler
10012                .handle_control_frame(
10013                    &health_client_ctx,
10014                    supervisor_health_probe_frame(101, "aft"),
10015                )
10016                .await
10017                .unwrap()
10018        });
10019        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10020            .await
10021            .unwrap()
10022            .unwrap();
10023        assert_eq!(
10024            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10025            ModuleControlRequest::HealthCheck {}
10026        );
10027
10028        handler
10029            .handle_control_frame(
10030                &module_ctx,
10031                health_response(health_frame.header.corr, HealthStatus::Degraded),
10032            )
10033            .await
10034            .unwrap();
10035        let health_response = health_task.await.unwrap();
10036        assert_eq!(health_response.len(), 1);
10037        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10038            ClientControlResponse::SupervisorHealthProbe {
10039                module_id,
10040                status,
10041                detail,
10042                metrics,
10043            } => {
10044                assert_eq!(module_id, "aft");
10045                assert_eq!(status, HealthStatus::Degraded);
10046                assert_eq!(detail.as_deref(), Some("warming"));
10047                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10048            }
10049            other => panic!("unexpected health response: {other:?}"),
10050        }
10051
10052        handler
10053            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10054            .await
10055            .unwrap();
10056        let route_response = route_task.await.unwrap();
10057        assert!(route_response.is_empty());
10058        let published = route_client_rx.recv().await.unwrap();
10059        assert!(matches!(
10060            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10061            ClientControlResponse::RouteOpen { .. }
10062        ));
10063    }
10064
10065    /// Start one `route.open` on `client_connection` and return its still-running
10066    /// handler task together with the `route.bind` the module received for it.
10067    /// The handler blocks until the module answers, so it has to run as a task
10068    /// while the test drives the module side.
10069    async fn relay_route_open(
10070        handler: &ControlHandler,
10071        client_connection: ConnectionId,
10072        client_egress: &FrameSink,
10073        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10074        corr: u64,
10075        module_id: &str,
10076        project_root_label: &str,
10077    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10078        let ctx = RouteCtx {
10079            connection_id: client_connection,
10080            egress: client_egress.clone(),
10081        };
10082        let handler = handler.clone();
10083        let project_root = unique_project_root(project_root_label);
10084        let module_id = module_id.to_string();
10085        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10086        let task = tokio::spawn(async move {
10087            let _guard = tracing::dispatcher::set_default(&dispatch);
10088            handler
10089                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10090                .await
10091                .unwrap()
10092        });
10093        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10094            .await
10095            .expect("module receives the relayed route.bind")
10096            .expect("module egress is open");
10097        (task, bind.frame)
10098    }
10099
10100    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10101        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10102            ModuleControlRequest::RouteBind {
10103                route_channel,
10104                epoch,
10105                ..
10106            } => (route_channel, epoch),
10107            other => panic!("expected a route.bind request, got {other:?}"),
10108        }
10109    }
10110
10111    fn published_route(frame: &Frame) -> (u16, u32) {
10112        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10113            ClientControlResponse::RouteOpen {
10114                route_channel,
10115                route_epoch,
10116            } => (route_channel, route_epoch),
10117            other => panic!("expected a route.open response, got {other:?}"),
10118        }
10119    }
10120
10121    /// Reproduction of a production outage. A client had `route.open`s in
10122    /// flight to a module and was already marked closing -- its egress had refused a
10123    /// module frame, so the daemon asked its connection to end -- while its sink
10124    /// was still open. When the module acked those binds, the daemon refused to
10125    /// commit a route for a closing client, and that refusal was returned from
10126    /// the MODULE connection's frame handler, where a router error that has no
10127    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
10128    /// the supervisor correctly did not respawn a clean exit, and every seat lost
10129    /// its tools for hours -- one client's teardown took down a connection
10130    /// carrying ~170 other routes.
10131    ///
10132    /// The window is opened here by calling the production path that opens it
10133    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
10134    /// state that matters is "in `closing_connections`, sink still open, relay
10135    /// still pending", and it lasts only from the close request until the
10136    /// connection loop reacts to it; a socket-level test can flood a client into
10137    /// that escalation but cannot pin the module's ack inside the window. Closing
10138    /// the socket instead takes the other path entirely -- connection teardown
10139    /// removes the pending relay under the same lock, so the ack finds nothing.
10140    #[tokio::test]
10141    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10142        let registry = Arc::new(Registry::default());
10143        let forwarding = Arc::new(ForwardingTable::default());
10144        let handler =
10145            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10146
10147        let module_connection = ConnectionId::new(30);
10148        let (module_ctx, mut module_rx) = route_ctx(module_connection);
10149        hello_via_sink(
10150            &handler,
10151            &module_ctx,
10152            &mut module_rx,
10153            hello_frame("aft", PROTOCOL_VERSION, 7),
10154        )
10155        .await;
10156
10157        let dying_client = ConnectionId::new(31);
10158        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10159
10160        // A published route on the dying client. The escalation below only marks
10161        // a connection closing for a route it has already published.
10162        let (first_task, first_bind) = relay_route_open(
10163            &handler,
10164            dying_client,
10165            &dying_ctx.egress,
10166            &mut module_rx,
10167            100,
10168            "aft",
10169            "closing-first",
10170        )
10171        .await;
10172        handler
10173            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10174            .await
10175            .unwrap();
10176        assert!(first_task.await.unwrap().is_empty());
10177        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10178
10179        // A second route.open from the same client, relayed and awaiting its ack.
10180        let (second_task, second_bind) = relay_route_open(
10181            &handler,
10182            dying_client,
10183            &dying_ctx.egress,
10184            &mut module_rx,
10185            101,
10186            "aft",
10187            "closing-second",
10188        )
10189        .await;
10190        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10191
10192        // The window: the client is closing, its sink is still open, and its
10193        // second bind is still pending.
10194        assert!(forwarding
10195            .escalate_client_delivery_failure(
10196                dying_client,
10197                first_channel,
10198                first_epoch,
10199                CloseReason::new(
10200                    "module_to_client_delivery_failed",
10201                    "client egress refused a module frame",
10202                ),
10203                crate::forwarding::UndeliveredFrame {
10204                    module_id: None,
10205                    sink: &dying_ctx.egress,
10206                },
10207            )
10208            .unwrap());
10209        assert!(!dying_ctx.egress.is_closed());
10210
10211        // The frame that used to end the module connection.
10212        let ack = handler
10213            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10214            .await;
10215        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10216        if module_loop_error.is_some() {
10217            // What the server's connection loop does with a router error that has
10218            // no ERROR-frame translation: end the connection, which releases the
10219            // module's registration and every route on it.
10220            handler.cleanup_connection(module_connection).unwrap();
10221        }
10222        // Read the module's next frame before opening the co-tenant's route, so
10223        // the GOODBYE assertion below is about THIS ack and not about later
10224        // traffic. `None` means the module was told nothing.
10225        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10226            .await
10227            .ok()
10228            .flatten();
10229
10230        // 1. The module connection is still registered.
10231        assert!(
10232            registry
10233                .get_module_by_connection(module_connection)
10234                .unwrap()
10235                .is_some(),
10236            "one client's closing connection ended the shared module connection: \
10237             {module_loop_error:?}"
10238        );
10239        // ...and still serving: another client can open and use a route on it.
10240        let cotenant = ConnectionId::new(32);
10241        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10242        let (cotenant_task, cotenant_bind) = relay_route_open(
10243            &handler,
10244            cotenant,
10245            &cotenant_ctx.egress,
10246            &mut module_rx,
10247            102,
10248            "aft",
10249            "closing-cotenant",
10250        )
10251        .await;
10252        handler
10253            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10254            .await
10255            .unwrap();
10256        assert!(cotenant_task.await.unwrap().is_empty());
10257        let (cotenant_channel, cotenant_epoch) =
10258            published_route(&cotenant_rx.recv().await.unwrap());
10259        assert!(matches!(
10260            forwarding
10261                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10262                .unwrap(),
10263            DataRoute::Client(DataRouteState::Bound(_))
10264        ));
10265
10266        // 2. The module was told to drop the binding it created for the route
10267        //    that will never be published.
10268        let goodbye = post_ack_module_frame
10269            .expect("module receives a GOODBYE for the abandoned route channel");
10270        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10271        assert_eq!(goodbye.header.channel, abandoned_channel);
10272        assert_eq!(goodbye.header.epoch, abandoned_epoch);
10273
10274        // 3. The dying client received nothing: no route was ever published to
10275        //    it. Its route.open is answered as unavailable, which the connection
10276        //    loop would write to a socket that is already going away.
10277        assert!(dying_rx.try_recv().is_err());
10278        let second_response = second_task.await.unwrap();
10279        assert_eq!(second_response.len(), 1);
10280        assert_eq!(
10281            parse_error(&second_response[0])["code"],
10282            "target_unavailable"
10283        );
10284    }
10285
10286    /// One `catalog.list` as a client sees it: the generation and the listed
10287    /// module ids with their readiness.
10288    async fn catalog_snapshot(
10289        handler: &ControlHandler,
10290        client_ctx: &RouteCtx,
10291        corr: u64,
10292    ) -> (u64, Vec<(String, bool)>) {
10293        let body =
10294            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10295        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10296        let reply = handler
10297            .handle_control_frame(client_ctx, request)
10298            .await
10299            .unwrap()
10300            .pop()
10301            .unwrap();
10302        let ClientControlResponse::CatalogList {
10303            generation,
10304            modules,
10305            ..
10306        } = serde_json::from_slice(&reply.body).unwrap()
10307        else {
10308            panic!("not a catalog.list reply");
10309        };
10310        (
10311            generation,
10312            modules
10313                .into_iter()
10314                .map(|entry| (entry.module_id, entry.ready))
10315                .collect(),
10316        )
10317    }
10318
10319    /// A handler whose HELLO for module "aft" on connection 40 is parked
10320    /// between reserving its registry entry and installing its forwarding
10321    /// endpoint. Returns the handler, the two-party barrier (wait once to know
10322    /// the HELLO is parked, once more to release it), the HELLO's thread, and
10323    /// the module's context and receiver.
10324    fn hello_held_before_forwarding() -> (
10325        ControlHandler,
10326        Arc<std::sync::Barrier>,
10327        std::thread::JoinHandle<Vec<Frame>>,
10328        RouteCtx,
10329        mpsc::Receiver<crate::router::OutboundFrame>,
10330    ) {
10331        let hold = Arc::new(std::sync::Barrier::new(2));
10332        let handler = ControlHandler::with_forwarding(
10333            Arc::new(Registry::default()),
10334            Arc::new(ForwardingTable::default()),
10335        )
10336        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10337        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10338        let hello = {
10339            let handler = handler.clone();
10340            let module_ctx = module_ctx.clone();
10341            std::thread::spawn(move || {
10342                handler
10343                    .handle_hello(
10344                        module_ctx.connection_id,
10345                        Some(module_ctx.egress.clone()),
10346                        hello_frame("aft", PROTOCOL_VERSION, 7),
10347                    )
10348                    .unwrap()
10349            })
10350        };
10351        (handler, hold, hello, module_ctx, module_rx)
10352    }
10353
10354    /// While a HELLO has reserved its registry entry but its forwarding
10355    /// endpoint is not installed yet, `route.open` to the module can only be
10356    /// refused. `catalog.list` must not list the module in that window: a
10357    /// client that opens a route as soon as it sees a module listed and ready
10358    /// would be refused for a module the daemon had just advertised. The
10359    /// refusal in the window stays the retryable `target_unavailable`, never
10360    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
10361    /// Once the HELLO completes, the module is listed and a route to it opens.
10362    #[tokio::test]
10363    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10364        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10365        let client = ConnectionId::new(41);
10366        let (client_ctx, mut client_rx) = route_ctx(client);
10367
10368        hold.wait();
10369        // Parked: the registry holds "aft", forwarding has no endpoint for it.
10370        assert!(handler.registry.get_module("aft").unwrap().is_some());
10371        assert!(!handler
10372            .forwarding
10373            .has_live_module_connection("aft")
10374            .unwrap());
10375        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10376        assert!(
10377            listed.is_empty(),
10378            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10379        );
10380        let refused = handler
10381            .handle_control_frame(
10382                &client_ctx,
10383                route_open_frame(301, "aft", unique_project_root("listed-window")),
10384            )
10385            .await
10386            .unwrap();
10387        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10388        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10389        let duplicate = handler
10390            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10391            .await
10392            .unwrap();
10393        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10394
10395        hold.wait();
10396        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10397        let ack = module_rx.try_recv().unwrap().frame;
10398        assert_eq!(ack.header.ty, FrameType::HelloAck);
10399        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10400        assert_eq!(listed, vec![("aft".to_string(), true)]);
10401
10402        let (open_task, bind) = relay_route_open(
10403            &handler,
10404            client,
10405            &client_ctx.egress,
10406            &mut module_rx,
10407            303,
10408            "aft",
10409            "listed-after",
10410        )
10411        .await;
10412        assert!(handler
10413            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10414            .await
10415            .unwrap()
10416            .is_empty());
10417        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10418        published_route(&client_rx.try_recv().unwrap().frame);
10419    }
10420
10421    /// A catalog watcher keys on the generation: when it changes, it re-reads
10422    /// the listing. So the generation must advance exactly when the module
10423    /// appears. If it advanced while the module was still hidden, a watcher
10424    /// would read the new generation without the module, and the module would
10425    /// later appear under that same generation with nothing telling the
10426    /// watcher to look again.
10427    #[tokio::test]
10428    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10429        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10430        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10431        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10432
10433        hold.wait();
10434        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10435        assert!(listed.is_empty());
10436        assert_eq!(
10437            during, before,
10438            "the generation moved while the module was not listed"
10439        );
10440
10441        hold.wait();
10442        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10443        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10444        assert_eq!(listed, vec![("aft".to_string(), true)]);
10445        assert_eq!(
10446            after,
10447            before + 1,
10448            "the module appeared without exactly one generation bump"
10449        );
10450    }
10451
10452    /// A HELLO whose forwarding endpoint cannot be installed (here the
10453    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
10454    /// refused and leaves nothing behind: no registry entry, no catalog entry,
10455    /// and the catalog generation where it was.
10456    #[tokio::test]
10457    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10458        let registry = Arc::new(Registry::default());
10459        let handler = ControlHandler::with_forwarding(
10460            Arc::clone(&registry),
10461            Arc::new(ForwardingTable::default()),
10462        );
10463        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10464        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10465        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10466        drop(module_rx);
10467
10468        let replies = handler
10469            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10470            .await
10471            .unwrap();
10472        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10473        assert!(registry.get_module("aft").unwrap().is_none());
10474        assert!(registry
10475            .get_module_by_connection(module_ctx.connection_id)
10476            .unwrap()
10477            .is_none());
10478        assert_eq!(
10479            catalog_snapshot(&handler, &client_ctx, 301).await,
10480            (before, Vec::new())
10481        );
10482    }
10483
10484    /// A handler whose `route.open` parks after queueing the route.bind and
10485    /// before marking it enqueued, with module "aft" registered on connection
10486    /// 40. Returns the two-party barrier that parks and releases it.
10487    async fn route_open_held_after_bind_send() -> (
10488        ControlHandler,
10489        Arc<tokio::sync::Barrier>,
10490        RouteCtx,
10491        mpsc::Receiver<crate::router::OutboundFrame>,
10492    ) {
10493        let hold = Arc::new(tokio::sync::Barrier::new(2));
10494        let handler = ControlHandler::with_forwarding(
10495            Arc::new(Registry::default()),
10496            Arc::new(ForwardingTable::default()),
10497        )
10498        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10499        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10500        hello_via_sink(
10501            &handler,
10502            &module_ctx,
10503            &mut module_rx,
10504            hello_frame("aft", PROTOCOL_VERSION, 7),
10505        )
10506        .await;
10507        (handler, hold, module_ctx, module_rx)
10508    }
10509
10510    /// A fast module acks the route.bind, and the reader commits the route,
10511    /// before `route.open` has marked the relay enqueued. The daemon used to
10512    /// read the missing relay as an abandoned bind and send the module a
10513    /// GOODBYE for the route it had just committed: the client was told its
10514    /// route was open, the module unbound the channel, and the client's first
10515    /// request was dropped on the module side as unbound.
10516    #[tokio::test]
10517    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10518        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10519        let client = ConnectionId::new(41);
10520        let (client_ctx, mut client_rx) = route_ctx(client);
10521
10522        let (open_task, bind) = relay_route_open(
10523            &handler,
10524            client,
10525            &client_ctx.egress,
10526            &mut module_rx,
10527            200,
10528            "aft",
10529            "acked-before-mark",
10530        )
10531        .await;
10532        let (module_channel, module_epoch) = route_bind_channel(&bind);
10533        // route.open is now parked between queueing the bind and marking it.
10534        hold.wait().await;
10535
10536        // The module answers inside that window, and the route commits.
10537        assert!(handler
10538            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10539            .await
10540            .unwrap()
10541            .is_empty());
10542        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10543
10544        hold.wait().await;
10545        assert!(
10546            open_task.await.unwrap().is_empty(),
10547            "route.open answers accepted"
10548        );
10549
10550        // Every frame route.open could have sent is already queued, because it
10551        // has returned; the module must have been sent nothing.
10552        match module_rx.try_recv() {
10553            Err(mpsc::error::TryRecvError::Empty) => {}
10554            Ok(outbound) => panic!(
10555                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10556                outbound.frame.header.ty,
10557                outbound.frame.header.channel,
10558                outbound.frame.header.epoch
10559            ),
10560            Err(err) => panic!("module egress closed: {err}"),
10561        }
10562
10563        // The client's first request on the route is the next thing the module
10564        // reads, on the binding it just created.
10565        let router = Router::with_control_handler(Arc::new(handler.clone()));
10566        let request = Frame::build(
10567            FrameType::Request,
10568            Flags::new(true, Priority::Interactive, false),
10569            client_channel,
10570            client_epoch,
10571            201,
10572            b"first request".to_vec(),
10573        )
10574        .unwrap();
10575        router
10576            .route_for_connection(&client_ctx, request)
10577            .await
10578            .unwrap();
10579        let delivered = module_rx.try_recv().unwrap().frame;
10580        assert_eq!(delivered.header.ty, FrameType::Request);
10581        assert_eq!(
10582            (delivered.header.channel, delivered.header.epoch),
10583            (module_channel, module_epoch)
10584        );
10585        assert_eq!(delivered.body, b"first request");
10586    }
10587
10588    /// The client goes away while `route.open` is still queueing the bind, so
10589    /// the relay is dropped with no module reply. The teardown cannot send the
10590    /// GOODBYE itself (it could reach the module ahead of the bind), so
10591    /// `route.open` must send it once the bind is queued; otherwise the module
10592    /// keeps a binding the daemon will never commit.
10593    #[tokio::test]
10594    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10595        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10596        let client = ConnectionId::new(42);
10597        let (client_ctx, _client_rx) = route_ctx(client);
10598
10599        let (open_task, bind) = relay_route_open(
10600            &handler,
10601            client,
10602            &client_ctx.egress,
10603            &mut module_rx,
10604            300,
10605            "aft",
10606            "released-before-mark",
10607        )
10608        .await;
10609        let (module_channel, module_epoch) = route_bind_channel(&bind);
10610        hold.wait().await;
10611
10612        handler.cleanup_connection(client).unwrap();
10613        assert!(
10614            module_rx.try_recv().is_err(),
10615            "client teardown must not GOODBYE a bind route.open is still queueing"
10616        );
10617
10618        hold.wait().await;
10619        let response = open_task.await.unwrap();
10620        assert_eq!(response.len(), 1);
10621        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10622
10623        let goodbye = module_rx
10624            .try_recv()
10625            .expect("module is told to drop the binding the daemon abandoned")
10626            .frame;
10627        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10628        assert_eq!(
10629            (goodbye.header.channel, goodbye.header.epoch),
10630            (module_channel, module_epoch)
10631        );
10632        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10633    }
10634
10635    /// The module acks inside the same window, but the client's egress closed
10636    /// first, so the reader refuses to commit. The ack proves the module holds
10637    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
10638    /// send a second one.
10639    #[tokio::test]
10640    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
10641        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10642        let client = ConnectionId::new(43);
10643        let (client_ctx, client_rx) = route_ctx(client);
10644
10645        let (open_task, bind) = relay_route_open(
10646            &handler,
10647            client,
10648            &client_ctx.egress,
10649            &mut module_rx,
10650            400,
10651            "aft",
10652            "gone-client-before-mark",
10653        )
10654        .await;
10655        let (module_channel, module_epoch) = route_bind_channel(&bind);
10656        hold.wait().await;
10657
10658        drop(client_rx);
10659        assert!(client_ctx.egress.is_closed());
10660        handler
10661            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10662            .await
10663            .unwrap();
10664
10665        hold.wait().await;
10666        let response = open_task.await.unwrap();
10667        assert_eq!(response.len(), 1);
10668        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10669
10670        let goodbye = module_rx
10671            .try_recv()
10672            .expect("module is told to drop the binding it acked")
10673            .frame;
10674        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10675        assert_eq!(
10676            (goodbye.header.channel, goodbye.header.epoch),
10677            (module_channel, module_epoch)
10678        );
10679        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10680        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
10681    }
10682
10683    /// The fence at the module-loop boundary, stated as its own contract: which
10684    /// forwarding failures are allowed to end the module connection that is being
10685    /// served. A `ConnectionClosing` naming some client is about that client, and
10686    /// a module connection is shared; the same error naming the module's own
10687    /// connection is about this connection and must stay fatal, as must failures
10688    /// that are about the forwarding table itself.
10689    #[test]
10690    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
10691        let handler = ControlHandler::default();
10692        let module_connection = ConnectionId::new(30);
10693        let client_connection = ConnectionId::new(31);
10694
10695        handler
10696            .refuse_to_end_module_connection_for_a_client(
10697                module_connection,
10698                77,
10699                ForwardingError::ConnectionClosing {
10700                    connection_id: client_connection,
10701                },
10702            )
10703            .expect("a closing client must never end the module connection");
10704
10705        assert!(matches!(
10706            handler.refuse_to_end_module_connection_for_a_client(
10707                module_connection,
10708                78,
10709                ForwardingError::ConnectionClosing {
10710                    connection_id: module_connection,
10711                },
10712            ),
10713            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
10714                connection_id
10715            })) if connection_id == module_connection
10716        ));
10717        assert!(matches!(
10718            handler.refuse_to_end_module_connection_for_a_client(
10719                module_connection,
10720                79,
10721                ForwardingError::Poisoned,
10722            ),
10723            Err(RouterError::Forwarding(ForwardingError::Poisoned))
10724        ));
10725        assert!(matches!(
10726            handler.refuse_to_end_module_connection_for_a_client(
10727                module_connection,
10728                80,
10729                ForwardingError::StaleModuleEndpoint,
10730            ),
10731            Err(RouterError::Forwarding(
10732                ForwardingError::StaleModuleEndpoint
10733            ))
10734        ));
10735    }
10736
10737    /// The spawn-attestation guard is what stops a connected module from claiming
10738    /// another module's identity and being stamped `Reserved` for it. Every other
10739    /// test that supplies a consumer_identity supplies a CORRECT one, because a
10740    /// correct one is what the rest of the flow needs -- so the guard's rejection
10741    /// branch was never the subject of an assertion, only its acceptance branch.
10742    ///
10743    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
10744    /// whole subc-core library suite green; only the forwarding integration tests
10745    /// notice, and they notice for unrelated reasons. This test exists so the
10746    /// refusal itself is asserted where the guard lives: it fails if the identity
10747    /// check stops refusing, which is the direction that matters, since a guard
10748    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
10749    #[tokio::test]
10750    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
10751        let registry = Arc::new(Registry::default());
10752        let forwarding = Arc::new(ForwardingTable::default());
10753        let supervisor = SupervisorHandle::new();
10754        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10755        let handler =
10756            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10757                .with_supervisor(supervisor);
10758
10759        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10760        hello_via_sink(
10761            &handler,
10762            &target_ctx,
10763            &mut target_rx,
10764            hello_frame("target", PROTOCOL_VERSION, 1),
10765        )
10766        .await;
10767
10768        // A real supervised module id presenting the wrong nonce. This is the
10769        // impersonation case: the attacker knows a privileged module_id, which is
10770        // public, and guesses at the nonce, which is not.
10771        let wrong_nonce = handler
10772            .handle_control_frame(
10773                &route_ctx(ConnectionId::new(91)).0,
10774                route_open_frame_with_admission_facts(
10775                    20,
10776                    "target",
10777                    unique_project_root("admission-facts"),
10778                    Some(subc_control::ConsumerIdentity {
10779                        module_id: "fed".to_string(),
10780                        launch_nonce: "not-the-real-nonce".to_string(),
10781                    }),
10782                    None,
10783                ),
10784            )
10785            .await
10786            .unwrap();
10787        assert_eq!(
10788            parse_error(&wrong_nonce[0])["code"],
10789            "bad_consumer_identity",
10790            "a mismatched launch nonce must be refused, not stamped Reserved"
10791        );
10792
10793        // A module id the supervisor never spawned at all, so no nonce exists to
10794        // compare against. An implementation that treats "no record" as "nothing
10795        // to check" fails open here while passing the case above.
10796        let never_spawned = handler
10797            .handle_control_frame(
10798                &route_ctx(ConnectionId::new(92)).0,
10799                route_open_frame_with_admission_facts(
10800                    21,
10801                    "target",
10802                    unique_project_root("admission-facts"),
10803                    Some(subc_control::ConsumerIdentity {
10804                        module_id: "never-spawned".to_string(),
10805                        launch_nonce: "any-nonce".to_string(),
10806                    }),
10807                    None,
10808                ),
10809            )
10810            .await
10811            .unwrap();
10812        assert_eq!(
10813            parse_error(&never_spawned[0])["code"],
10814            "bad_consumer_identity",
10815            "an unspawned module_id must be refused rather than accepted for lack of a record"
10816        );
10817    }
10818
10819    /// The refusal test above proves the guard says NO. Nothing proved it can say
10820    /// YES, and the difference is not academic: replacing the whole authorization
10821    /// with `false` -- admitting no consumer identity at all, revoking Reserved
10822    /// standing for every supervised module in the fleet -- leaves 110 of the 111
10823    /// library tests GREEN. The one that notices does so by HANGING, because it
10824    /// waits for a bind that can no longer happen.
10825    ///
10826    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
10827    /// or flaky test, invites a RETRY rather than an investigation, and the retry
10828    /// hangs too and gets blamed on the runner. So a total revocation of the
10829    /// daemon's trust grant would have shipped behind a symptom nobody attributes
10830    /// to code.
10831    ///
10832    /// The bias is structural rather than accidental. A REFUSAL looks like a
10833    /// failure someone writes a test for; a GRANT looks like the happy path. Every
10834    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
10835    /// suite for that reason, and this one is the purest case in the daemon.
10836    ///
10837    /// This test asserts the EFFECT rather than the absence of an error: the module
10838    /// receives a RouteBind and it carries `Reserved` naming the attested module.
10839    /// A guard that admitted nobody would produce no bind at all; one that admitted
10840    /// everybody would stamp the wrong principal, which the refusal test catches.
10841    #[tokio::test]
10842    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10843        let registry = Arc::new(Registry::default());
10844        let forwarding = Arc::new(ForwardingTable::default());
10845        let supervisor = SupervisorHandle::new();
10846        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10847        let handler =
10848            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10849                .with_supervisor(supervisor);
10850
10851        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10852        hello_via_sink(
10853            &handler,
10854            &target_ctx,
10855            &mut target_rx,
10856            hello_frame("target", PROTOCOL_VERSION, 1),
10857        )
10858        .await;
10859
10860        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10861        let route_handler = handler.clone();
10862        let route_task = tokio::spawn(async move {
10863            route_handler
10864                .handle_control_frame(
10865                    &client_ctx,
10866                    route_open_frame_with_admission_facts(
10867                        30,
10868                        "target",
10869                        unique_project_root("admission-facts"),
10870                        Some(subc_control::ConsumerIdentity {
10871                            module_id: "fed".to_string(),
10872                            launch_nonce: "fed-nonce".to_string(),
10873                        }),
10874                        None,
10875                    ),
10876                )
10877                .await
10878                .unwrap()
10879        });
10880
10881        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
10882        // the very mutation it exists to catch -- a guard that admits nobody -- no
10883        // bind is ever sent, so it HUNG rather than failing. That reproduces the
10884        // exact defect being fixed: a total revocation detected only as a stalled
10885        // suite, which reads as flakiness and invites a retry. An acceptance test
10886        // that waits for an effect must bound the wait, or a red becomes a hang.
10887        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10888            .await
10889            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10890            .expect("module control channel closed before route.bind");
10891        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10892        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10893            panic!("expected route.bind")
10894        };
10895        assert_eq!(
10896            principal,
10897            Some(Principal::Reserved {
10898                module_id: "fed".to_string()
10899            }),
10900            "a correctly attested consumer must be stamped Reserved for its own id"
10901        );
10902
10903        handler
10904            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10905            .await
10906            .unwrap();
10907        assert!(route_task.await.unwrap().is_empty());
10908        assert!(
10909            matches!(
10910                serde_json::from_slice::<ClientControlResponse>(
10911                    &client_rx.recv().await.unwrap().body
10912                )
10913                .unwrap(),
10914                ClientControlResponse::RouteOpen { .. }
10915            ),
10916            "the route must actually open, not merely avoid an error"
10917        );
10918    }
10919
10920    #[tokio::test(start_paused = true)]
10921    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10922        let registry = Arc::new(Registry::default());
10923        let forwarding = Arc::new(ForwardingTable::default());
10924        let supervisor = SupervisorHandle::new();
10925        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10926        let handler =
10927            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10928                .with_supervisor(supervisor);
10929
10930        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10931        hello_via_sink(
10932            &handler,
10933            &target_ctx,
10934            &mut target_rx,
10935            hello_frame("target", PROTOCOL_VERSION, 1),
10936        )
10937        .await;
10938
10939        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10940        let direct_handler = handler.clone();
10941        let direct_open = tokio::spawn(async move {
10942            direct_handler
10943                .handle_control_frame(
10944                    &direct_ctx,
10945                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
10946                )
10947                .await
10948                .unwrap()
10949        });
10950        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10951            .await
10952            .expect("no direct route.bind within 5s")
10953            .expect("target control channel closed before direct route.bind");
10954        handler
10955            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10956            .await
10957            .unwrap();
10958        assert!(direct_open.await.unwrap().is_empty());
10959        let _ = direct_rx.recv().await.unwrap();
10960
10961        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10962        let reserved_handler = handler.clone();
10963        let reserved_open = tokio::spawn(async move {
10964            reserved_handler
10965                .handle_control_frame(
10966                    &reserved_ctx,
10967                    route_open_frame_with_admission_facts(
10968                        3,
10969                        "target",
10970                        unique_project_root("admission-facts"),
10971                        Some(ConsumerIdentity {
10972                            module_id: "fed".to_string(),
10973                            launch_nonce: "fed-nonce".to_string(),
10974                        }),
10975                        None,
10976                    ),
10977                )
10978                .await
10979                .unwrap()
10980        });
10981        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10982            .await
10983            .expect("no reserved route.bind within 5s")
10984            .expect("target control channel closed before reserved route.bind");
10985        handler
10986            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10987            .await
10988            .unwrap();
10989        assert!(reserved_open.await.unwrap().is_empty());
10990        let _ = reserved_rx.recv().await.unwrap();
10991
10992        forwarding
10993            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10994            .unwrap();
10995        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10996        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10997            module_id: Some("target".to_string()),
10998        })
10999        .unwrap();
11000        let census_frame =
11001            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11002        let response = handler
11003            .handle_control_frame(&census_ctx, census_frame)
11004            .await
11005            .unwrap()
11006            .pop()
11007            .unwrap();
11008        let actual: Value = serde_json::from_slice(&response.body).unwrap();
11009        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11010        assert!(matches!(
11011            decoded,
11012            ClientControlResponse::SupervisorRoutes { .. }
11013        ));
11014        let routes = actual["modules"][0]["routes"].as_array().unwrap();
11015        assert_eq!(routes.len(), 2);
11016        assert!(routes.iter().all(|route| route["draining"] == true));
11017        // The census carries WHY: the reason the drain was begun with, in the
11018        // route.closing vocabulary, on every draining route this drain marked.
11019        assert!(
11020            routes.iter().all(|route| route["drain_reason"] == "reload"),
11021            "draining routes must name the drain's reason: {routes:?}"
11022        );
11023        assert!(routes.iter().any(|route| {
11024            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11025        }));
11026        assert!(routes.iter().any(|route| {
11027            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11028        }));
11029
11030        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11031            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11032        if std::env::var_os("UPDATE_GOLDEN").is_some() {
11033            std::fs::write(
11034                &golden_path,
11035                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11036            )
11037            .unwrap();
11038        }
11039        let expected: Value =
11040            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11041        assert_eq!(actual, expected);
11042    }
11043
11044    async fn query_live_roots(
11045        handler: &ControlHandler,
11046        module_ctx: &RouteCtx,
11047    ) -> ModuleControlResponseToModule {
11048        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11049        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11050        let response = handler
11051            .handle_control_frame(module_ctx, frame)
11052            .await
11053            .unwrap()
11054            .pop()
11055            .unwrap();
11056        serde_json::from_slice(&response.body).unwrap()
11057    }
11058
11059    #[tokio::test(start_paused = true)]
11060    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11061        let registry = Arc::new(Registry::default());
11062        let forwarding = Arc::new(ForwardingTable::default());
11063        let handler = ControlHandler::with_forwarding(registry, forwarding);
11064        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11065        hello_via_sink(
11066            &handler,
11067            &target_ctx,
11068            &mut target_rx,
11069            hello_frame("target", PROTOCOL_VERSION, 1),
11070        )
11071        .await;
11072        let root = unique_project_root("live-roots-known");
11073        let path = ProjectRootId::from_path_allowing_missing(root.path())
11074            .unwrap()
11075            .as_path()
11076            .to_path_buf();
11077        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11078        let open_handler = handler.clone();
11079        let opened = tokio::spawn(async move {
11080            open_handler
11081                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11082                .await
11083                .unwrap()
11084        });
11085        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11086            .await
11087            .unwrap()
11088            .unwrap();
11089        handler
11090            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11091            .await
11092            .unwrap();
11093        assert!(opened.await.unwrap().is_empty());
11094        let _ = client_rx.recv().await.unwrap();
11095
11096        let root = unique_project_root("live-roots-pending");
11097        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11098            .unwrap()
11099            .as_path()
11100            .to_path_buf();
11101        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11102        let open_handler = handler.clone();
11103        let pending = tokio::spawn(async move {
11104            open_handler
11105                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11106                .await
11107                .unwrap()
11108        });
11109        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11110            .await
11111            .unwrap()
11112            .unwrap();
11113        let actual = query_live_roots(&handler, &target_ctx).await;
11114        let ModuleControlResponseToModule::LiveRoots {
11115            roots,
11116            unknown_root_bindings,
11117            total_bindings,
11118        } = actual
11119        else {
11120            panic!("expected live roots")
11121        };
11122        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11123        assert_eq!(unknown_root_bindings, 0);
11124        assert_eq!(
11125            roots.len(),
11126            2,
11127            "root-known arm must retain each canonical root"
11128        );
11129        assert_eq!(
11130            total_bindings,
11131            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11132        );
11133        let counts = roots
11134            .iter()
11135            .map(|root| (root.project_root.clone(), root.bound, root.pending))
11136            .collect::<Vec<_>>();
11137        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11138        expected.sort_by(|a, b| a.0.cmp(&b.0));
11139        assert_eq!(
11140            counts, expected,
11141            "roots must sort by path and count pending separately"
11142        );
11143        handler
11144            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11145            .await
11146            .unwrap();
11147        assert!(pending.await.unwrap().is_empty());
11148    }
11149
11150    #[tokio::test(start_paused = true)]
11151    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11152        let forwarding = Arc::new(ForwardingTable::default());
11153        let handler =
11154            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11155        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11156        hello_via_sink(
11157            &handler,
11158            &target_ctx,
11159            &mut target_rx,
11160            hello_frame("target", PROTOCOL_VERSION, 1),
11161        )
11162        .await;
11163        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11164        let pending = forwarding
11165            .begin_route_bind_relay_for_test(
11166                client_ctx.connection_id,
11167                client_ctx.egress.clone(),
11168                2,
11169                "target",
11170            )
11171            .unwrap();
11172        forwarding
11173            .complete_pending_relay(
11174                target_ctx.connection_id,
11175                pending.corr,
11176                RouteBindRelayOutcome::Accepted,
11177            )
11178            .unwrap();
11179        let actual = query_live_roots(&handler, &target_ctx).await;
11180        let ModuleControlResponseToModule::LiveRoots {
11181            roots,
11182            unknown_root_bindings,
11183            total_bindings,
11184        } = actual
11185        else {
11186            panic!("expected live roots")
11187        };
11188        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11189        assert_eq!(
11190            unknown_root_bindings, 1,
11191            "unknown-root arm must not read as no bindings"
11192        );
11193        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11194        assert_eq!(
11195            total_bindings,
11196            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11197        );
11198    }
11199
11200    /// A module reads HELLO_ACK as its first frame and exits on anything else,
11201    /// so the ack has to be on its outbound queue before the module is
11202    /// routable. The connection loop writes a handler's replies only after the
11203    /// handler returns; this test stops in exactly that gap, runs a real
11204    /// route.open from another connection, and only then writes whatever the
11205    /// HELLO handler returned, the way the loop would. If the ack were still a
11206    /// reply, the route.bind request would reach the module first.
11207    #[tokio::test(start_paused = true)]
11208    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11209        let forwarding = Arc::new(ForwardingTable::default());
11210        let handler =
11211            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11212        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11213        let replies = handler
11214            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11215            .await
11216            .unwrap();
11217        let queued_by_hello = module_rx.len();
11218
11219        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11220        let open_handler = handler.clone();
11221        let open = tokio::spawn(async move {
11222            open_handler
11223                .handle_control_frame(
11224                    &client_ctx,
11225                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11226                )
11227                .await
11228                .unwrap()
11229        });
11230        // Let the route.open run until its route.bind is on the module's queue.
11231        let mut spins = 0;
11232        while module_rx.len() == queued_by_hello {
11233            spins += 1;
11234            assert!(spins < 10_000, "route.open never queued a route.bind");
11235            tokio::task::yield_now().await;
11236        }
11237
11238        // Now the connection loop's half: write the HELLO handler's replies.
11239        for reply in replies {
11240            module_ctx.egress.send(reply).await.unwrap();
11241        }
11242
11243        let first = module_rx.recv().await.unwrap().frame;
11244        assert_eq!(
11245            first.header.ty,
11246            FrameType::HelloAck,
11247            "the first frame a registering module reads must be its HELLO_ACK"
11248        );
11249        assert_eq!(first.header.corr, 7);
11250        let second = module_rx.recv().await.unwrap().frame;
11251        assert_eq!(second.header.ty, FrameType::Request);
11252        assert!(
11253            matches!(
11254                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11255                ModuleControlRequest::RouteBind { .. }
11256            ),
11257            "the route.bind follows the ack"
11258        );
11259        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11260
11261        handler
11262            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11263            .await
11264            .unwrap();
11265        assert!(open.await.unwrap().is_empty());
11266        let _ = client_rx.recv().await.unwrap();
11267    }
11268
11269    #[tokio::test(start_paused = true)]
11270    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11271        let handler = ControlHandler::with_forwarding(
11272            Arc::new(Registry::default()),
11273            Arc::new(ForwardingTable::default()),
11274        );
11275        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11276        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11277        hello_via_sink(
11278            &handler,
11279            &first_ctx,
11280            &mut first_rx,
11281            hello_frame("first", PROTOCOL_VERSION, 1),
11282        )
11283        .await;
11284        hello_via_sink(
11285            &handler,
11286            &second_ctx,
11287            &mut second_rx,
11288            hello_frame("second", PROTOCOL_VERSION, 2),
11289        )
11290        .await;
11291        let root = unique_project_root("second-only");
11292        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11293        let cloned = handler.clone();
11294        let open = tokio::spawn(async move {
11295            cloned
11296                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11297                .await
11298                .unwrap()
11299        });
11300        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11301            .await
11302            .unwrap()
11303            .unwrap();
11304        let first = query_live_roots(&handler, &first_ctx).await;
11305        let second = query_live_roots(&handler, &second_ctx).await;
11306        assert!(
11307            matches!(
11308                first,
11309                ModuleControlResponseToModule::LiveRoots {
11310                    total_bindings: 0,
11311                    ..
11312                }
11313            ),
11314            "cross-module scope must not expose another module's roots"
11315        );
11316        assert!(
11317            matches!(
11318                second,
11319                ModuleControlResponseToModule::LiveRoots {
11320                    total_bindings: 1,
11321                    ..
11322                }
11323            ),
11324            "second module must see its pending route"
11325        );
11326        handler
11327            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11328            .await
11329            .unwrap();
11330        assert!(open.await.unwrap().is_empty());
11331    }
11332
11333    #[tokio::test(start_paused = true)]
11334    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11335        let handler = ControlHandler::with_forwarding(
11336            Arc::new(Registry::default()),
11337            Arc::new(ForwardingTable::default()),
11338        );
11339        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11340        hello_via_sink(
11341            &handler,
11342            &target_ctx,
11343            &mut target_rx,
11344            hello_frame("target", PROTOCOL_VERSION, 1),
11345        )
11346        .await;
11347        let actual = query_live_roots(&handler, &target_ctx).await;
11348        let ModuleControlResponseToModule::LiveRoots {
11349            roots,
11350            unknown_root_bindings,
11351            total_bindings,
11352        } = actual
11353        else {
11354            panic!("expected live roots")
11355        };
11356        assert!(roots.is_empty());
11357        assert_eq!(unknown_root_bindings, 0);
11358        assert_eq!(total_bindings, 0);
11359        assert_eq!(
11360            total_bindings,
11361            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11362        );
11363    }
11364
11365    /// Read the vendored fed corpus rather than hand-building a package.
11366    ///
11367    /// A hand-built object encodes what the test author believed the carrier
11368    /// emits. These vectors are what it actually emits, and one of them exists
11369    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
11370    /// ignores additive unknown fields at the traversal emit terminus."
11371    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11372        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11373            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11374        let text = std::fs::read_to_string(&path)
11375            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11376        let vectors: Vec<(String, Value)> = text
11377            .lines()
11378            .filter(|line| !line.trim().is_empty())
11379            .map(|line| {
11380                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11381                let id = entry["corpus_id"]
11382                    .as_str()
11383                    .expect("every vector carries a corpus_id")
11384                    .to_string();
11385                (id, entry["package"].clone())
11386            })
11387            .collect();
11388        // Pin the count: a corpus that silently shrinks would take its coverage
11389        // with it, and a suite reading N-1 vectors reports the same clean pass
11390        // as one reading N.
11391        assert_eq!(
11392            vectors.len(),
11393            3,
11394            "vendored fed corpus changed size; re-sync from subc-federation"
11395        );
11396
11397        // Pin what makes the corpus DISCRIMINATING, not just present.
11398        //
11399        // The relay test below takes its expected value from the corpus, so the
11400        // corpus supplies the test's power to detect a lossy relay rather than
11401        // its correctness. A relay that dropped unrecognised fields would still
11402        // be caught -- but only by a package carrying fields it does not know.
11403        // Shrink every package to the handful of keys any implementation would
11404        // recognise and the test keeps passing over an input that can no longer
11405        // fail, which is the same clean green as a corpus that shrank away.
11406        //
11407        // So assert the precondition rather than duplicating the packages here:
11408        // at least one vector must carry a field beyond the small common set.
11409        // That is one claim to maintain instead of nine, and it fails loudly if
11410        // a re-sync ever flattens the corpus.
11411        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11412        let richest = vectors
11413            .iter()
11414            .filter_map(|(_, package)| package.as_object())
11415            .map(|object| {
11416                object
11417                    .keys()
11418                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11419                    .count()
11420            })
11421            .max()
11422            .unwrap_or(0);
11423        assert!(
11424            richest >= 2,
11425            "vendored corpus no longer carries a package with unmodelled fields, \
11426             so the relay test can no longer distinguish a verbatim relay from a lossy one"
11427        );
11428
11429        vectors
11430    }
11431
11432    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
11433    ///
11434    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
11435    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
11436    /// three-key object, so a relay that quietly dropped fields it did not
11437    /// recognise would satisfy it. These vectors carry nine keys including ones
11438    /// this crate has no type for, so a typed relay fails here and only here.
11439    #[tokio::test]
11440    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11441        for (corpus_id, package) in fed_admission_facts_vectors() {
11442            let registry = Arc::new(Registry::default());
11443            let forwarding = Arc::new(ForwardingTable::default());
11444            let supervisor = SupervisorHandle::new();
11445            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11446            let handler =
11447                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11448                    .with_supervisor(supervisor)
11449                    .with_admission_facts_config(
11450                        Some("fed".to_string()),
11451                        Some(vec!["target".to_string()]),
11452                    );
11453
11454            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11455            hello_via_sink(
11456                &handler,
11457                &target_ctx,
11458                &mut target_rx,
11459                hello_frame("target", PROTOCOL_VERSION, 1),
11460            )
11461            .await;
11462
11463            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11464            let route_handler = handler.clone();
11465            let expected = package.clone();
11466            let route_task = tokio::spawn(async move {
11467                route_handler
11468                    .handle_control_frame(
11469                        &client_ctx,
11470                        route_open_frame_with_admission_facts(
11471                            20,
11472                            "target",
11473                            unique_project_root("admission-facts"),
11474                            Some(subc_control::ConsumerIdentity {
11475                                module_id: "fed".to_string(),
11476                                launch_nonce: "fed-nonce".to_string(),
11477                            }),
11478                            Some(package),
11479                        ),
11480                    )
11481                    .await
11482                    .unwrap()
11483            });
11484
11485            let bind_frame = target_rx.recv().await.unwrap();
11486            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11487            let ModuleControlRequest::RouteBind {
11488                admission_facts, ..
11489            } = bind
11490            else {
11491                panic!("{corpus_id}: expected route.bind")
11492            };
11493            assert_eq!(
11494                admission_facts,
11495                Some(expected),
11496                "{corpus_id}: relay must not add, drop or reshape any field"
11497            );
11498
11499            handler
11500                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11501                .await
11502                .unwrap();
11503            route_task.await.unwrap();
11504        }
11505    }
11506
11507    #[tokio::test]
11508    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11509        let registry = Arc::new(Registry::default());
11510        let forwarding = Arc::new(ForwardingTable::default());
11511        let supervisor = SupervisorHandle::new();
11512        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11513        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11514        let handler =
11515            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11516                .with_supervisor(supervisor)
11517                .with_admission_facts_config(
11518                    Some("fed".to_string()),
11519                    Some(vec!["target".to_string()]),
11520                );
11521
11522        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11523        hello_via_sink(
11524            &handler,
11525            &target_ctx,
11526            &mut target_rx,
11527            hello_frame("target", PROTOCOL_VERSION, 1),
11528        )
11529        .await;
11530        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11531        hello_via_sink(
11532            &handler,
11533            &other_ctx,
11534            &mut other_rx,
11535            hello_frame("other", PROTOCOL_VERSION, 2),
11536        )
11537        .await;
11538
11539        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11540        let expected_facts = facts.clone();
11541        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11542        let route_handler = handler.clone();
11543        let route_task = tokio::spawn(async move {
11544            route_handler
11545                .handle_control_frame(
11546                    &client_ctx,
11547                    route_open_frame_with_admission_facts(
11548                        10,
11549                        "target",
11550                        unique_project_root("admission-facts"),
11551                        Some(subc_control::ConsumerIdentity {
11552                            module_id: "fed".to_string(),
11553                            launch_nonce: "fed-nonce".to_string(),
11554                        }),
11555                        Some(facts.clone()),
11556                    ),
11557                )
11558                .await
11559                .unwrap()
11560        });
11561        let bind_frame = target_rx.recv().await.unwrap();
11562        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11563        let ModuleControlRequest::RouteBind {
11564            admission_facts, ..
11565        } = bind
11566        else {
11567            panic!("expected route.bind")
11568        };
11569        assert_eq!(admission_facts, Some(expected_facts));
11570        handler
11571            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11572            .await
11573            .unwrap();
11574        assert!(route_task.await.unwrap().is_empty());
11575        assert!(matches!(
11576            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11577                .unwrap(),
11578            ClientControlResponse::RouteOpen { .. }
11579        ));
11580
11581        let direct = handler
11582            .handle_control_frame(
11583                &route_ctx(ConnectionId::new(73)).0,
11584                route_open_frame_with_admission_facts(
11585                    11,
11586                    "target",
11587                    unique_project_root("admission-facts"),
11588                    None,
11589                    Some(json!({"x": 1})),
11590                ),
11591            )
11592            .await
11593            .unwrap();
11594        assert_eq!(
11595            parse_error(&direct[0])["code"],
11596            "admission_facts_not_permitted"
11597        );
11598
11599        let different_reserved = handler
11600            .handle_control_frame(
11601                &route_ctx(ConnectionId::new(77)).0,
11602                route_open_frame_with_admission_facts(
11603                    15,
11604                    "target",
11605                    unique_project_root("admission-facts"),
11606                    Some(subc_control::ConsumerIdentity {
11607                        module_id: "other".to_string(),
11608                        launch_nonce: "other-nonce".to_string(),
11609                    }),
11610                    Some(json!({"x": 1})),
11611                ),
11612            )
11613            .await
11614            .unwrap();
11615        assert_eq!(
11616            parse_error(&different_reserved[0])["code"],
11617            "admission_facts_not_permitted"
11618        );
11619
11620        let other_target = handler
11621            .handle_control_frame(
11622                &route_ctx(ConnectionId::new(74)).0,
11623                route_open_frame_with_admission_facts(
11624                    12,
11625                    "other",
11626                    unique_project_root("admission-facts"),
11627                    Some(subc_control::ConsumerIdentity {
11628                        module_id: "fed".to_string(),
11629                        launch_nonce: "fed-nonce".to_string(),
11630                    }),
11631                    Some(json!({"x": 1})),
11632                ),
11633            )
11634            .await
11635            .unwrap();
11636        assert_eq!(
11637            parse_error(&other_target[0])["code"],
11638            "admission_facts_target_not_allowed"
11639        );
11640
11641        let nonexistent = handler
11642            .handle_control_frame(
11643                &route_ctx(ConnectionId::new(75)).0,
11644                route_open_frame_with_admission_facts(
11645                    13,
11646                    "missing",
11647                    unique_project_root("admission-facts"),
11648                    None,
11649                    Some(json!({"x": 1})),
11650                ),
11651            )
11652            .await
11653            .unwrap();
11654        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
11655
11656        let described = handler
11657            .handle_control_frame(
11658                &route_ctx(ConnectionId::new(76)).0,
11659                Frame::build(
11660                    FrameType::Request,
11661                    control_flags(),
11662                    0,
11663                    0,
11664                    14,
11665                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11666                )
11667                .unwrap(),
11668            )
11669            .await
11670            .unwrap();
11671        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11672            serde_json::from_slice(&described[0].body).unwrap()
11673        else {
11674            panic!("expected server.describe response")
11675        };
11676        assert!(capabilities
11677            .iter()
11678            .any(|cap| cap == "admission_facts_relay_v1"));
11679    }
11680
11681    #[tokio::test]
11682    async fn admission_facts_without_configured_carrier_are_rejected() {
11683        let registry = Arc::new(Registry::default());
11684        let forwarding = Arc::new(ForwardingTable::default());
11685        let handler = ControlHandler::with_forwarding(registry, forwarding);
11686        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
11687        hello_via_sink(
11688            &handler,
11689            &target_ctx,
11690            &mut target_rx,
11691            hello_frame("target", PROTOCOL_VERSION, 1),
11692        )
11693        .await;
11694
11695        let responses = handler
11696            .handle_control_frame(
11697                &route_ctx(ConnectionId::new(79)).0,
11698                route_open_frame_with_admission_facts(
11699                    16,
11700                    "target",
11701                    unique_project_root("admission-facts"),
11702                    None,
11703                    Some(json!({"x": 1})),
11704                ),
11705            )
11706            .await
11707            .unwrap();
11708        assert_eq!(
11709            parse_error(&responses[0])["code"],
11710            "admission_facts_not_permitted"
11711        );
11712    }
11713
11714    #[tokio::test]
11715    async fn route_open_relays_consumer_capabilities_verbatim() {
11716        let registry = Arc::new(Registry::default());
11717        let forwarding = Arc::new(ForwardingTable::default());
11718        let handler =
11719            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11720        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
11721        hello_via_sink(
11722            &handler,
11723            &module_ctx,
11724            &mut module_rx,
11725            hello_frame("aft", PROTOCOL_VERSION, 7),
11726        )
11727        .await;
11728
11729        let expected = vec!["elicitation".to_string(), "roots".to_string()];
11730        let expected_for_request = expected.clone();
11731        let project_root = unique_project_root("consumer-capabilities-present");
11732        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
11733        let route_handler = handler.clone();
11734        let route_task = tokio::spawn(async move {
11735            route_handler
11736                .handle_control_frame(
11737                    &client_ctx,
11738                    route_open_frame_with_consumer_capabilities(
11739                        401,
11740                        "aft",
11741                        project_root,
11742                        Some(expected_for_request),
11743                    ),
11744                )
11745                .await
11746                .unwrap()
11747        });
11748        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11749            .await
11750            .unwrap()
11751            .unwrap();
11752        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11753        let ModuleControlRequest::RouteBind {
11754            consumer_capabilities,
11755            ..
11756        } = bind
11757        else {
11758            panic!("expected route.bind request, got {bind:?}");
11759        };
11760        assert_eq!(consumer_capabilities, Some(expected.clone()));
11761
11762        handler
11763            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11764            .await
11765            .unwrap();
11766        let route_response = route_task.await.unwrap();
11767        assert!(route_response.is_empty());
11768        let published = client_rx.recv().await.unwrap();
11769        assert!(matches!(
11770            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11771            ClientControlResponse::RouteOpen { .. }
11772        ));
11773    }
11774
11775    #[tokio::test]
11776    async fn route_open_without_consumer_capabilities_relays_none() {
11777        let registry = Arc::new(Registry::default());
11778        let forwarding = Arc::new(ForwardingTable::default());
11779        let handler =
11780            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11781        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11782        hello_via_sink(
11783            &handler,
11784            &module_ctx,
11785            &mut module_rx,
11786            hello_frame("aft", PROTOCOL_VERSION, 7),
11787        )
11788        .await;
11789
11790        let project_root = unique_project_root("consumer-capabilities-absent");
11791        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11792        let route_handler = handler.clone();
11793        let route_task = tokio::spawn(async move {
11794            route_handler
11795                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11796                .await
11797                .unwrap()
11798        });
11799        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11800            .await
11801            .unwrap()
11802            .unwrap();
11803        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11804        let ModuleControlRequest::RouteBind {
11805            consumer_capabilities,
11806            ..
11807        } = bind
11808        else {
11809            panic!("expected route.bind request, got {bind:?}");
11810        };
11811        assert_eq!(consumer_capabilities, None);
11812
11813        handler
11814            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11815            .await
11816            .unwrap();
11817        let route_response = route_task.await.unwrap();
11818        assert!(route_response.is_empty());
11819        let published = client_rx.recv().await.unwrap();
11820        assert!(matches!(
11821            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11822            ClientControlResponse::RouteOpen { .. }
11823        ));
11824    }
11825
11826    /// Opens a route to a freshly registered `aft` with `sent` as the
11827    /// route.open's role_versions, acks the bind, and returns the role_versions
11828    /// the module's bind carried.
11829    async fn bind_role_versions_for(
11830        sent: Option<BTreeMap<String, String>>,
11831        connection: u64,
11832    ) -> Option<BTreeMap<String, String>> {
11833        let registry = Arc::new(Registry::default());
11834        let forwarding = Arc::new(ForwardingTable::default());
11835        let handler =
11836            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11837        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11838        hello_via_sink(
11839            &handler,
11840            &module_ctx,
11841            &mut module_rx,
11842            hello_frame("aft", PROTOCOL_VERSION, 7),
11843        )
11844        .await;
11845        let project_root = unique_project_root("role-versions");
11846        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11847        let route_handler = handler.clone();
11848        let route_task = tokio::spawn(async move {
11849            route_handler
11850                .handle_control_frame(
11851                    &client_ctx,
11852                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
11853                )
11854                .await
11855                .unwrap()
11856        });
11857        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11858            .await
11859            .expect("a well-formed route.open reaches the module as a bind")
11860            .unwrap();
11861        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11862        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11863            panic!("expected route.bind request, got {bind:?}");
11864        };
11865        handler
11866            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11867            .await
11868            .unwrap();
11869        assert!(route_task.await.unwrap().is_empty());
11870        let published = client_rx.recv().await.unwrap();
11871        assert!(matches!(
11872            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11873            ClientControlResponse::RouteOpen { .. }
11874        ));
11875        role_versions
11876    }
11877
11878    #[tokio::test]
11879    async fn route_open_relays_role_versions_verbatim() {
11880        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11881        assert_eq!(
11882            bind_role_versions_for(Some(sent.clone()), 141).await,
11883            Some(sent)
11884        );
11885    }
11886
11887    /// An empty map declares nothing, so the provider sees no field rather
11888    /// than an empty object it would have to treat as a second "none".
11889    #[tokio::test]
11890    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11891        assert_eq!(bind_role_versions_for(None, 143).await, None);
11892        assert_eq!(
11893            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11894            None
11895        );
11896    }
11897
11898    #[tokio::test]
11899    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11900        let registry = Arc::new(Registry::default());
11901        let forwarding = Arc::new(ForwardingTable::default());
11902        let handler =
11903            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11904        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11905        hello_via_sink(
11906            &handler,
11907            &module_ctx,
11908            &mut module_rx,
11909            hello_frame("aft", PROTOCOL_VERSION, 7),
11910        )
11911        .await;
11912
11913        let nine: BTreeMap<String, String> = (0..9)
11914            .map(|index| (format!("role-{index}"), "v1".to_string()))
11915            .collect();
11916        for (label, malformed) in [
11917            (
11918                "invalid role name",
11919                role_versions(&[("Tool_Provider", "v1")]),
11920            ),
11921            ("invalid version", role_versions(&[("tool-provider", "v0")])),
11922            ("nine entries", nine),
11923        ] {
11924            // A refused open answers at once; one that reached the module
11925            // would wait for its bind ack and trip this timeout.
11926            let responses = tokio::time::timeout(
11927                Duration::from_secs(1),
11928                handler.handle_control_frame(
11929                    &route_ctx(ConnectionId::new(148)).0,
11930                    route_open_frame_with_role_versions(
11931                        404,
11932                        "aft",
11933                        unique_project_root("role-versions-malformed"),
11934                        Some(malformed),
11935                    ),
11936                ),
11937            )
11938            .await
11939            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11940            .unwrap();
11941            assert_eq!(responses.len(), 1, "{label}");
11942            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11943            let error = parse_error(&responses[0]);
11944            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11945            assert_eq!(
11946                error["detail"]["field"], "role_versions",
11947                "{label}: {error}"
11948            );
11949            assert!(
11950                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11951                "{label}: a malformed declaration is terminal"
11952            );
11953            assert!(
11954                module_rx.try_recv().is_err(),
11955                "{label}: the module must never see a bind"
11956            );
11957        }
11958    }
11959
11960    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
11961    /// consumer can tell this daemon forwards the field from one that would
11962    /// drop it.
11963    #[tokio::test]
11964    async fn route_role_versions_capability_is_advertised() {
11965        let handler = ControlHandler::new(Arc::new(Registry::default()));
11966        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11967        let ack = hello_via_sink(
11968            &handler,
11969            &ctx,
11970            &mut rx,
11971            hello_frame("m", PROTOCOL_VERSION, 1),
11972        )
11973        .await;
11974        let ack = parse_ack(&ack);
11975        assert!(
11976            ack.subc_capabilities
11977                .iter()
11978                .any(|c| c == "route-role-versions/v1"),
11979            "{:?}",
11980            ack.subc_capabilities
11981        );
11982
11983        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11984        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11985        let reply = handler
11986            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11987            .await
11988            .unwrap()
11989            .pop()
11990            .unwrap();
11991        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11992            serde_json::from_slice(&reply.body).unwrap()
11993        else {
11994            panic!("not a server.describe reply");
11995        };
11996        assert!(
11997            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11998            "{capabilities:?}"
11999        );
12000    }
12001
12002    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
12003    /// catalog entry without `events` (the module declares none) from a daemon
12004    /// that never reports declared events.
12005    #[tokio::test]
12006    async fn catalog_events_capability_is_advertised() {
12007        let handler = ControlHandler::new(Arc::new(Registry::default()));
12008        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12009        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12010        let reply = handler
12011            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12012            .await
12013            .unwrap()
12014            .pop()
12015            .unwrap();
12016        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12017            serde_json::from_slice(&reply.body).unwrap()
12018        else {
12019            panic!("not a server.describe reply");
12020        };
12021        assert!(
12022            capabilities
12023                .iter()
12024                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12025            "{capabilities:?}"
12026        );
12027    }
12028
12029    #[tokio::test]
12030    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12031        let handler = ControlHandler::new(Arc::new(Registry::default()));
12032        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12033        let ack = hello_via_sink(
12034            &handler,
12035            &module,
12036            &mut module_rx,
12037            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12038        )
12039        .await;
12040        let ack = parse_ack(&ack);
12041        assert!(
12042            ack.subc_capabilities
12043                .iter()
12044                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12045            "{:?}",
12046            ack.subc_capabilities
12047        );
12048        assert!(ack
12049            .subc_capabilities
12050            .iter()
12051            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12052
12053        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12054        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12055        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12056        let reply = handler
12057            .handle_control_frame(&client, frame)
12058            .await
12059            .unwrap()
12060            .pop()
12061            .unwrap();
12062        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12063            serde_json::from_slice(&reply.body).unwrap()
12064        else {
12065            panic!("not a server.describe reply");
12066        };
12067        assert!(
12068            capabilities
12069                .iter()
12070                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12071            "{capabilities:?}"
12072        );
12073        assert!(capabilities
12074            .iter()
12075            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12076    }
12077
12078    #[tokio::test]
12079    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12080        let handler = ControlHandler::new(Arc::new(Registry::default()));
12081        let (module, mut rx) = route_ctx(ConnectionId::new(156));
12082        let ack = hello_via_sink(
12083            &handler,
12084            &module,
12085            &mut rx,
12086            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12087        )
12088        .await;
12089        assert!(parse_ack(&ack)
12090            .subc_capabilities
12091            .iter()
12092            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12093        assert!(parse_ack(&ack)
12094            .subc_capabilities
12095            .iter()
12096            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12097        let (client, _rx) = route_ctx(ConnectionId::new(157));
12098        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12099        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12100        let reply = handler
12101            .handle_control_frame(&client, frame)
12102            .await
12103            .unwrap()
12104            .pop()
12105            .unwrap();
12106        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12107            serde_json::from_slice(&reply.body).unwrap()
12108        else {
12109            panic!("server.describe reply")
12110        };
12111        assert!(capabilities
12112            .iter()
12113            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12114        assert!(capabilities
12115            .iter()
12116            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12117    }
12118
12119    #[tokio::test]
12120    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12121        let handler = ControlHandler::new(Arc::new(Registry::default()));
12122        let (module, mut rx) = route_ctx(ConnectionId::new(158));
12123        let ack = hello_via_sink(
12124            &handler,
12125            &module,
12126            &mut rx,
12127            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12128        )
12129        .await;
12130        assert!(parse_ack(&ack)
12131            .subc_capabilities
12132            .iter()
12133            .any(|cap| cap == "scope-workdir/v1"));
12134        let (client, _rx) = route_ctx(ConnectionId::new(159));
12135        let frame = Frame::build(
12136            FrameType::Request,
12137            control_flags(),
12138            0,
12139            0,
12140            5,
12141            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12142        )
12143        .unwrap();
12144        let reply = handler
12145            .handle_control_frame(&client, frame)
12146            .await
12147            .unwrap()
12148            .remove(0);
12149        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12150            serde_json::from_slice(&reply.body).unwrap()
12151        else {
12152            panic!("server.describe reply")
12153        };
12154        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12155    }
12156
12157    #[tokio::test]
12158    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12159        let handler = ControlHandler::new(Arc::new(Registry::default()));
12160        let (module, mut rx) = route_ctx(ConnectionId::new(154));
12161        let ack = hello_via_sink(
12162            &handler,
12163            &module,
12164            &mut rx,
12165            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12166        )
12167        .await;
12168        assert!(parse_ack(&ack)
12169            .subc_capabilities
12170            .iter()
12171            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12172        let (client, _rx) = route_ctx(ConnectionId::new(155));
12173        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12174        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12175        let reply = handler
12176            .handle_control_frame(&client, frame)
12177            .await
12178            .unwrap()
12179            .pop()
12180            .unwrap();
12181        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12182            serde_json::from_slice(&reply.body).unwrap()
12183        else {
12184            panic!("not a server.describe reply")
12185        };
12186        assert!(capabilities
12187            .iter()
12188            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12189    }
12190
12191    #[tokio::test]
12192    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12193        let registry = Arc::new(Registry::default());
12194        let forwarding = Arc::new(ForwardingTable::default());
12195        let handler =
12196            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12197                .with_health_probe_timeout(Duration::from_secs(5));
12198        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12199        hello_via_sink(
12200            &handler,
12201            &module_ctx,
12202            &mut module_rx,
12203            non_routable_hello_frame_with_control_ops(
12204                "mcp",
12205                300,
12206                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12207            ),
12208        )
12209        .await;
12210        assert!(registry
12211            .get_module("mcp")
12212            .unwrap()
12213            .unwrap()
12214            .manifest
12215            .provides
12216            .is_empty());
12217
12218        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12219        let route_response = handler
12220            .handle_control_frame(
12221                &route_client_ctx,
12222                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12223            )
12224            .await
12225            .unwrap();
12226        assert_eq!(route_response[0].header.ty, FrameType::Error);
12227        assert_eq!(
12228            parse_error(&route_response[0])["code"],
12229            "target_unavailable"
12230        );
12231        assert!(parse_error(&route_response[0])["message"]
12232            .as_str()
12233            .unwrap()
12234            .contains("does not provide the requested target"));
12235        assert!(module_rx.try_recv().is_err());
12236
12237        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12238        let health_handler = handler.clone();
12239        let health_task = tokio::spawn(async move {
12240            health_handler
12241                .handle_control_frame(
12242                    &health_client_ctx,
12243                    supervisor_health_probe_frame(302, "mcp"),
12244                )
12245                .await
12246                .unwrap()
12247        });
12248        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12249            .await
12250            .unwrap()
12251            .unwrap();
12252        assert_eq!(
12253            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12254            ModuleControlRequest::HealthCheck {}
12255        );
12256        handler
12257            .handle_control_frame(
12258                &module_ctx,
12259                health_response(health_frame.header.corr, HealthStatus::Ok),
12260            )
12261            .await
12262            .unwrap();
12263        let health_response = health_task.await.unwrap();
12264        assert_eq!(health_response[0].header.ty, FrameType::Response);
12265        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12266            ClientControlResponse::SupervisorHealthProbe {
12267                module_id, status, ..
12268            } => {
12269                assert_eq!(module_id, "mcp");
12270                assert_eq!(status, HealthStatus::Ok);
12271            }
12272            other => panic!("unexpected health response: {other:?}"),
12273        }
12274
12275        // Exercise the forwarding cleanup path directly while leaving the registry
12276        // advertisement in place. If cleanup leaves a stale control sink behind,
12277        // the next probe will enqueue onto it and wait for the long probe timeout
12278        // instead of returning an immediate no-connection error.
12279        forwarding
12280            .cleanup_connection(module_ctx.connection_id)
12281            .unwrap();
12282        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12283        let cleanup_response = tokio::time::timeout(
12284            Duration::from_millis(200),
12285            handler.handle_control_frame(
12286                &cleanup_probe_ctx,
12287                supervisor_health_probe_frame(303, "mcp"),
12288            ),
12289        )
12290        .await
12291        .expect("probe should fail immediately when the control lane is gone")
12292        .unwrap();
12293        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12294        assert_eq!(
12295            parse_error(&cleanup_response[0])["code"],
12296            "target_unavailable"
12297        );
12298        assert!(parse_error(&cleanup_response[0])["message"]
12299            .as_str()
12300            .unwrap()
12301            .contains("no module connection"));
12302
12303        handler
12304            .cleanup_connection(module_ctx.connection_id)
12305            .unwrap();
12306    }
12307
12308    #[tokio::test]
12309    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12310        let registry = Arc::new(Registry::default());
12311        let supervisor_handle = SupervisorHandle::new();
12312        let supervisor =
12313            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12314                .with_handle(supervisor_handle.clone())
12315                .with_connection_file_path(
12316                    std::env::temp_dir()
12317                        .join(format!("subc-route-open-warming-{}", std::process::id())),
12318                );
12319        let module = supervisor
12320            .supervise_configured(
12321                ModuleSpec {
12322                    module_id: "warming".to_string(),
12323                    program: fake_aft_stub_path(),
12324                    args: Vec::new(),
12325                    env: Vec::new(),
12326                    reserved: false,
12327                    reserved_prefixes: Vec::new(),
12328                    protocol: ModuleProtocol::Subc,
12329                    overlap: Default::default(),
12330                },
12331                true,
12332            )
12333            .unwrap();
12334        assert_eq!(module.state().unwrap(), ModuleState::Running);
12335
12336        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12337        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12338        let response = handler
12339            .handle_control_frame(
12340                &ctx,
12341                route_open_frame(304, "warming", unique_project_root("warming")),
12342            )
12343            .await
12344            .unwrap();
12345        module.stop().await.unwrap();
12346
12347        assert_eq!(response[0].header.ty, FrameType::Error);
12348        let error = parse_error(&response[0]);
12349        assert_eq!(error["code"], "module_warming");
12350        assert!(error["message"]
12351            .as_str()
12352            .unwrap()
12353            .contains("state=running, enabled=true, live=false"));
12354    }
12355
12356    #[test]
12357    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12358        let handler = ControlHandler::new(Arc::new(Registry::default()));
12359        let capture = EventCapture::default();
12360        let _subscriber =
12361            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12362        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12363        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12364        let pending = (0..limit).collect::<Vec<_>>();
12365        let response = handler
12366            .route_open_capacity_refusal(
12367                &ctx,
12368                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12369                "busy",
12370                pending.len(),
12371                limit,
12372            )
12373            .unwrap();
12374        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12375        let event = capture
12376            .events()
12377            .into_iter()
12378            .find(|event| {
12379                event.target == "control"
12380                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12381            })
12382            .expect("connection admission refusal event");
12383        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12384        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12385    }
12386
12387    #[test]
12388    fn route_open_target_cap_logs_admission_reason_and_capacity() {
12389        let handler = ControlHandler::new(Arc::new(Registry::default()));
12390        let capture = EventCapture::default();
12391        let _subscriber =
12392            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12393        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12394        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12395        let guards = (0..limit)
12396            .map(|_| {
12397                handler
12398                    .route_bind_concurrency
12399                    .try_admit("busy", limit)
12400                    .unwrap()
12401            })
12402            .collect::<Vec<_>>();
12403        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12404            Err(in_flight) => in_flight,
12405            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12406        };
12407        let response = handler
12408            .route_open_target_capacity_refusal(
12409                &ctx,
12410                &route_open_frame(397, "busy", unique_project_root("target-cap")),
12411                "busy",
12412                in_flight,
12413            )
12414            .unwrap();
12415        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12416        let event = capture
12417            .events()
12418            .into_iter()
12419            .find(|event| {
12420                event.target == "control"
12421                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12422            })
12423            .expect("target admission refusal event");
12424        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12425        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12426        drop(guards);
12427    }
12428
12429    #[tokio::test(flavor = "current_thread")]
12430    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12431        let handler = ControlHandler::new(Arc::new(Registry::default()));
12432        let capture = EventCapture::default();
12433        let _subscriber =
12434            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12435        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12436        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12437
12438        let _ = handler
12439            .handle_client_control_request(
12440                &ctx,
12441                frame,
12442                ClientControlRequest::SupervisorSetEnabled {
12443                    module_id: "broca".to_string(),
12444                    enabled: false,
12445                },
12446                &mut None,
12447            )
12448            .await
12449            .unwrap();
12450
12451        let events = capture
12452            .events()
12453            .into_iter()
12454            .filter(|event| {
12455                event.target == "control"
12456                    && event.fields.get("message").map(String::as_str)
12457                        == Some("supervisor request received")
12458            })
12459            .collect::<Vec<_>>();
12460        assert_eq!(events.len(), 1, "one supervisor request log line");
12461        let event = &events[0];
12462        assert_eq!(event.level, tracing::Level::INFO);
12463        assert_eq!(
12464            event.fields.get("op"),
12465            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12466        );
12467        assert_eq!(
12468            event.fields.get("module_id"),
12469            Some(&"\"broca\"".to_string())
12470        );
12471        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12472        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12473        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12474    }
12475
12476    #[tokio::test(flavor = "current_thread")]
12477    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12478        let registry = Arc::new(Registry::default());
12479        let module_connection = ConnectionId::new(303);
12480        registry
12481            .register_with_control_ops(
12482                manifest("aft", PROTOCOL_VERSION),
12483                PROTOCOL_VERSION,
12484                module_connection,
12485                module_baseline_control_ops(),
12486            )
12487            .unwrap();
12488        let handler = ControlHandler::new(registry);
12489        let capture = EventCapture::default();
12490        let _subscriber =
12491            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12492        let (ctx, _rx) = route_ctx(module_connection);
12493        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12494
12495        let _ = handler
12496            .handle_client_control_request(
12497                &ctx,
12498                frame,
12499                ClientControlRequest::SupervisorSetEnabled {
12500                    module_id: "broca".to_string(),
12501                    enabled: false,
12502                },
12503                &mut None,
12504            )
12505            .await
12506            .unwrap();
12507
12508        let callers = capture
12509            .events()
12510            .into_iter()
12511            .filter(|event| {
12512                event.fields.get("message").map(String::as_str)
12513                    == Some("supervisor request received")
12514            })
12515            .map(|event| event.fields.get("caller").cloned())
12516            .collect::<Vec<_>>();
12517        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12518    }
12519
12520    #[tokio::test(flavor = "current_thread")]
12521    async fn supervisor_list_does_not_log_request_received() {
12522        let handler = ControlHandler::new(Arc::new(Registry::default()));
12523        let capture = EventCapture::default();
12524        let _subscriber =
12525            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12526        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12527        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12528
12529        handler
12530            .handle_client_control_request(
12531                &ctx,
12532                frame,
12533                ClientControlRequest::SupervisorList {},
12534                &mut None,
12535            )
12536            .await
12537            .unwrap();
12538
12539        assert!(capture.events().into_iter().all(|event| {
12540            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12541        }));
12542    }
12543
12544    #[tokio::test(flavor = "current_thread")]
12545    async fn supervisor_rescan_preview_does_not_log_request_received() {
12546        let handler = ControlHandler::new(Arc::new(Registry::default()));
12547        let capture = EventCapture::default();
12548        let _subscriber =
12549            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12550        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
12551        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12552
12553        handler
12554            .handle_client_control_request(
12555                &ctx,
12556                frame,
12557                ClientControlRequest::SupervisorRescan { preview: true },
12558                &mut None,
12559            )
12560            .await
12561            .unwrap();
12562
12563        assert!(capture.events().into_iter().all(|event| {
12564            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12565        }));
12566    }
12567
12568    struct ApproveOperator;
12569    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
12570        fn prompt(
12571            &self,
12572            _: &str,
12573            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
12574        ) -> crate::operator_confirm::ProviderResult {
12575            crate::operator_confirm::ProviderResult::Approved
12576        }
12577    }
12578
12579    #[tokio::test]
12580    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
12581        assert_eq!(
12582            ControlHandler::default().privileged_operator_prompt,
12583            PrivilegedOperatorPrompt::Log
12584        );
12585        for (initial, changed, running) in [
12586            ("log", "enforce", PrivilegedOperatorPrompt::Log),
12587            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
12588        ] {
12589            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
12590            let path = dir.join("subc.jsonc");
12591            std::fs::write(
12592                &path,
12593                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
12594            )
12595            .unwrap();
12596            let config = crate::daemon_config::load(&path).unwrap().unwrap();
12597            let registry = Arc::new(Registry::default());
12598            let handle = SupervisorHandle::new();
12599            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
12600                .with_handle(handle.clone());
12601            let handler = ControlHandler::new(registry)
12602                .with_supervisor(handle)
12603                .with_privileged_operator_prompt(config.privileged_operator_prompt)
12604                .with_supervisor_rescan(supervisor, &path, config.port);
12605            handler.forwarding.operator_confirms().configure(
12606                Arc::new(ApproveOperator),
12607                crate::operator_confirm::OperatorLimits::default(),
12608            );
12609            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
12610
12611            for (disk_mode, needs_restart) in [
12612                (Some(initial), false),
12613                (Some(changed), true),
12614                (Some(changed), true),
12615                (None, running == PrivilegedOperatorPrompt::Enforce),
12616                (Some(initial), false),
12617            ] {
12618                let mut doc = json!({"version":1});
12619                if let Some(mode) = disk_mode {
12620                    doc["privileged_operator_prompt"] = json!(mode);
12621                }
12622                std::fs::write(&path, doc.to_string()).unwrap();
12623                for preview in [true, false] {
12624                    let request = ClientControlRequest::SupervisorRescan { preview };
12625                    let frame = Frame::build(
12626                        FrameType::Request,
12627                        control_flags(),
12628                        0,
12629                        0,
12630                        1,
12631                        serde_json::to_vec(&request).unwrap(),
12632                    )
12633                    .unwrap();
12634                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12635                    assert_eq!(frames.len(), 1);
12636                    let ClientControlResponse::SupervisorRescan { result } =
12637                        serde_json::from_slice(&frames[0].body).unwrap()
12638                    else {
12639                        panic!("expected a rescan response");
12640                    };
12641                    let expected = if needs_restart {
12642                        vec!["privileged_operator_prompt"]
12643                    } else {
12644                        Vec::new()
12645                    };
12646                    assert_eq!(result.restart_required, expected);
12647                    assert_eq!(handler.privileged_operator_prompt, running);
12648                    assert_eq!(
12649                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
12650                        running
12651                    );
12652                }
12653            }
12654        }
12655    }
12656
12657    #[tokio::test]
12658    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
12659        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
12660        let path = dir.join("subc.jsonc");
12661        let registry = Arc::new(Registry::default());
12662        let handle = SupervisorHandle::new();
12663        let supervisor =
12664            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
12665        let handler = ControlHandler::new(registry)
12666            .with_supervisor(handle)
12667            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
12668            .with_supervisor_rescan(supervisor, &path, None);
12669        handler.forwarding.operator_confirms().configure(
12670            Arc::new(ApproveOperator),
12671            crate::operator_confirm::OperatorLimits::default(),
12672        );
12673        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
12674        for preview in [true, false] {
12675            std::fs::write(
12676                &path,
12677                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
12678            )
12679            .unwrap();
12680            let request = ClientControlRequest::SupervisorRescan { preview };
12681            let frame = Frame::build(
12682                FrameType::Request,
12683                control_flags(),
12684                0,
12685                0,
12686                1,
12687                serde_json::to_vec(&request).unwrap(),
12688            )
12689            .unwrap();
12690            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12691            assert_eq!(frames.len(), 1);
12692            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
12693            assert_eq!(
12694                handler.privileged_operator_prompt,
12695                PrivilegedOperatorPrompt::Enforce
12696            );
12697        }
12698    }
12699
12700    /// One wire code has several senders, so the refusal line names the check
12701    /// that refused. This drives the shared refusal path for ordinary refusals
12702    /// with an unregistered
12703    /// target and requires the branch label on the event.
12704    #[tokio::test]
12705    async fn route_open_refusal_names_the_check_that_refused() {
12706        let handler = ControlHandler::new(Arc::new(Registry::default()));
12707        let capture = EventCapture::default();
12708        let _subscriber =
12709            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12710        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12711        let response = handler
12712            .handle_control_frame(
12713                &ctx,
12714                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
12715            )
12716            .await
12717            .unwrap();
12718
12719        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12720        let event = capture
12721            .events()
12722            .into_iter()
12723            .find(|event| {
12724                event.target == "control"
12725                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12726            })
12727            .expect("route.open refusal event");
12728        assert_eq!(
12729            event.fields.get("reason"),
12730            Some(&"\"not_registered\"".to_string())
12731        );
12732    }
12733
12734    #[tokio::test]
12735    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
12736        let registry = Arc::new(Registry::default());
12737        let supervisor_handle = SupervisorHandle::new();
12738        let supervisor =
12739            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12740                .with_handle(supervisor_handle.clone())
12741                .with_connection_file_path(std::env::temp_dir().join(format!(
12742                    "subc-route-open-refusal-info-{}",
12743                    std::process::id()
12744                )));
12745        let module = supervisor
12746            .supervise_configured(
12747                ModuleSpec {
12748                    module_id: "warming".to_string(),
12749                    program: fake_aft_stub_path(),
12750                    args: Vec::new(),
12751                    env: Vec::new(),
12752                    reserved: false,
12753                    reserved_prefixes: Vec::new(),
12754                    protocol: ModuleProtocol::Subc,
12755                    overlap: Default::default(),
12756                },
12757                true,
12758            )
12759            .unwrap();
12760        assert_eq!(module.state().unwrap(), ModuleState::Running);
12761
12762        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12763        assert!(handler
12764            .counters()
12765            .snapshot()
12766            .get("route_open_refused_by_code")
12767            .is_none());
12768        let capture = EventCapture::default();
12769        let _subscriber =
12770            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12771        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
12772        let response = handler
12773            .handle_control_frame(
12774                &ctx,
12775                route_open_frame(394, "warming", unique_project_root("refusal-info")),
12776            )
12777            .await
12778            .unwrap();
12779        module.stop().await.unwrap();
12780
12781        assert_eq!(parse_error(&response[0])["code"], "module_warming");
12782        let event = capture
12783            .events()
12784            .into_iter()
12785            .find(|event| {
12786                event.target == "control"
12787                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
12788            })
12789            .expect("route.open refusal event");
12790        assert_eq!(
12791            event.fields.get("module_id"),
12792            Some(&"\"warming\"".to_string())
12793        );
12794        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
12795        assert_eq!(
12796            event.fields.get("reason"),
12797            Some(&"\"supervised_not_registered\"".to_string())
12798        );
12799        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
12800        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
12801        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
12802        assert_eq!(
12803            handler.counters().snapshot()["route_open_refused_by_code"],
12804            json!({ "module_warming": 1 })
12805        );
12806    }
12807
12808    const OUTAGE_START: &str = "route.open refusing module: not serving";
12809    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
12810
12811    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
12812        capture
12813            .events()
12814            .into_iter()
12815            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
12816            .collect()
12817    }
12818
12819    fn supervise_stub(
12820        registry: &Arc<Registry>,
12821        module_id: &str,
12822        enabled: bool,
12823    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
12824        let supervisor_handle = SupervisorHandle::new();
12825        let supervisor =
12826            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
12827                .with_handle(supervisor_handle.clone())
12828                .with_connection_file_path(std::env::temp_dir().join(format!(
12829                    "subc-route-outage-{module_id}-{}",
12830                    std::process::id()
12831                )));
12832        let module = supervisor
12833            .supervise_configured(
12834                ModuleSpec {
12835                    module_id: module_id.to_string(),
12836                    program: fake_aft_stub_path(),
12837                    args: Vec::new(),
12838                    env: Vec::new(),
12839                    reserved: false,
12840                    reserved_prefixes: Vec::new(),
12841                    protocol: ModuleProtocol::Subc,
12842                    overlap: Default::default(),
12843                },
12844                enabled,
12845            )
12846            .unwrap();
12847        (supervisor_handle, module)
12848    }
12849
12850    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
12851        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
12852            module_id: module_id.to_string(),
12853            drain_timeout_ms: Some(50),
12854        })
12855        .unwrap();
12856        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
12857    }
12858
12859    /// Two handlers built over one forwarding table must share one outage
12860    /// tracker; separate trackers would each log their own opening line for
12861    /// the same outage.
12862    #[test]
12863    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
12864        let registry = Arc::new(Registry::default());
12865        let forwarding = Arc::new(ForwardingTable::default());
12866        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12867        let second = ControlHandler::with_forwarding(registry, forwarding);
12868        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
12869    }
12870
12871    /// A client can name any module id it likes. Refusing an unknown one,
12872    /// however often, must not create outage state or outage lines, or the
12873    /// tracker would be a memory sink any client could fill.
12874    #[tokio::test(flavor = "current_thread")]
12875    async fn route_open_unknown_module_refusals_add_no_outage_state() {
12876        let handler = ControlHandler::new(Arc::new(Registry::default()));
12877        let capture = EventCapture::default();
12878        let _subscriber =
12879            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12880        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12881        for corr in 0..8 {
12882            let response = handler
12883                .handle_control_frame(
12884                    &ctx,
12885                    route_open_frame(
12886                        380 + corr,
12887                        &format!("nobody-{corr}"),
12888                        unique_project_root("outage-unknown"),
12889                    ),
12890                )
12891                .await
12892                .unwrap();
12893            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12894        }
12895
12896        assert_eq!(handler.route_outages.tracked_module_count(), 0);
12897        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12898        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12899    }
12900
12901    /// Drives the refusal path end to end: a supervised module that served
12902    /// before and stopped being registered with no instruction to stop is a
12903    /// WARN, and the same module refused after an operator `supervisor.restart`
12904    /// is an INFO.
12905    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12906    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12907        let registry = Arc::new(Registry::default());
12908        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
12909        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12910        let capture = EventCapture::default();
12911        let _subscriber =
12912            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12913        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12914        // The stub never registers, so pretend it served once: otherwise every
12915        // refusal would fall in its startup window.
12916        handler.route_outages.record_accepted("outage-restart");
12917
12918        let response = handler
12919            .handle_control_frame(
12920                &ctx,
12921                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12922            )
12923            .await
12924            .unwrap();
12925        assert_eq!(response[0].header.ty, FrameType::Error);
12926        let starts = outage_lines(&capture, OUTAGE_START);
12927        assert_eq!(starts.len(), 1, "{starts:?}");
12928        assert_eq!(starts[0].level, tracing::Level::WARN);
12929        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12930        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12931        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12932        handler.route_outages.record_accepted("outage-restart");
12933        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12934
12935        let restart = handler
12936            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12937            .await
12938            .unwrap();
12939        assert_eq!(
12940            restart[0].header.ty,
12941            FrameType::Response,
12942            "{:?}",
12943            parse_error(&restart[0])
12944        );
12945        handler
12946            .handle_control_frame(
12947                &ctx,
12948                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
12949            )
12950            .await
12951            .unwrap();
12952        module.stop().await.unwrap();
12953
12954        let starts = outage_lines(&capture, OUTAGE_START);
12955        assert_eq!(starts.len(), 2, "{starts:?}");
12956        assert_eq!(starts[1].level, tracing::Level::INFO);
12957        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
12958    }
12959
12960    /// A restart refused before it touched the module (here: the module is
12961    /// disabled) must clear its operator mark, so the next real outage is
12962    /// still reported as a warning.
12963    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12964    async fn failed_operator_restart_leaves_no_operator_mark() {
12965        let registry = Arc::new(Registry::default());
12966        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
12967        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12968        let capture = EventCapture::default();
12969        let _subscriber =
12970            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12971        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
12972        handler.route_outages.record_accepted("outage-disabled");
12973
12974        let restart = handler
12975            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
12976            .await
12977            .unwrap();
12978        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
12979        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
12980
12981        handler
12982            .handle_control_frame(
12983                &ctx,
12984                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
12985            )
12986            .await
12987            .unwrap();
12988        let starts = outage_lines(&capture, OUTAGE_START);
12989        assert_eq!(starts.len(), 1, "{starts:?}");
12990        assert_eq!(starts[0].level, tracing::Level::WARN);
12991    }
12992
12993    #[tokio::test(flavor = "current_thread")]
12994    async fn route_open_unknown_module_escapes_target_module_id() {
12995        let handler = ControlHandler::new(Arc::new(Registry::default()));
12996        let capture = EventCapture::default();
12997        let _subscriber =
12998            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12999        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13000        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13001        let response = handler
13002            .handle_control_frame(
13003                &ctx,
13004                route_open_frame(
13005                    395,
13006                    hostile_module_id,
13007                    unique_project_root("hostile-target-module-id"),
13008                ),
13009            )
13010            .await
13011            .unwrap();
13012
13013        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13014        let event = capture
13015            .events()
13016            .into_iter()
13017            .find(|event| {
13018                event.target == "control"
13019                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13020            })
13021            .expect("route.open unknown-module refusal event");
13022        let logged = event.fields.get("module_id").expect("module_id field");
13023        assert!(!logged.bytes().any(|byte| byte < 0x20));
13024        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13025    }
13026
13027    #[tokio::test(flavor = "current_thread")]
13028    async fn route_open_module_rejection_uses_daemon_counter_key() {
13029        let registry = Arc::new(Registry::default());
13030        let forwarding = Arc::new(ForwardingTable::default());
13031        let handler =
13032            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13033        let module_connection = ConnectionId::new(95);
13034        let (module_ctx, mut module_rx) = route_ctx(module_connection);
13035        hello_via_sink(
13036            &handler,
13037            &module_ctx,
13038            &mut module_rx,
13039            hello_frame("aft", PROTOCOL_VERSION, 395),
13040        )
13041        .await;
13042
13043        let client_connection = ConnectionId::new(96);
13044        let (client_ctx, _client_rx) = route_ctx(client_connection);
13045        let capture = EventCapture::default();
13046        let _subscriber =
13047            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13048        let (route_task, bind) = relay_route_open(
13049            &handler,
13050            client_connection,
13051            &client_ctx.egress,
13052            &mut module_rx,
13053            396,
13054            "aft",
13055            "hostile-module-code",
13056        )
13057        .await;
13058        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13059        let rejection = Frame::build(
13060            FrameType::Error,
13061            control_flags(),
13062            0,
13063            0,
13064            bind.header.corr,
13065            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13066        )
13067        .unwrap();
13068        handler
13069            .handle_control_frame(&module_ctx, rejection)
13070            .await
13071            .unwrap();
13072
13073        let response = route_task.await.unwrap();
13074        assert_eq!(parse_error(&response[0])["code"], hostile_code);
13075        let counters = handler.counters().snapshot();
13076        assert_eq!(
13077            counters["route_open_refused_by_code"],
13078            json!({ "module_rejected": 1 })
13079        );
13080        assert!(counters["route_open_refused_by_code"]
13081            .get(hostile_code)
13082            .is_none());
13083
13084        let event = capture
13085            .events()
13086            .into_iter()
13087            .find(|event| {
13088                event.target == "control"
13089                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13090            })
13091            .expect("route.open module-rejection refusal event");
13092        let logged = event.fields.get("module_code").expect("module_code field");
13093        assert!(!logged.bytes().any(|byte| byte < 0x20));
13094        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13095    }
13096
13097    #[tokio::test]
13098    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13099        let registry = Arc::new(Registry::default());
13100        let supervisor_handle = SupervisorHandle::new();
13101        let missing_program = std::env::temp_dir().join(format!(
13102            "subc-route-open-missing-program-{}",
13103            std::process::id()
13104        ));
13105        let supervisor =
13106            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13107                .with_handle(supervisor_handle.clone());
13108        let module = supervisor
13109            .supervise_configured(
13110                ModuleSpec {
13111                    module_id: "failed".to_string(),
13112                    program: missing_program,
13113                    args: Vec::new(),
13114                    env: Vec::new(),
13115                    reserved: false,
13116                    reserved_prefixes: Vec::new(),
13117                    protocol: ModuleProtocol::Subc,
13118                    overlap: Default::default(),
13119                },
13120                true,
13121            )
13122            .unwrap();
13123        assert_eq!(module.state().unwrap(), ModuleState::Failed);
13124
13125        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13126        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13127        let response = handler
13128            .handle_control_frame(
13129                &ctx,
13130                route_open_frame(305, "failed", unique_project_root("failed")),
13131            )
13132            .await
13133            .unwrap();
13134
13135        assert_eq!(response[0].header.ty, FrameType::Error);
13136        let error = parse_error(&response[0]);
13137        assert_eq!(error["code"], "target_unavailable");
13138        assert!(error["message"]
13139            .as_str()
13140            .unwrap()
13141            .contains("state=failed, enabled=true, live=false"));
13142    }
13143
13144    #[tokio::test]
13145    async fn route_open_role_mismatch_remains_target_unavailable() {
13146        let registry = Arc::new(Registry::default());
13147        let handler = ControlHandler::new(Arc::clone(&registry));
13148        handler
13149            .handle_control(
13150                ConnectionId::new(41),
13151                non_routable_hello_frame_with_control_ops("health-only", 306, None),
13152            )
13153            .unwrap();
13154
13155        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13156        let response = handler
13157            .handle_control_frame(
13158                &ctx,
13159                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13160            )
13161            .await
13162            .unwrap();
13163
13164        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13165        assert!(parse_error(&response[0])["message"]
13166            .as_str()
13167            .unwrap()
13168            .contains("does not provide the requested target"));
13169    }
13170
13171    #[tokio::test]
13172    async fn route_open_inactive_registration_remains_target_unavailable() {
13173        let registry = Arc::new(Registry::default());
13174        let handler = ControlHandler::new(Arc::clone(&registry));
13175        handler
13176            .handle_control(
13177                ConnectionId::new(43),
13178                hello_frame("inactive", PROTOCOL_VERSION, 308),
13179            )
13180            .unwrap();
13181        assert!(registry
13182            .set_module_state_for_test("inactive", ChannelState::Closed)
13183            .unwrap());
13184
13185        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13186        let response = handler
13187            .handle_control_frame(
13188                &ctx,
13189                route_open_frame(309, "inactive", unique_project_root("inactive")),
13190            )
13191            .await
13192            .unwrap();
13193
13194        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13195        assert!(parse_error(&response[0])["message"]
13196            .as_str()
13197            .unwrap()
13198            .contains("is not active"));
13199    }
13200
13201    #[tokio::test]
13202    async fn late_health_reply_is_recorded_through_the_module_response_path() {
13203        let registry = Arc::new(Registry::default());
13204        let forwarding = Arc::new(ForwardingTable::default());
13205        let supervisor_handle = SupervisorHandle::new();
13206        let supervisor =
13207            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
13208                .with_forwarding(Arc::clone(&forwarding))
13209                .with_handle(supervisor_handle.clone());
13210        let module = supervisor
13211            .supervise_configured(
13212                crate::ModuleSpec {
13213                    module_id: "late-health-response".to_string(),
13214                    program: PathBuf::from("disabled-module"),
13215                    args: Vec::new(),
13216                    env: Vec::new(),
13217                    reserved: false,
13218                    reserved_prefixes: Vec::new(),
13219                    protocol: ModuleProtocol::Subc,
13220                    overlap: Default::default(),
13221                },
13222                false,
13223            )
13224            .unwrap();
13225        let handler =
13226            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13227                .with_supervisor(supervisor_handle);
13228        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13229        handler
13230            .handle_control_frame(
13231                &module_ctx,
13232                hello_frame_with_control_ops(
13233                    "late-health-response",
13234                    PROTOCOL_VERSION,
13235                    7,
13236                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13237                ),
13238            )
13239            .await
13240            .unwrap();
13241        let probe_started_at = Instant::now() - Duration::from_millis(80);
13242        let pending = forwarding
13243            .begin_health_probe_rpc_for(
13244                "late-health-response",
13245                MODULE_CONTROL_OP_HEALTH_CHECK,
13246                probe_started_at,
13247                Instant::now() - Duration::from_millis(1),
13248            )
13249            .unwrap();
13250        assert!(forwarding
13251            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13252            .unwrap());
13253
13254        let responses = handler
13255            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13256            .await
13257            .unwrap();
13258
13259        assert!(responses.is_empty());
13260        let health = module.status().unwrap().health;
13261        assert_eq!(health.late_answer_count, 1);
13262        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13263    }
13264
13265    #[tokio::test]
13266    async fn health_probe_timeout_and_module_death_are_typed() {
13267        let registry = Arc::new(Registry::default());
13268        let forwarding = Arc::new(ForwardingTable::default());
13269        let handler =
13270            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13271                .with_health_probe_timeout(Duration::from_millis(50));
13272        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13273        hello_via_sink(
13274            &handler,
13275            &module_ctx,
13276            &mut module_rx,
13277            hello_frame_with_control_ops(
13278                "aft",
13279                PROTOCOL_VERSION,
13280                7,
13281                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13282            ),
13283        )
13284        .await;
13285
13286        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13287        let responses = handler
13288            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
13289            .await
13290            .unwrap();
13291        assert_eq!(responses[0].header.ty, FrameType::Error);
13292        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
13293        let _ = module_rx.try_recv();
13294
13295        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
13296        let health_handler = handler.clone();
13297        let death_task = tokio::spawn(async move {
13298            health_handler
13299                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
13300                .await
13301                .unwrap()
13302        });
13303        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13304            .await
13305            .unwrap()
13306            .unwrap();
13307        handler
13308            .cleanup_connection(module_ctx.connection_id)
13309            .unwrap();
13310        let responses = death_task.await.unwrap();
13311        assert_eq!(responses[0].header.ty, FrameType::Error);
13312        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
13313    }
13314
13315    #[test]
13316    fn hello_requires_exact_protocol_version() {
13317        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
13318            let registry = Arc::new(Registry::default());
13319            let handler = ControlHandler::new(Arc::clone(&registry));
13320            let responses = handler
13321                .handle_control(
13322                    ConnectionId::new(connection),
13323                    hello_frame("aft", offered, 9),
13324                )
13325                .unwrap();
13326
13327            assert_eq!(responses.len(), 1);
13328            assert_eq!(responses[0].header.ty, FrameType::Error);
13329            let error = parse_error(&responses[0]);
13330            assert_eq!(error["code"], "version_unsupported");
13331            assert!(registry.get_module("aft").unwrap().is_none());
13332            assert_eq!(registry.active_registration_count().unwrap(), 0);
13333        }
13334    }
13335
13336    #[test]
13337    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
13338        let registry = Arc::new(Registry::default());
13339        let forwarding = Arc::new(ForwardingTable::default());
13340        let handler =
13341            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13342        let module_connection = ConnectionId::new(301);
13343        let registration = registry
13344            .register_with_control_ops(
13345                manifest("aft-push", PROTOCOL_VERSION),
13346                PROTOCOL_VERSION,
13347                module_connection,
13348                module_baseline_control_ops(),
13349            )
13350            .unwrap();
13351        let (module_tx, _module_rx) = mpsc::channel(8);
13352        let endpoint = forwarding
13353            .register_module_connection(
13354                module_connection,
13355                "aft-push".to_string(),
13356                PROTOCOL_VERSION,
13357                manifest_concurrency(&registration.manifest),
13358                FrameSink::new(module_tx),
13359            )
13360            .unwrap();
13361
13362        // A push op this version does not know is ignored (forward-compat), not errored.
13363        let unknown = Frame::build(
13364            FrameType::Push,
13365            control_flags(),
13366            0,
13367            0,
13368            5,
13369            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
13370        )
13371        .unwrap();
13372        let out = handler.handle_status_update(endpoint, unknown).unwrap();
13373        assert!(
13374            out.is_empty(),
13375            "unknown push op must be ignored, got {out:?}"
13376        );
13377
13378        // A malformed body for a KNOWN op is a real error worth surfacing.
13379        let malformed = Frame::build(
13380            FrameType::Push,
13381            control_flags(),
13382            0,
13383            0,
13384            6,
13385            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13386        )
13387        .unwrap();
13388        let out = handler.handle_status_update(endpoint, malformed).unwrap();
13389        assert_eq!(out.len(), 1);
13390        assert_eq!(out[0].header.ty, FrameType::Error);
13391        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13392    }
13393
13394    #[test]
13395    fn hello_rejected_when_connection_already_owns_client_routes() {
13396        let registry = Arc::new(Registry::default());
13397        let forwarding = Arc::new(ForwardingTable::default());
13398        let handler =
13399            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13400        // Commits a client route on connection 202 (bound to a module on conn 101).
13401        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
13402        let client_connection = ConnectionId::new(202);
13403
13404        // That same connection now tries to register as a module: rejected, so one
13405        // connection never holds both client-route and module-endpoint state.
13406        let responses = handler
13407            .handle_control(
13408                client_connection,
13409                hello_frame("aft-second", PROTOCOL_VERSION, 9),
13410            )
13411            .unwrap();
13412        assert_eq!(responses[0].header.ty, FrameType::Error);
13413        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13414        assert!(registry.get_module("aft-second").unwrap().is_none());
13415    }
13416
13417    #[tokio::test]
13418    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13419        let registry = Arc::new(Registry::default());
13420        let forwarding = Arc::new(ForwardingTable::default());
13421        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13422        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13423        hello_via_sink(
13424            &handler,
13425            &module_ctx,
13426            &mut module_rx,
13427            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13428        )
13429        .await;
13430        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13431        let pending = forwarding
13432            .begin_route_bind_relay_for_test(
13433                client_ctx.connection_id,
13434                client_ctx.egress.clone(),
13435                2,
13436                "alpha",
13437            )
13438            .unwrap();
13439        forwarding
13440            .complete_pending_relay(
13441                module_ctx.connection_id,
13442                pending.corr,
13443                RouteBindRelayOutcome::Accepted,
13444            )
13445            .unwrap();
13446        client_rx.try_recv().unwrap();
13447        for module_id in ["beta", "alpha"] {
13448            let replies = handler
13449                .handle_control_frame(
13450                    &module_ctx,
13451                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13452                )
13453                .await
13454                .unwrap();
13455            assert_eq!(replies.len(), 1, "second HELLO must be refused");
13456            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13457        }
13458        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13459        assert!(registry.get_module("beta").unwrap().is_none());
13460        assert!(matches!(
13461            forwarding
13462                .lookup_data_route(
13463                    client_ctx.connection_id,
13464                    pending.client_channel,
13465                    pending.client_epoch,
13466                )
13467                .unwrap(),
13468            DataRoute::Client(DataRouteState::Bound(_))
13469        ));
13470        assert!(handler
13471            .hello_launch_nonces
13472            .lock()
13473            .unwrap()
13474            .presented(module_ctx.connection_id, Some("alpha-nonce")));
13475        assert!(module_rx.try_recv().is_err());
13476    }
13477
13478    #[test]
13479    fn reserved_module_hello_requires_matching_launch_nonce() {
13480        let registry = Arc::new(Registry::default());
13481        let supervisor = SupervisorHandle::new();
13482        // The supervisor recorded the nonce it injected when it spawned the reserved
13483        // module; the HELLO verifier checks against the same shared handle.
13484        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13485        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13486
13487        // A HELLO with NO nonce is rejected.
13488        let no_nonce = handler
13489            .handle_control(
13490                ConnectionId::new(1),
13491                hello_frame("vault", PROTOCOL_VERSION, 1),
13492            )
13493            .unwrap();
13494        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13495        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13496        assert!(registry.get_module("vault").unwrap().is_none());
13497
13498        // A HELLO with the WRONG nonce is rejected.
13499        let wrong = handler
13500            .handle_control(
13501                ConnectionId::new(2),
13502                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13503            )
13504            .unwrap();
13505        assert_eq!(wrong[0].header.ty, FrameType::Error);
13506        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13507        assert!(registry.get_module("vault").unwrap().is_none());
13508
13509        // A HELLO with the CORRECT nonce registers.
13510        let ok = handler
13511            .handle_control(
13512                ConnectionId::new(3),
13513                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13514            )
13515            .unwrap();
13516        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13517        assert!(registry.get_module("vault").unwrap().is_some());
13518    }
13519
13520    #[test]
13521    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13522        let registry = Arc::new(Registry::default());
13523        let supervisor = SupervisorHandle::new();
13524        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13525        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13526        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13527
13528        let squat = handler
13529            .handle_control(
13530                ConnectionId::new(1),
13531                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
13532            )
13533            .unwrap();
13534        assert_eq!(squat[0].header.ty, FrameType::Error);
13535        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
13536        assert!(parse_error(&squat[0])["message"]
13537            .as_str()
13538            .unwrap()
13539            .contains("fed:"));
13540
13541        let accepted_peer = handler
13542            .handle_control(
13543                ConnectionId::new(2),
13544                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
13545            )
13546            .unwrap();
13547        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
13548
13549        let accepted_short = handler
13550            .handle_control(
13551                ConnectionId::new(3),
13552                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
13553            )
13554            .unwrap();
13555        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
13556
13557        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
13558            let response = handler
13559                .handle_control(
13560                    ConnectionId::new(conn),
13561                    hello_frame(module_id, PROTOCOL_VERSION, conn),
13562                )
13563                .unwrap();
13564            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
13565        }
13566    }
13567
13568    #[test]
13569    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
13570        let registry = Arc::new(Registry::default());
13571        let supervisor = SupervisorHandle::new();
13572        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13573        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13574        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
13575        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13576
13577        let owner_nonce = handler
13578            .handle_control(
13579                ConnectionId::new(1),
13580                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
13581            )
13582            .unwrap();
13583        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
13584        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
13585        assert!(registry.get_module("fed:special").unwrap().is_none());
13586
13587        let exact_nonce = handler
13588            .handle_control(
13589                ConnectionId::new(2),
13590                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
13591            )
13592            .unwrap();
13593        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
13594        assert!(registry.get_module("fed:special").unwrap().is_some());
13595    }
13596
13597    #[test]
13598    fn non_reserved_module_ignores_launch_nonce() {
13599        let registry = Arc::new(Registry::default());
13600        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
13601        // registration succeeds whether a spawned process echoes a nonce or not.
13602        let handler = ControlHandler::new(Arc::clone(&registry));
13603        let no_nonce = handler
13604            .handle_control(
13605                ConnectionId::new(1),
13606                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
13607            )
13608            .unwrap();
13609        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
13610        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
13611
13612        let echoed_nonce = handler
13613            .handle_control(
13614                ConnectionId::new(2),
13615                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
13616            )
13617            .unwrap();
13618        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
13619        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
13620    }
13621
13622    #[test]
13623    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
13624        let handler = ControlHandler::default();
13625        let conn = ConnectionId::new(1);
13626        let malformed = Frame::build(
13627            FrameType::Hello,
13628            control_flags(),
13629            0,
13630            0,
13631            3,
13632            b"{not json".to_vec(),
13633        )
13634        .unwrap();
13635
13636        let error = handler.handle_control(conn, malformed).unwrap();
13637        assert_eq!(error[0].header.ty, FrameType::Error);
13638        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
13639
13640        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13641        let pong = handler.handle_control(conn, ping).unwrap();
13642        assert_eq!(pong[0].header.ty, FrameType::Pong);
13643        assert_eq!(pong[0].header.corr, 4);
13644    }
13645
13646    #[test]
13647    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
13648        let registry = Arc::new(Registry::default());
13649        let handler = ControlHandler::new(Arc::clone(&registry));
13650
13651        handler
13652            .handle_control(
13653                ConnectionId::new(1),
13654                hello_frame("aft", PROTOCOL_VERSION, 1),
13655            )
13656            .unwrap();
13657        let duplicate = handler
13658            .handle_control(
13659                ConnectionId::new(2),
13660                hello_frame("aft", PROTOCOL_VERSION, 2),
13661            )
13662            .unwrap();
13663
13664        assert_eq!(duplicate[0].header.ty, FrameType::Error);
13665        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13666        let registration = registry.get_module("aft").unwrap().unwrap();
13667        assert_eq!(registration.connection_id, ConnectionId::new(1));
13668    }
13669
13670    #[test]
13671    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
13672        let registry = Arc::new(Registry::default());
13673        let forwarding = Arc::new(ForwardingTable::default());
13674        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
13675        let handler =
13676            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13677                .with_process_liveness(process_liveness);
13678        let (ctx, route_channel, route_epoch) =
13679            bind_liveness_route(&registry, &forwarding, "aft-dead");
13680        let responses = handler
13681            .handle_route_poll(
13682                &ctx,
13683                route_poll_frame(41, PollKind::Liveness, route_channel),
13684                route_channel,
13685                route_epoch,
13686                PollKind::Liveness,
13687            )
13688            .unwrap();
13689
13690        assert_eq!(responses.len(), 1);
13691        assert_eq!(responses[0].header.ty, FrameType::Response);
13692        assert_route_poll_liveness(&responses[0], false);
13693    }
13694
13695    #[test]
13696    fn liveness_poll_without_process_source_uses_bound_route() {
13697        let registry = Arc::new(Registry::default());
13698        let forwarding = Arc::new(ForwardingTable::default());
13699        let handler =
13700            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13701        let (ctx, route_channel, route_epoch) =
13702            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
13703        let responses = handler
13704            .handle_route_poll(
13705                &ctx,
13706                route_poll_frame(42, PollKind::Liveness, route_channel),
13707                route_channel,
13708                route_epoch,
13709                PollKind::Liveness,
13710            )
13711            .unwrap();
13712
13713        assert_route_poll_liveness(&responses[0], true);
13714    }
13715
13716    #[test]
13717    fn liveness_poll_untracked_process_source_uses_bound_route() {
13718        let registry = Arc::new(Registry::default());
13719        let forwarding = Arc::new(ForwardingTable::default());
13720        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
13721        let handler =
13722            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13723                .with_process_liveness(process_liveness);
13724        let (ctx, route_channel, route_epoch) =
13725            bind_liveness_route(&registry, &forwarding, "aft-untracked");
13726        let responses = handler
13727            .handle_route_poll(
13728                &ctx,
13729                route_poll_frame(43, PollKind::Liveness, route_channel),
13730                route_channel,
13731                route_epoch,
13732                PollKind::Liveness,
13733            )
13734            .unwrap();
13735
13736        assert_route_poll_liveness(&responses[0], true);
13737    }
13738
13739    #[tokio::test]
13740    async fn unknown_op_returns_unknown_control_op() {
13741        let handler = ControlHandler::default();
13742        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
13743        let request = Frame::build(
13744            FrameType::Request,
13745            control_flags(),
13746            0,
13747            0,
13748            55,
13749            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
13750        )
13751        .unwrap();
13752
13753        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13754
13755        assert_eq!(response.len(), 1);
13756        assert_eq!(response[0].header.ty, FrameType::Error);
13757        assert_eq!(response[0].header.corr, 55);
13758        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
13759    }
13760
13761    #[tokio::test]
13762    async fn supervisor_provenance_rejects_unknown_exact_module() {
13763        let handler = ControlHandler::default();
13764        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
13765        let request = Frame::build(
13766            FrameType::Request,
13767            control_flags(),
13768            0,
13769            0,
13770            57,
13771            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
13772        )
13773        .unwrap();
13774
13775        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13776
13777        assert_eq!(response.len(), 1);
13778        assert_eq!(response[0].header.ty, FrameType::Error);
13779        assert_eq!(response[0].header.corr, 57);
13780        let error = parse_error(&response[0]);
13781        assert_eq!(error["code"], "unknown_module");
13782        assert_eq!(error["message"], "module_id 'missing' is not supervised");
13783    }
13784
13785    #[test]
13786    fn provenance_probe_override_keeps_handler_tests_deterministic() {
13787        let expected = subc_control::RunningImageAgreement::Unavailable {
13788            reason: subc_control::RunningImageUnavailableReason::HashFailed,
13789        };
13790        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
13791        assert_eq!(handler.provenance_probe_override, Some(expected));
13792    }
13793
13794    #[test]
13795    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
13796        let verdict = reload_verdict(
13797            std::path::Path::new("/bin/new"),
13798            Some(std::path::Path::new("/bin/old")),
13799            subc_control::RunningImageAgreement::Unavailable {
13800                reason: subc_control::RunningImageUnavailableReason::HashFailed,
13801            },
13802        );
13803        assert!(matches!(
13804            verdict.path,
13805            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
13806                if configured == std::path::Path::new("/bin/new")
13807                    && spawned_from == std::path::Path::new("/bin/old")
13808        ));
13809    }
13810
13811    #[test]
13812    fn launch_spec_without_running_digest_is_unavailable_not_match() {
13813        use subc_control::{
13814            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
13815        };
13816        let configured = crate::supervise::ModuleSpec {
13817            module_id: "legacy".into(),
13818            program: "/module".into(),
13819            args: Vec::new(),
13820            env: Vec::new(),
13821            reserved: false,
13822            reserved_prefixes: Vec::new(),
13823            protocol: subc_control::ModuleProtocol::Subc,
13824            overlap: Default::default(),
13825        };
13826        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
13827        assert_eq!(
13828            launch_spec_agreement(&configured, false, Some(&digest)),
13829            Agreement::Unavailable {
13830                reason: Reason::NotRunning
13831            }
13832        );
13833        assert_eq!(
13834            launch_spec_agreement(&configured, true, None),
13835            Agreement::Unavailable {
13836                reason: Reason::SpawnedSpecUnavailable
13837            }
13838        );
13839        assert_eq!(
13840            launch_spec_agreement(&configured, true, Some(&digest)),
13841            Agreement::Match
13842        );
13843    }
13844
13845    #[cfg(windows)]
13846    #[test]
13847    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
13848        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
13849        let configured = dir.join("module.exe");
13850        std::fs::write(&configured, b"executable path fixture").unwrap();
13851        let spawned = std::fs::canonicalize(&configured).unwrap();
13852        assert_ne!(
13853            configured, spawned,
13854            "fixture must exercise different path spellings"
13855        );
13856        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
13857        let lower_case = configured.to_string_lossy().to_lowercase();
13858        for spelling in [
13859            &configured,
13860            Path::new(&forward_slashes),
13861            Path::new(&lower_case),
13862        ] {
13863            let verdict = reload_verdict(
13864                spelling,
13865                Some(&spawned),
13866                subc_control::RunningImageAgreement::Unavailable {
13867                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13868                },
13869            );
13870            assert_eq!(
13871                verdict.path,
13872                ReloadPathAgreement::Match,
13873                "configured {spelling:?}, spawned {spawned:?}"
13874            );
13875        }
13876    }
13877
13878    #[cfg(windows)]
13879    #[test]
13880    fn windows_reload_path_keeps_different_existing_program_pending() {
13881        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
13882        let old = dir.join("old.exe");
13883        let new = dir.join("new.exe");
13884        std::fs::write(&old, b"same executable content").unwrap();
13885        std::fs::copy(&old, &new).unwrap();
13886        let spawned = std::fs::canonicalize(&old).unwrap();
13887        let image = subc_control::RunningImageAgreement::Unavailable {
13888            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13889        };
13890        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
13891        assert!(
13892            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
13893            if configured == new && spawned_from == spawned)
13894        );
13895        let missing = dir.join("missing.exe");
13896        assert!(matches!(
13897            reload_verdict(&missing, Some(&spawned), image).path,
13898            ReloadPathAgreement::Mismatch { .. }
13899        ));
13900    }
13901
13902    #[test]
13903    fn reload_verdict_detects_replaced_image_at_same_path() {
13904        let image = subc_control::RunningImageAgreement::Mismatch {
13905            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13906                digest: "old".into(),
13907            },
13908            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13909                digest: "new".into(),
13910            },
13911        };
13912        let verdict = reload_verdict(
13913            std::path::Path::new("/bin/same"),
13914            Some(std::path::Path::new("/bin/same")),
13915            image.clone(),
13916        );
13917        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13918        assert_eq!(verdict.image, image);
13919    }
13920
13921    #[test]
13922    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13923        let image = subc_control::RunningImageAgreement::Unavailable {
13924            reason: subc_control::RunningImageUnavailableReason::NotRunning,
13925        };
13926        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13927        assert_eq!(
13928            verdict.path,
13929            subc_control::ReloadPathAgreement::Unavailable {
13930                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13931            }
13932        );
13933        assert_eq!(verdict.image, image);
13934
13935        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13936            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13937        };
13938        let verdict = reload_verdict(
13939            std::path::Path::new("/bin/same"),
13940            Some(std::path::Path::new("/bin/same")),
13941            unconfirmed.clone(),
13942        );
13943        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13944        assert_eq!(verdict.image, unconfirmed);
13945    }
13946
13947    #[test]
13948    fn reload_verdict_preserves_each_image_unavailability_reason() {
13949        use subc_control::RunningImageUnavailableReason as Reason;
13950
13951        for reason in [
13952            Reason::NotRunning,
13953            Reason::UnsupportedPlatform,
13954            Reason::RunningExecutableUnreadable,
13955            Reason::SpawnedPathUnreadable,
13956            Reason::HashFailed,
13957            Reason::ProcessIdentityUnconfirmed,
13958            Reason::Unknown("future_probe_reason".to_string()),
13959        ] {
13960            let image = subc_control::RunningImageAgreement::Unavailable {
13961                reason: reason.clone(),
13962            };
13963            let verdict = reload_verdict(
13964                std::path::Path::new("/bin/same"),
13965                Some(std::path::Path::new("/bin/same")),
13966                image.clone(),
13967            );
13968            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13969            assert_eq!(verdict.image, image, "{reason:?}");
13970        }
13971    }
13972
13973    #[tokio::test]
13974    async fn malformed_control_bodies_return_invalid_control_body() {
13975        let handler = ControlHandler::default();
13976        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
13977
13978        for (corr, body) in [
13979            (56, br#"{"route_channel":1}"#.as_slice()),
13980            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
13981            (
13982                58,
13983                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
13984            ),
13985        ] {
13986            let request = Frame::build(
13987                FrameType::Request,
13988                control_flags(),
13989                0,
13990                0,
13991                corr,
13992                body.to_vec(),
13993            )
13994            .unwrap();
13995            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13996
13997            assert_eq!(response.len(), 1);
13998            assert_eq!(response[0].header.ty, FrameType::Error);
13999            assert_eq!(response[0].header.corr, corr);
14000            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14001        }
14002    }
14003
14004    #[tokio::test]
14005    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14006        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14007        let registry = Arc::new(Registry::default());
14008        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14009        let router = Router::with_control_handler(Arc::clone(&control));
14010        let connection = router.begin_connection();
14011        let (ctx, mut rx) = route_ctx(connection.id());
14012
14013        router
14014            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14015            .await
14016            .unwrap();
14017        let response = rx.recv().await.unwrap();
14018        let ack = parse_ack(&response);
14019        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14020        let channel = 1;
14021
14022        let goodbye =
14023            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14024        router.route_for_connection(&ctx, goodbye).await.unwrap();
14025        assert!(rx.try_recv().is_err());
14026        assert!(registry.get_module("aft").unwrap().is_none());
14027
14028        router
14029            .route_for_connection(&ctx, channel_request(channel, 13))
14030            .await
14031            .unwrap();
14032        let error_frame = rx.recv().await.unwrap();
14033        assert_eq!(error_frame.header.ty, FrameType::Error);
14034        assert_eq!(error_frame.header.channel, channel);
14035        let captured = crate::router::test_log::captured_logs(&logs);
14036        assert_eq!(
14037            captured
14038                .lines()
14039                .filter(|line| {
14040                    line.contains("module_id=aft")
14041                        && line.contains("reason=explicit_goodbye")
14042                        && line.contains("module registration ended")
14043                })
14044                .count(),
14045            1,
14046            "unexpected GOODBYE registry log: {captured}"
14047        );
14048    }
14049
14050    #[tokio::test]
14051    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14052        let registry = Arc::new(Registry::default());
14053        let handler = ControlHandler::new(registry).with_capability_config(
14054            [("prov".to_string(), true), ("cons".to_string(), true)],
14055            BTreeMap::new(),
14056        );
14057        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14058        register_capability_manifest(
14059            &handler,
14060            &provider_ctx,
14061            &mut provider_rx,
14062            capability_manifest("prov", &["thing/v1"], &[]),
14063            1,
14064        )
14065        .await;
14066        let mut consumer = capability_manifest("cons", &[], &[]);
14067        consumer.capabilities.as_mut().unwrap().requires.push(
14068            subc_protocol::manifest::CapabilityRequirement {
14069                capability: "thing/v1".to_string(),
14070                need: subc_protocol::manifest::CapabilityNeed::Required,
14071            },
14072        );
14073        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14074        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14075        assert_eq!(
14076            handler.capability_evaluator.verdict("cons", "thing/v1"),
14077            Some(CapabilityVerdict::Provided)
14078        );
14079        let (mut client_rx, _) = open_route_for_capability_test(
14080            &handler,
14081            &provider_ctx,
14082            &mut provider_rx,
14083            703,
14084            3,
14085            "prov",
14086            None,
14087        )
14088        .await;
14089        let goodbye =
14090            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14091        handler
14092            .handle_control_frame(&provider_ctx, goodbye)
14093            .await
14094            .unwrap();
14095        assert_eq!(
14096            handler.capability_evaluator.verdict("cons", "thing/v1"),
14097            Some(CapabilityVerdict::NeverProvided)
14098        );
14099        let closed = client_rx
14100            .try_recv()
14101            .expect("GOODBYE pushes route.closed before route GOODBYE");
14102        assert!(
14103            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14104            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14105        );
14106        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14107        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14108    }
14109
14110    struct RouteClosedFixture {
14111        handler: ControlHandler,
14112        #[cfg(unix)]
14113        supervisor: Supervisor,
14114        module_ctx: RouteCtx,
14115        #[cfg(unix)]
14116        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14117        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14118        route: Arc<crate::forwarding::RouteBinding>,
14119    }
14120
14121    async fn route_closed_fixture() -> RouteClosedFixture {
14122        let registry = Arc::new(Registry::default());
14123        let handler = ControlHandler::new(Arc::clone(&registry));
14124        #[cfg(unix)]
14125        let supervisor = Supervisor::new(registry, RestartPolicy::default())
14126            .with_forwarding(Arc::clone(&handler.forwarding));
14127        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14128        register_capability_manifest(
14129            &handler,
14130            &module_ctx,
14131            &mut module_rx,
14132            capability_manifest("prov", &[], &[]),
14133            1,
14134        )
14135        .await;
14136        let (client_rx, bind) = open_route_for_capability_test(
14137            &handler,
14138            &module_ctx,
14139            &mut module_rx,
14140            702,
14141            2,
14142            "prov",
14143            None,
14144        )
14145        .await;
14146        let ModuleControlRequest::RouteBind {
14147            route_channel: channel,
14148            epoch: route_epoch,
14149            ..
14150        } = bind
14151        else {
14152            panic!("expected route.bind");
14153        };
14154        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14155            handler
14156                .forwarding
14157                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14158                .unwrap()
14159        else {
14160            panic!("the acknowledged route must be bound");
14161        };
14162        RouteClosedFixture {
14163            handler,
14164            #[cfg(unix)]
14165            supervisor,
14166            module_ctx,
14167            #[cfg(unix)]
14168            module_rx,
14169            client_rx,
14170            route,
14171        }
14172    }
14173
14174    impl RouteClosedFixture {
14175        #[cfg(unix)]
14176        async fn cut(&mut self) {
14177            let notices = async {
14178                let mut draining = self.module_rx.recv().await.unwrap();
14179                assert!(matches!(
14180                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14181                        &draining.body
14182                    )
14183                    .unwrap(),
14184                    subc_protocol::session::ModuleControlCommand::Draining {
14185                        reason: RouteCloseReason::Restart,
14186                        ..
14187                    }
14188                ));
14189                draining.flushed.take().unwrap().send(()).unwrap();
14190                let mut closing = self.client_rx.recv().await.unwrap();
14191                assert!(matches!(
14192                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14193                    ClientControlPush::RouteClosing {
14194                        reason: RouteCloseReason::Restart,
14195                        ..
14196                    }
14197                ));
14198                closing.flushed.take().unwrap().send(()).unwrap();
14199            };
14200            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14201            result.unwrap();
14202        }
14203
14204        fn disconnect(&mut self) -> ClientControlPush {
14205            self.handler
14206                .cleanup_connection(self.module_ctx.connection_id)
14207                .unwrap();
14208            let closed = self.client_rx.try_recv().unwrap();
14209            assert_eq!(closed.header.ty, FrameType::Push);
14210            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14211            assert!(matches!(
14212                &push,
14213                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14214                    if channels == &[self.route.client_channel]
14215            ));
14216            assert_eq!(
14217                self.client_rx.try_recv().unwrap().header.ty,
14218                FrameType::Goodbye
14219            );
14220            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14221            push
14222        }
14223    }
14224
14225    #[cfg(unix)]
14226    #[tokio::test(start_paused = true)]
14227    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14228        let mut fixture = route_closed_fixture().await;
14229        fixture.cut().await;
14230        assert!(matches!(
14231            fixture.disconnect(),
14232            ClientControlPush::RouteClosed {
14233                reason: RouteCloseReason::Restart,
14234                drained: true,
14235                ..
14236            }
14237        ));
14238    }
14239
14240    #[cfg(unix)]
14241    #[tokio::test(start_paused = true)]
14242    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14243        let mut fixture = route_closed_fixture().await;
14244        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14245        let started = tokio::time::Instant::now();
14246        fixture.cut().await;
14247        assert!(started.elapsed() >= Duration::from_secs(2));
14248        assert_eq!(fixture.route.flow.in_flight(), 1);
14249        assert!(matches!(
14250            fixture.disconnect(),
14251            ClientControlPush::RouteClosed {
14252                reason: RouteCloseReason::Restart,
14253                drained: false,
14254                ..
14255            }
14256        ));
14257    }
14258
14259    #[tokio::test]
14260    async fn module_connection_crash_reports_crash_and_not_drained() {
14261        let mut fixture = route_closed_fixture().await;
14262        // The server calls cleanup_connection on module EOF. Without a planned
14263        // daemon shutdown, that disconnect must remain a crash with no prior
14264        // route.closing notice, even when the route has no in-flight calls.
14265        assert!(matches!(
14266            fixture.disconnect(),
14267            ClientControlPush::RouteClosed {
14268                reason: RouteCloseReason::Crash,
14269                drained: false,
14270                ..
14271            }
14272        ));
14273    }
14274
14275    #[tokio::test]
14276    async fn dropping_router_connection_releases_registration() {
14277        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14278        let registry = Arc::new(Registry::default());
14279        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14280        let router = Router::with_control_handler(Arc::clone(&control));
14281        let connection = router.begin_connection();
14282        let connection_id = connection.id();
14283        let (ctx, mut rx) = route_ctx(connection_id);
14284
14285        router
14286            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
14287            .await
14288            .unwrap();
14289        let response = rx.recv().await.unwrap();
14290        let ack = parse_ack(&response);
14291        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14292        assert!(registry.get_module("aft").unwrap().is_some());
14293
14294        drop(connection);
14295
14296        assert!(registry.get_module("aft").unwrap().is_none());
14297        assert_eq!(registry.active_registration_count().unwrap(), 0);
14298
14299        control
14300            .cleanup_connection(ConnectionId::new(u64::MAX))
14301            .unwrap();
14302        let captured = crate::router::test_log::captured_logs(&logs);
14303        println!("captured registry lifecycle logs:\n{captured}");
14304        let events: Vec<_> = captured
14305            .lines()
14306            .filter(|line| {
14307                line.contains("module registered module_id=aft ")
14308                    || line.contains("module registration ended")
14309            })
14310            .collect();
14311        assert_eq!(
14312            events.len(),
14313            2,
14314            "unexpected registry lifecycle logs: {captured}"
14315        );
14316        assert!(events[0].contains("module registered module_id=aft "));
14317        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
14318        assert!(events[1].contains(&format!(
14319            "module_id=aft connection_id={}",
14320            connection_id.get()
14321        )));
14322        assert!(events[1].contains("reason=connection_closed"));
14323        assert!(events[1].contains("module registration ended"));
14324        assert_eq!(
14325            captured
14326                .lines()
14327                .filter(|line| line.contains("module registered module_id=aft "))
14328                .count(),
14329            1,
14330            "legacy registration admission line must appear once: {captured}"
14331        );
14332    }
14333
14334    #[tokio::test]
14335    async fn hello_registration_keeps_legacy_module_registered_line_once() {
14336        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14337        let registry = Arc::new(Registry::default());
14338        let control = ControlHandler::new(Arc::clone(&registry));
14339        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
14340        hello_via_sink(
14341            &control,
14342            &ctx,
14343            &mut rx,
14344            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
14345        )
14346        .await;
14347
14348        let captured = crate::router::test_log::captured_logs(&logs);
14349        let admissions: Vec<_> = captured
14350            .lines()
14351            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
14352            .collect();
14353        assert_eq!(
14354            admissions.len(),
14355            1,
14356            "legacy admission line must remain exactly once: {captured}"
14357        );
14358        assert!(admissions[0].contains("routable_provider=true"));
14359        assert!(admissions[0].contains("connection_id=777"));
14360    }
14361
14362    fn capability_manifest(
14363        module_id: &str,
14364        provides: &[&str],
14365        must_never_reach: &[&str],
14366    ) -> ModuleManifest {
14367        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
14368        manifest.capabilities = Some(CapabilityDeclarations {
14369            provides: provides
14370                .iter()
14371                .map(|capability| (*capability).to_string())
14372                .collect(),
14373            requires: Vec::new(),
14374            must_never_reach: must_never_reach
14375                .iter()
14376                .map(|capability| (*capability).to_string())
14377                .collect(),
14378        });
14379        manifest
14380    }
14381
14382    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14383        Frame::build(
14384            FrameType::Hello,
14385            control_flags(),
14386            0,
14387            0,
14388            corr,
14389            serde_json::to_vec(&ModuleHelloBody {
14390                protocol_ver: manifest.protocol_ver,
14391                manifest,
14392                control_ops: None,
14393                launch_nonce: None,
14394            })
14395            .expect("capability test HELLO serializes"),
14396        )
14397        .expect("capability test HELLO frame builds")
14398    }
14399
14400    fn catalog_update_with_capabilities_frame(
14401        corr: u64,
14402        capabilities: CapabilityDeclarations,
14403    ) -> Frame {
14404        Frame::build(
14405            FrameType::Request,
14406            control_flags(),
14407            0,
14408            0,
14409            corr,
14410            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14411                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14412                capabilities: Some(capabilities),
14413                ready: None,
14414            })
14415            .expect("capability catalog.update serializes"),
14416        )
14417        .expect("capability catalog.update frame builds")
14418    }
14419
14420    async fn register_capability_manifest(
14421        handler: &ControlHandler,
14422        ctx: &RouteCtx,
14423        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14424        manifest: ModuleManifest,
14425        corr: u64,
14426    ) {
14427        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14428    }
14429
14430    async fn open_route_for_capability_test(
14431        handler: &ControlHandler,
14432        target_ctx: &RouteCtx,
14433        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14434        client_connection_id: u64,
14435        corr: u64,
14436        target_module_id: &str,
14437        consumer_identity: Option<ConsumerIdentity>,
14438    ) -> (
14439        mpsc::Receiver<crate::router::OutboundFrame>,
14440        ModuleControlRequest,
14441    ) {
14442        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14443        let route_handler = handler.clone();
14444        let target_module_id = target_module_id.to_string();
14445        let route_task = tokio::spawn(async move {
14446            route_handler
14447                .handle_control_frame(
14448                    &client_ctx,
14449                    route_open_frame_with_admission_facts(
14450                        corr,
14451                        &target_module_id,
14452                        unique_project_root("admission-facts"),
14453                        consumer_identity,
14454                        None,
14455                    ),
14456                )
14457                .await
14458                .expect("capability test route.open succeeds")
14459        });
14460        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14461            .await
14462            .expect("capability test route.open must reach route.bind")
14463            .expect("target control receiver stays open");
14464        let bind_request: ModuleControlRequest =
14465            serde_json::from_slice(&bind.body).expect("route.bind decodes");
14466        handler
14467            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14468            .await
14469            .expect("capability test route.bind ACK succeeds");
14470        assert!(route_task.await.expect("route.open task joins").is_empty());
14471        let opened = client_rx
14472            .recv()
14473            .await
14474            .expect("successful route.open publishes a response");
14475        assert!(matches!(
14476            serde_json::from_slice::<ClientControlResponse>(&opened.body),
14477            Ok(ClientControlResponse::RouteOpen { .. })
14478        ));
14479        (client_rx, bind_request)
14480    }
14481
14482    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
14483        assert_eq!(frame.header.ty, FrameType::Push);
14484        assert_eq!(frame.header.channel, 0);
14485        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14486            .expect("route.closed control push decodes");
14487        let ClientControlPush::RouteClosed { channels, .. } = &push else {
14488            panic!("expected route.closed");
14489        };
14490        assert_eq!(channels.len(), 1, "exactly one violating route closed");
14491        let channels = channels.clone();
14492        assert_eq!(
14493            push,
14494            ClientControlPush::RouteClosed {
14495                module_id: target_module_id.to_string(),
14496                channels,
14497                reason: RouteCloseReason::CapabilityDenied,
14498                drained: false,
14499                abandoned: 0,
14500                excluded_subscriptions: 0,
14501                terminal: Some(false),
14502            }
14503        );
14504    }
14505
14506    #[tokio::test]
14507    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14508        let registry = Arc::new(Registry::default());
14509        let forwarding = Arc::new(ForwardingTable::default());
14510        let supervisor = SupervisorHandle::new();
14511        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14512        let handler =
14513            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14514                .with_supervisor(supervisor);
14515        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14516        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14517        register_capability_manifest(
14518            &handler,
14519            &target_ctx,
14520            &mut target_rx,
14521            capability_manifest("target", &["credentials-provider/v1"], &[]),
14522            1,
14523        )
14524        .await;
14525        register_capability_manifest(
14526            &handler,
14527            &opener_ctx,
14528            &mut opener_rx,
14529            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14530            2,
14531        )
14532        .await;
14533
14534        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
14535        let replies = handler
14536            .handle_control_frame(
14537                &client_ctx,
14538                route_open_frame_with_admission_facts(
14539                    3,
14540                    "target",
14541                    unique_project_root("admission-facts"),
14542                    Some(ConsumerIdentity {
14543                        module_id: "opener".to_string(),
14544                        launch_nonce: "opener-nonce".to_string(),
14545                    }),
14546                    None,
14547                ),
14548            )
14549            .await
14550            .expect("denied route.open returns a typed frame");
14551        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14552        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14553        assert!(
14554            target_rx.try_recv().is_err(),
14555            "forbidden route.open must not relay route.bind"
14556        );
14557    }
14558
14559    #[tokio::test]
14560    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
14561        let registry = Arc::new(Registry::default());
14562        let forwarding = Arc::new(ForwardingTable::default());
14563        let supervisor = SupervisorHandle::new();
14564        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14565        let handler =
14566            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14567                .with_supervisor(supervisor);
14568        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
14569        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
14570        register_capability_manifest(
14571            &handler,
14572            &target_ctx,
14573            &mut target_rx,
14574            capability_manifest("target", &["credentials-provider/v1"], &[]),
14575            1,
14576        )
14577        .await;
14578        register_capability_manifest(
14579            &handler,
14580            &old_opener_ctx,
14581            &mut old_opener_rx,
14582            capability_manifest("opener", &[], &[]),
14583            2,
14584        )
14585        .await;
14586        let (mut client_rx, _) = open_route_for_capability_test(
14587            &handler,
14588            &target_ctx,
14589            &mut target_rx,
14590            712,
14591            3,
14592            "target",
14593            Some(ConsumerIdentity {
14594                module_id: "opener".to_string(),
14595                launch_nonce: "opener-nonce".to_string(),
14596            }),
14597        )
14598        .await;
14599        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14600
14601        handler
14602            .cleanup_connection(old_opener_ctx.connection_id)
14603            .expect("old opener registration cleans up");
14604        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
14605        register_capability_manifest(
14606            &handler,
14607            &new_opener_ctx,
14608            &mut new_opener_rx,
14609            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14610            4,
14611        )
14612        .await;
14613
14614        assert_capability_denied_push(
14615            client_rx
14616                .try_recv()
14617                .expect("HELLO deny addition must emit route.closed")
14618                .frame,
14619            "target",
14620        );
14621        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14622        assert!(matches!(
14623            target_rx.try_recv(),
14624            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14625        ));
14626    }
14627
14628    #[tokio::test]
14629    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
14630        let registry = Arc::new(Registry::default());
14631        let forwarding = Arc::new(ForwardingTable::default());
14632        let supervisor = SupervisorHandle::new();
14633        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14634        let handler =
14635            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14636                .with_supervisor(supervisor);
14637        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
14638        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
14639        register_capability_manifest(
14640            &handler,
14641            &target_ctx,
14642            &mut target_rx,
14643            capability_manifest("target", &[], &[]),
14644            1,
14645        )
14646        .await;
14647        register_capability_manifest(
14648            &handler,
14649            &opener_ctx,
14650            &mut opener_rx,
14651            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14652            2,
14653        )
14654        .await;
14655        let (mut client_rx, _) = open_route_for_capability_test(
14656            &handler,
14657            &target_ctx,
14658            &mut target_rx,
14659            722,
14660            3,
14661            "target",
14662            Some(ConsumerIdentity {
14663                module_id: "opener".to_string(),
14664                launch_nonce: "opener-nonce".to_string(),
14665            }),
14666        )
14667        .await;
14668        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14669
14670        let replies = handler
14671            .handle_control_frame(
14672                &target_ctx,
14673                catalog_update_with_capabilities_frame(
14674                    4,
14675                    CapabilityDeclarations {
14676                        provides: vec!["credentials-provider/v1".to_string()],
14677                        requires: Vec::new(),
14678                        must_never_reach: Vec::new(),
14679                    },
14680                ),
14681            )
14682            .await
14683            .expect("claim catalog.update succeeds");
14684        assert!(matches!(
14685            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
14686            Ok(ModuleControlResponseToModule::CatalogUpdate {})
14687        ));
14688        assert_capability_denied_push(
14689            client_rx
14690                .try_recv()
14691                .expect("claim addition must emit route.closed")
14692                .frame,
14693            "target",
14694        );
14695        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14696        assert!(matches!(
14697            target_rx.try_recv(),
14698            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14699        ));
14700    }
14701
14702    #[tokio::test]
14703    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
14704        let registry = Arc::new(Registry::default());
14705        let forwarding = Arc::new(ForwardingTable::default());
14706        let supervisor = SupervisorHandle::new();
14707        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14708        let handler =
14709            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14710                .with_supervisor(supervisor);
14711        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
14712        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
14713        register_capability_manifest(
14714            &handler,
14715            &target_ctx,
14716            &mut target_rx,
14717            capability_manifest("target", &["credentials-provider/v1"], &[]),
14718            1,
14719        )
14720        .await;
14721        register_capability_manifest(
14722            &handler,
14723            &opener_ctx,
14724            &mut opener_rx,
14725            capability_manifest("opener", &[], &[]),
14726            2,
14727        )
14728        .await;
14729        let (mut client_rx, _) = open_route_for_capability_test(
14730            &handler,
14731            &target_ctx,
14732            &mut target_rx,
14733            732,
14734            3,
14735            "target",
14736            Some(ConsumerIdentity {
14737                module_id: "opener".to_string(),
14738                launch_nonce: "opener-nonce".to_string(),
14739            }),
14740        )
14741        .await;
14742        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14743
14744        handler
14745            .handle_control_frame(
14746                &target_ctx,
14747                catalog_update_with_capabilities_frame(
14748                    4,
14749                    CapabilityDeclarations {
14750                        provides: Vec::new(),
14751                        requires: Vec::new(),
14752                        must_never_reach: Vec::new(),
14753                    },
14754                ),
14755            )
14756            .await
14757            .expect("claim removal catalog.update succeeds");
14758        assert_eq!(
14759            forwarding.active_binding_count().unwrap(),
14760            1,
14761            "removing an attested target claim must leave the route census unchanged"
14762        );
14763        assert!(
14764            client_rx.try_recv().is_err(),
14765            "claim removal must not emit route.closed capability_denied"
14766        );
14767        assert!(
14768            target_rx.try_recv().is_err(),
14769            "claim removal must not send the target a route GOODBYE"
14770        );
14771    }
14772
14773    /// A direct client may open a route to a denied capability provider; this
14774    /// policy applies only to attested supervised module origins, not to direct clients.
14775    #[tokio::test]
14776    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
14777        let registry = Arc::new(Registry::default());
14778        let forwarding = Arc::new(ForwardingTable::default());
14779        let supervisor = SupervisorHandle::new();
14780        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14781        let handler =
14782            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14783                .with_supervisor(supervisor);
14784        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
14785        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
14786        register_capability_manifest(
14787            &handler,
14788            &target_ctx,
14789            &mut target_rx,
14790            capability_manifest("target", &["credentials-provider/v1"], &[]),
14791            1,
14792        )
14793        .await;
14794        register_capability_manifest(
14795            &handler,
14796            &opener_ctx,
14797            &mut opener_rx,
14798            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14799            2,
14800        )
14801        .await;
14802
14803        let (_client_rx, bind) = open_route_for_capability_test(
14804            &handler,
14805            &target_ctx,
14806            &mut target_rx,
14807            742,
14808            3,
14809            "target",
14810            None,
14811        )
14812        .await;
14813        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14814            panic!("direct scope-honesty route must bind");
14815        };
14816        assert_eq!(principal, Some(Principal::Direct));
14817        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14818    }
14819
14820    /// A module that denies a capability receives no self-route exemption when it
14821    /// also attestedly provides that capability.
14822    #[tokio::test]
14823    async fn must_never_reach_self_route_is_capability_forbidden() {
14824        let registry = Arc::new(Registry::default());
14825        let forwarding = Arc::new(ForwardingTable::default());
14826        let supervisor = SupervisorHandle::new();
14827        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
14828        let handler =
14829            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14830                .with_supervisor(supervisor);
14831        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
14832        register_capability_manifest(
14833            &handler,
14834            &self_ctx,
14835            &mut self_rx,
14836            capability_manifest(
14837                "self-provider",
14838                &["credentials-provider/v1"],
14839                &["credentials-provider/v1"],
14840            ),
14841            1,
14842        )
14843        .await;
14844
14845        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
14846        let replies = handler
14847            .handle_control_frame(
14848                &client_ctx,
14849                route_open_frame_with_admission_facts(
14850                    2,
14851                    "self-provider",
14852                    unique_project_root("admission-facts"),
14853                    Some(ConsumerIdentity {
14854                        module_id: "self-provider".to_string(),
14855                        launch_nonce: "self-nonce".to_string(),
14856                    }),
14857                    None,
14858                ),
14859            )
14860            .await
14861            .expect("self-route refusal returns a typed frame");
14862        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14863        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14864        assert!(
14865            self_rx.try_recv().is_err(),
14866            "self denial must not relay route.bind"
14867        );
14868    }
14869
14870    #[test]
14871    fn unsupported_channel_zero_frame_returns_error() {
14872        let handler = ControlHandler::default();
14873        let request = Frame::build(
14874            FrameType::Request,
14875            control_flags(),
14876            0,
14877            0,
14878            21,
14879            b"opaque".to_vec(),
14880        )
14881        .unwrap();
14882
14883        let response = handler
14884            .handle_control(ConnectionId::new(1), request)
14885            .unwrap();
14886
14887        assert_eq!(response[0].header.ty, FrameType::Error);
14888        assert_eq!(
14889            parse_error(&response[0])["code"],
14890            "unsupported_control_frame"
14891        );
14892    }
14893
14894    /// Blue/green swap at the control-plane boundary. The supervisor that opens
14895    /// a swap is not wired yet, so the candidate is registered here directly
14896    /// into the registry and forwarding candidate slots, the way the swap's
14897    /// HELLO admission will.
14898    mod swap {
14899        use super::*;
14900
14901        const INCUMBENT: ConnectionId = ConnectionId::new(30);
14902        const CANDIDATE: ConnectionId = ConnectionId::new(40);
14903
14904        struct Swap {
14905            registry: Arc<Registry>,
14906            forwarding: Arc<ForwardingTable>,
14907            handler: ControlHandler,
14908            incumbent_ctx: RouteCtx,
14909            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14910            candidate_ctx: RouteCtx,
14911            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14912        }
14913
14914        async fn swap_with_incumbent() -> Swap {
14915            let registry = Arc::new(Registry::default());
14916            let forwarding = Arc::new(ForwardingTable::default());
14917            let handler =
14918                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14919            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14920            hello_via_sink(
14921                &handler,
14922                &incumbent_ctx,
14923                &mut incumbent_rx,
14924                hello_frame("aft", PROTOCOL_VERSION, 7),
14925            )
14926            .await;
14927            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14928            Swap {
14929                registry,
14930                forwarding,
14931                handler,
14932                incumbent_ctx,
14933                incumbent_rx,
14934                candidate_ctx,
14935                candidate_rx,
14936            }
14937        }
14938
14939        fn register_candidate(swap: &Swap, ready: Option<bool>) {
14940            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14941            candidate_manifest.ready = ready;
14942            let registration = swap
14943                .registry
14944                .register_candidate_with_control_ops(
14945                    candidate_manifest,
14946                    PROTOCOL_VERSION,
14947                    CANDIDATE,
14948                    module_baseline_control_ops(),
14949                )
14950                .unwrap();
14951            swap.forwarding
14952                .register_candidate_module_connection(
14953                    CANDIDATE,
14954                    "aft".to_string(),
14955                    PROTOCOL_VERSION,
14956                    manifest_concurrency(&registration.manifest),
14957                    swap.candidate_ctx.egress.clone(),
14958                )
14959                .unwrap();
14960        }
14961
14962        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
14963            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
14964            swap.registry.promote_candidate("aft").unwrap().unwrap();
14965            cutover.incumbent.unwrap()
14966        }
14967
14968        /// A swap candidate's HELLO reserves its candidate registry entry
14969        /// before installing its candidate forwarding endpoint. The swap
14970        /// supervisor promotes a ready candidate as soon as it reads it as
14971        /// registered, and promotion cuts forwarding over to the candidate
14972        /// endpoint, so the lookup the supervisor reads
14973        /// (`Registry::published_registration`) must not return the candidate
14974        /// until that endpoint exists. Otherwise the cutover finds no candidate
14975        /// endpoint and the swap fails as if the candidate had disconnected.
14976        #[tokio::test]
14977        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
14978            let mut swap = swap_with_incumbent().await;
14979            let hold = Arc::new(std::sync::Barrier::new(2));
14980            let handler = swap
14981                .handler
14982                .clone()
14983                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14984            handler
14985                .supervisor
14986                .open_swap("aft", "candidate-nonce".to_string());
14987            let hello = {
14988                let handler = handler.clone();
14989                let candidate_ctx = swap.candidate_ctx.clone();
14990                std::thread::spawn(move || {
14991                    handler
14992                        .handle_hello(
14993                            candidate_ctx.connection_id,
14994                            Some(candidate_ctx.egress.clone()),
14995                            hello_frame_with_nonce(
14996                                "aft",
14997                                PROTOCOL_VERSION,
14998                                8,
14999                                Some("candidate-nonce"),
15000                            ),
15001                        )
15002                        .unwrap()
15003                })
15004            };
15005
15006            hold.wait();
15007            // Parked: the candidate slot is claimed but not published, and
15008            // forwarding has no candidate endpoint.
15009            assert!(swap
15010                .registry
15011                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15012                .unwrap()
15013                .is_some());
15014            assert!(swap
15015                .registry
15016                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15017                .unwrap()
15018                .is_none());
15019            assert!(swap
15020                .forwarding
15021                .module_endpoint_for_connection(CANDIDATE)
15022                .unwrap()
15023                .is_none());
15024
15025            hold.wait();
15026            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15027            assert_eq!(
15028                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15029                FrameType::HelloAck
15030            );
15031            let published = swap
15032                .registry
15033                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15034                .unwrap()
15035                .expect("candidate is published once its HELLO completes");
15036            assert_eq!(published.connection_id, CANDIDATE);
15037            assert!(swap
15038                .forwarding
15039                .module_endpoint_for_connection(CANDIDATE)
15040                .unwrap()
15041                .is_some());
15042        }
15043
15044        fn keyed_total(counters: &Value, key: &str) -> u64 {
15045            counters[key]
15046                .as_object()
15047                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15048                .unwrap_or(0)
15049        }
15050
15051        #[tokio::test]
15052        async fn replacement_logs_old_end_and_new_admission_once() {
15053            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15054            let mut swap = swap_with_incumbent().await;
15055            swap.handler
15056                .supervisor
15057                .open_swap("aft", "candidate-nonce".to_string());
15058            hello_via_sink(
15059                &swap.handler,
15060                &swap.candidate_ctx,
15061                &mut swap.candidate_rx,
15062                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15063            )
15064            .await;
15065            cutover(&swap);
15066            swap.handler.cleanup_connection(INCUMBENT).unwrap();
15067
15068            let captured = crate::router::test_log::captured_logs(&logs);
15069            println!("captured replacement registry logs:\n{captured}");
15070            let new_admissions: Vec<_> = captured
15071                .lines()
15072                .filter(|line| {
15073                    line.contains("connection_id=40")
15074                        && line.contains("swap candidate registered; not routable until cutover")
15075                })
15076                .collect();
15077            assert_eq!(
15078                new_admissions.len(),
15079                1,
15080                "unexpected admission logs: {captured}"
15081            );
15082            assert!(new_admissions[0].contains("module_id=aft"));
15083            assert!(new_admissions[0].contains("connection_id=40"));
15084            assert!(
15085                new_admissions[0].contains("swap candidate registered; not routable until cutover")
15086            );
15087            assert!(new_admissions[0].contains("ready=true"));
15088
15089            let old_admissions: Vec<_> = captured
15090                .lines()
15091                .filter(|line| {
15092                    line.contains("module registered module_id=aft ")
15093                        && line.contains("connection_id=30")
15094                })
15095                .collect();
15096            assert_eq!(
15097                old_admissions.len(),
15098                1,
15099                "unexpected incumbent admission: {captured}"
15100            );
15101
15102            let old_ends: Vec<_> = captured
15103                .lines()
15104                .filter(|line| {
15105                    line.contains("connection_id=30") && line.contains("module registration ended")
15106                })
15107                .collect();
15108            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15109            assert!(old_ends[0].contains("module_id=aft"));
15110            assert!(old_ends[0].contains("connection_id=30"));
15111            assert!(old_ends[0].contains("reason=replaced"));
15112            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15113            assert_eq!(
15114                captured
15115                    .lines()
15116                    .filter(|line| line.contains("module registration promoted"))
15117                    .count(),
15118                1,
15119                "unexpected promotion logs: {captured}"
15120            );
15121        }
15122
15123        /// An ack from the incumbent for a bind it was sent before cutover,
15124        /// arriving before the incumbent is drained. The incumbent is the live
15125        /// connection carrying every other client's routes, so the ack must
15126        /// not end it: the waiting client is told to retry, the reservation is
15127        /// given back, and the incumbent is told to drop just that binding.
15128        #[tokio::test]
15129        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15130            let mut swap = swap_with_incumbent().await;
15131            let handler = swap.handler.clone();
15132
15133            // A co-tenant route, bound on the incumbent before the swap.
15134            let cotenant = ConnectionId::new(31);
15135            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15136            let (cotenant_task, cotenant_bind) = relay_route_open(
15137                &handler,
15138                cotenant,
15139                &cotenant_ctx.egress,
15140                &mut swap.incumbent_rx,
15141                100,
15142                "aft",
15143                "swap-cotenant",
15144            )
15145            .await;
15146            handler
15147                .handle_control_frame(
15148                    &swap.incumbent_ctx,
15149                    route_bind_ack(cotenant_bind.header.corr),
15150                )
15151                .await
15152                .unwrap();
15153            assert!(cotenant_task.await.unwrap().is_empty());
15154            let (cotenant_channel, cotenant_epoch) =
15155                published_route(&cotenant_rx.recv().await.unwrap());
15156
15157            // A second route.open, relayed to the incumbent and not yet acked.
15158            let caller = ConnectionId::new(32);
15159            let (caller_ctx, mut caller_rx) = route_ctx(caller);
15160            let (caller_task, caller_bind) = relay_route_open(
15161                &handler,
15162                caller,
15163                &caller_ctx.egress,
15164                &mut swap.incumbent_rx,
15165                101,
15166                "aft",
15167                "swap-caller",
15168            )
15169            .await;
15170            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15171
15172            register_candidate(&swap, None);
15173            cutover(&swap);
15174
15175            // The incumbent acks after promotion and before any drain.
15176            let ack = handler
15177                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15178                .await;
15179            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15180            if module_loop_error.is_some() {
15181                // What the connection loop does with an untranslated router
15182                // error: end the connection, releasing every route on it.
15183                handler.cleanup_connection(INCUMBENT).unwrap();
15184            }
15185
15186            // 1. The incumbent's other routes survive.
15187            assert!(
15188                cotenant_rx.try_recv().is_err(),
15189                "the co-tenant route on the incumbent was torn down by one late ack: \
15190                 {module_loop_error:?}"
15191            );
15192            assert!(matches!(
15193                swap.forwarding
15194                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15195                    .unwrap(),
15196                DataRoute::Client(DataRouteState::Bound(_))
15197            ));
15198            assert_eq!(module_loop_error, None);
15199            assert!(swap
15200                .registry
15201                .get_module_by_connection(INCUMBENT)
15202                .unwrap()
15203                .is_some());
15204
15205            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
15206            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15207                .await
15208                .expect("the incumbent is told to drop the abandoned binding")
15209                .unwrap()
15210                .frame;
15211            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15212            assert_eq!(goodbye.header.channel, abandoned_channel);
15213            assert_eq!(goodbye.header.epoch, abandoned_epoch);
15214            assert!(swap.incumbent_rx.try_recv().is_err());
15215
15216            // 3. The waiting client gets a retryable refusal and no route.
15217            let response = caller_task.await.unwrap();
15218            assert_eq!(response.len(), 1);
15219            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15220            assert!(caller_rx.try_recv().is_err());
15221
15222            // 4. The reservation pair is given back, and the pending bind
15223            //    settled exactly once: one accepted open (the co-tenant) and one
15224            //    refused open (the caller), nothing counted twice.
15225            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15226            let counters = handler.counters().snapshot();
15227            assert_eq!(
15228                keyed_total(&counters, "route_open_accepted_by_principal"),
15229                1
15230            );
15231            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15232            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15233        }
15234
15235        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
15236        /// id would resolve to the promoted candidate and every new route.open
15237        /// would be refused as reloading, leaving neither process routable.
15238        #[tokio::test]
15239        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15240            let mut swap = swap_with_incumbent().await;
15241            register_candidate(&swap, None);
15242            let incumbent = cutover(&swap);
15243            swap.forwarding
15244                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15245                .unwrap()
15246                .expect("the incumbent is still registered");
15247
15248            let client = ConnectionId::new(33);
15249            let (client_ctx, mut client_rx) = route_ctx(client);
15250            let route_handler = swap.handler.clone();
15251            let open_ctx = RouteCtx {
15252                connection_id: client,
15253                egress: client_ctx.egress.clone(),
15254            };
15255            let mut route_task = tokio::spawn(async move {
15256                route_handler
15257                    .handle_control_frame(
15258                        &open_ctx,
15259                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
15260                    )
15261                    .await
15262                    .unwrap()
15263            });
15264            let bind = tokio::select! {
15265                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
15266                response = &mut route_task => {
15267                    let response = response.unwrap();
15268                    panic!(
15269                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
15270                        parse_error(&response[0])["code"]
15271                    );
15272                }
15273            };
15274            swap.handler
15275                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
15276                .await
15277                .unwrap();
15278            assert!(route_task.await.unwrap().is_empty());
15279            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
15280            match swap
15281                .forwarding
15282                .lookup_data_route(client, channel, epoch)
15283                .unwrap()
15284            {
15285                DataRoute::Client(DataRouteState::Bound(route)) => {
15286                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
15287                }
15288                other => panic!("expected a bound route on the candidate, got {other:?}"),
15289            }
15290            assert!(swap.incumbent_rx.try_recv().is_err());
15291        }
15292
15293        /// A candidate declares itself ready with `catalog.update` on its own
15294        /// connection. If the connection-keyed registry lookups searched only the
15295        /// active slot, this would answer `not_registered` and the candidate
15296        /// would never become ready.
15297        #[tokio::test]
15298        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
15299            let swap = swap_with_incumbent().await;
15300            register_candidate(&swap, Some(false));
15301            let update = Frame::build(
15302                FrameType::Request,
15303                control_flags(),
15304                0,
15305                0,
15306                55,
15307                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15308                    provides: manifest("aft", PROTOCOL_VERSION).provides,
15309                    capabilities: None,
15310                    ready: Some(true),
15311                })
15312                .unwrap(),
15313            )
15314            .unwrap();
15315
15316            let replies = swap
15317                .handler
15318                .handle_control_frame(&swap.candidate_ctx, update)
15319                .await
15320                .unwrap();
15321
15322            assert_eq!(replies.len(), 1);
15323            assert_eq!(
15324                replies[0].header.ty,
15325                FrameType::Response,
15326                "candidate catalog.update was refused: {:?}",
15327                serde_json::from_slice::<Value>(&replies[0].body).ok()
15328            );
15329            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
15330            assert_eq!(
15331                swap.registry
15332                    .get_module("aft")
15333                    .unwrap()
15334                    .unwrap()
15335                    .connection_id,
15336                INCUMBENT
15337            );
15338        }
15339    }
15340
15341    /// The HELLO gate while the supervisor has a swap open: only the nonce it
15342    /// minted for the candidate admits a second process, into the candidate
15343    /// slot, and that check runs ahead of the reserved-module gate.
15344    mod swap_admission {
15345        use super::*;
15346
15347        const INCUMBENT_NONCE: &str = "incumbent-nonce";
15348        const CANDIDATE_NONCE: &str = "candidate-nonce";
15349
15350        fn handler_with_incumbent(
15351            module_id: &str,
15352            reserved: bool,
15353        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
15354            let registry = Arc::new(Registry::default());
15355            let supervisor = SupervisorHandle::new();
15356            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
15357            if reserved {
15358                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
15359            }
15360            let handler =
15361                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
15362            let incumbent = handler
15363                .handle_control(
15364                    ConnectionId::new(1),
15365                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
15366                )
15367                .unwrap();
15368            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
15369            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
15370            (registry, supervisor, handler)
15371        }
15372
15373        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
15374        /// admits every nonce, so while a swap is open the swap gate is the only
15375        /// thing between a key-holder and the candidate slot. A nonce the
15376        /// supervisor did not mint, or none at all, is refused, and neither the
15377        /// incumbent's registration nor the candidate slot moves.
15378        #[test]
15379        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15380            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15381
15382            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15383                let replies = handler
15384                    .handle_control(
15385                        ConnectionId::new(connection),
15386                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15387                    )
15388                    .unwrap();
15389                assert_eq!(replies[0].header.ty, FrameType::Error);
15390                assert_eq!(
15391                    parse_error(&replies[0])["code"],
15392                    "swap_token_invalid",
15393                    "nonce {nonce:?}"
15394                );
15395            }
15396            assert!(registry.get_candidate("aft").unwrap().is_none());
15397            assert_eq!(
15398                registry.get_module("aft").unwrap().unwrap().connection_id,
15399                ConnectionId::new(1)
15400            );
15401
15402            // Control: the minted token is admitted, into the candidate slot,
15403            // and only once.
15404            let admitted = handler
15405                .handle_control(
15406                    ConnectionId::new(4),
15407                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15408                )
15409                .unwrap();
15410            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15411            assert_eq!(
15412                registry
15413                    .get_candidate("aft")
15414                    .unwrap()
15415                    .unwrap()
15416                    .connection_id,
15417                ConnectionId::new(4)
15418            );
15419            assert_eq!(
15420                registry.get_module("aft").unwrap().unwrap().connection_id,
15421                ConnectionId::new(1),
15422                "the candidate must not take the active slot"
15423            );
15424            let replayed = handler
15425                .handle_control(
15426                    ConnectionId::new(5),
15427                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15428                )
15429                .unwrap();
15430            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15431
15432            // The case only this gate covers: the incumbent has died mid-swap,
15433            // so its duplicate refusal is gone too, and without the gate a
15434            // key-holder would take the id's ACTIVE slot.
15435            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15436            let squatter = handler
15437                .handle_control(
15438                    ConnectionId::new(6),
15439                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15440                )
15441                .unwrap();
15442            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15443            assert!(
15444                registry.get_module("aft").unwrap().is_none(),
15445                "a squatter took the active slot of an id being swapped"
15446            );
15447        }
15448
15449        /// Design mutation arm (iii). A reserved module's candidate presents a
15450        /// nonce the reserved gate has never seen (that gate holds the
15451        /// incumbent's), so the swap gate must run first or the candidate is
15452        /// refused `reserved_module` and a reserved module can never be swapped.
15453        #[test]
15454        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15455            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15456
15457            let replies = handler
15458                .handle_control(
15459                    ConnectionId::new(2),
15460                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15461                )
15462                .unwrap();
15463
15464            assert_eq!(
15465                replies[0].header.ty,
15466                FrameType::HelloAck,
15467                "reserved candidate refused: {:?}",
15468                serde_json::from_slice::<Value>(&replies[0].body).ok()
15469            );
15470            assert_eq!(
15471                registry
15472                    .get_candidate("vault")
15473                    .unwrap()
15474                    .unwrap()
15475                    .connection_id,
15476                ConnectionId::new(2)
15477            );
15478        }
15479
15480        /// With no swap open the gate is inert: the incumbent's reserved gate
15481        /// and duplicate refusal behave exactly as before.
15482        #[test]
15483        fn without_an_open_swap_the_ordinary_gates_decide() {
15484            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
15485            supervisor.close_swap("vault");
15486
15487            let candidate = handler
15488                .handle_control(
15489                    ConnectionId::new(2),
15490                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15491                )
15492                .unwrap();
15493            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
15494            let duplicate = handler
15495                .handle_control(
15496                    ConnectionId::new(3),
15497                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15498                )
15499                .unwrap();
15500            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15501            assert!(registry.get_candidate("vault").unwrap().is_none());
15502        }
15503    }
15504
15505    /// `scope.sync` and `scope.describe` through the real control handler: who
15506    /// may sync is decided by the registration and launch nonce of the module
15507    /// connection, never by the request body.
15508    mod scopes {
15509        mod idle_tests {
15510            use super::*;
15511            include!("scope_idle_tests.rs");
15512        }
15513        mod route_open_timings {
15514            use super::*;
15515            include!("route_open_timing_tests.rs");
15516        }
15517        mod activity {
15518            include!("activity_scope_tests.rs");
15519        }
15520        mod delta {
15521            include!("scope_handler_delta_tests.rs");
15522        }
15523        mod call_modes {
15524            include!("scope_call_mode_tests.rs");
15525        }
15526        #[tokio::test]
15527        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
15528            let rig = rig().await;
15529            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
15530                "s".into(),
15531                1,
15532                "tool".into(),
15533            ))
15534            .unwrap();
15535            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
15536            let (direct, _rx) = wide_ctx(999);
15537            for (ctx, expected) in [
15538                (&rig.owner, FrameType::Response),
15539                (&direct, FrameType::Error),
15540            ] {
15541                let frame =
15542                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
15543                        .unwrap();
15544                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
15545                assert_eq!(replies[0].header.ty, expected);
15546                if expected == FrameType::Response {
15547                    let reply: subc_control::ScopeServedByReply =
15548                        serde_json::from_slice(&replies[0].body).unwrap();
15549                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
15550                } else {
15551                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
15552                }
15553            }
15554            let reply = call(
15555                &rig.handler,
15556                &rig.owner,
15557                &ModuleControlRequestFromModule::ScopeDescribe {
15558                    owner: Principal::Reserved {
15559                        module_id: OWNER.into(),
15560                    },
15561                    scope_ref: "s".into(),
15562                },
15563            )
15564            .await;
15565            assert!(matches!(
15566                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
15567                ModuleControlResponseToModule::ScopeDescribe { .. }
15568            ));
15569        }
15570
15571        use subc_protocol::scope::{
15572            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
15573            ScopeStatus,
15574        };
15575
15576        use super::*;
15577
15578        const OWNER: &str = "prefrontal-core";
15579
15580        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
15581            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
15582        }
15583
15584        async fn call(
15585            handler: &ControlHandler,
15586            ctx: &RouteCtx,
15587            request: &ModuleControlRequestFromModule,
15588        ) -> Frame {
15589            let body = serde_json::to_vec(request).unwrap();
15590            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
15591            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
15592            assert_eq!(replies.len(), 1, "{replies:?}");
15593            replies.pop().unwrap()
15594        }
15595
15596        async fn sync(
15597            handler: &ControlHandler,
15598            ctx: &RouteCtx,
15599            generation: u64,
15600            scopes: Vec<ScopeRecord>,
15601        ) -> Result<ModuleControlResponseToModule, String> {
15602            let reply = call(
15603                handler,
15604                ctx,
15605                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
15606            )
15607            .await;
15608            match reply.header.ty {
15609                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
15610                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
15611            }
15612        }
15613
15614        async fn describe(
15615            handler: &ControlHandler,
15616            ctx: &RouteCtx,
15617            owner: &str,
15618            scope_ref: &str,
15619        ) -> ModuleControlResponseToModule {
15620            let reply = call(
15621                handler,
15622                ctx,
15623                &ModuleControlRequestFromModule::ScopeDescribe {
15624                    owner: Principal::Reserved {
15625                        module_id: owner.to_string(),
15626                    },
15627                    scope_ref: scope_ref.to_string(),
15628                },
15629            )
15630            .await;
15631            assert_eq!(
15632                reply.header.ty,
15633                FrameType::Response,
15634                "{:?}",
15635                parse_error(&reply)
15636            );
15637            serde_json::from_slice(&reply.body).unwrap()
15638        }
15639
15640        #[tokio::test]
15641        async fn scope_describe_never_echoes_bind_proof_or_hash() {
15642            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15643            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15644            let mut rig = rig().await;
15645            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15646                .await;
15647            let _bound = rig
15648                .bound(
15649                    Some(AFT),
15650                    PLEXUS,
15651                    Some(
15652                        rig.selector("s", Some(1))
15653                            .with_bind_proof(Some(PROOF.into())),
15654                    ),
15655                )
15656                .await;
15657            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
15658            let output = serde_json::to_string(&reply).unwrap();
15659            assert!(output.contains("\"status\":\"live\""));
15660            assert!(!output.contains(PROOF));
15661            assert!(!output.contains(HASH));
15662        }
15663
15664        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
15665        async fn module(
15666            handler: &ControlHandler,
15667            connection: u64,
15668            module_id: &str,
15669            nonce: Option<&str>,
15670        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15671            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
15672            hello_via_sink(
15673                handler,
15674                &ctx,
15675                &mut rx,
15676                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
15677            )
15678            .await;
15679            (ctx, rx)
15680        }
15681
15682        /// `direct` and every other client connection has no registration, so
15683        /// it can neither sync nor own a scope.
15684        #[tokio::test]
15685        async fn a_client_connection_cannot_sync_or_describe() {
15686            let handler = ControlHandler::new(Arc::new(Registry::default()));
15687            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
15688            for request in [
15689                ModuleControlRequestFromModule::ScopeSync {
15690                    generation: 1,
15691                    scopes: vec![head("s", 1)],
15692                },
15693                ModuleControlRequestFromModule::ScopeDescribe {
15694                    owner: Principal::Direct,
15695                    scope_ref: "s".to_string(),
15696                },
15697            ] {
15698                let reply = call(&handler, &ctx, &request).await;
15699                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
15700            }
15701            assert!(
15702                !handler
15703                    .scopes
15704                    .read()
15705                    .unwrap()
15706                    .describe(
15707                        &Principal::Reserved {
15708                            module_id: OWNER.to_string()
15709                        },
15710                        "s"
15711                    )
15712                    .owner_synced
15713            );
15714        }
15715
15716        /// A module the supervisor did not spawn registers without a launch
15717        /// nonce, so it is never an owner's current launch.
15718        #[tokio::test]
15719        async fn a_module_without_a_supervised_launch_cannot_sync() {
15720            let handler = ControlHandler::new(Arc::new(Registry::default()));
15721            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
15722            assert_eq!(
15723                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
15724                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15725            );
15726        }
15727
15728        #[tokio::test]
15729        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
15730            let supervisor = SupervisorHandle::new();
15731            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15732            let handler = ControlHandler::new(Arc::new(Registry::default()))
15733                .with_supervisor(supervisor.clone());
15734            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15735            sync(&handler, &incumbent, 1, vec![head("s", 1)])
15736                .await
15737                .expect("the current launch syncs");
15738
15739            // A swap candidate registers with the swap token and is refused
15740            // while the incumbent keeps syncing.
15741            supervisor.open_swap(OWNER, "n2".to_string());
15742            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
15743            assert_eq!(
15744                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15745                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15746            );
15747            sync(&handler, &incumbent, 2, vec![head("s", 1)])
15748                .await
15749                .expect("the serving owner syncs during the swap");
15750
15751            // The swap fails and is rolled back. The candidate never held sync
15752            // authority, and still cannot sync.
15753            supervisor.close_swap(OWNER);
15754            assert_eq!(
15755                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15756                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15757            );
15758            sync(&handler, &incumbent, 3, vec![head("s", 1)])
15759                .await
15760                .expect("the serving owner syncs after the rollback");
15761            handler.cleanup_connection(candidate.connection_id).unwrap();
15762
15763            // A swap that cuts over. Promotion records the candidate's nonce as
15764            // the module's spawn nonce, which is what `set_spawn_nonce` does
15765            // here; the promoted connection then takes authority at any
15766            // generation and the superseded incumbent is refused.
15767            supervisor.open_swap(OWNER, "n3".to_string());
15768            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
15769            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
15770            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
15771                .await
15772                .expect("the promoted launch takes authority");
15773            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15774                panic!("unexpected reply {reply:?}");
15775            };
15776            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
15777            assert_eq!(
15778                sync(&handler, &incumbent, 4, Vec::new()).await,
15779                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15780            );
15781        }
15782
15783        /// Authority dies with its connection: the cleanup path releases it,
15784        /// so the owner's next connection takes it at any generation.
15785        #[tokio::test]
15786        async fn closing_the_authority_connection_frees_sync_authority() {
15787            let supervisor = SupervisorHandle::new();
15788            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15789            let handler = ControlHandler::new(Arc::new(Registry::default()))
15790                .with_supervisor(supervisor.clone());
15791            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15792            sync(&handler, &first, 10, vec![head("s", 1)])
15793                .await
15794                .unwrap();
15795            handler.cleanup_connection(first.connection_id).unwrap();
15796
15797            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
15798            sync(&handler, &second, 1, vec![head("s", 1)])
15799                .await
15800                .expect("the next connection takes the released authority");
15801        }
15802
15803        #[tokio::test]
15804        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
15805            let supervisor = SupervisorHandle::new();
15806            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15807            let handler =
15808                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
15809            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
15810            sync(&handler, &first, 10, vec![head("s", 1)])
15811                .await
15812                .unwrap();
15813            handler
15814                .handle_control_frame(
15815                    &first,
15816                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
15817                )
15818                .await
15819                .unwrap();
15820            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
15821            sync(&handler, &second, 1, vec![head("s", 1)])
15822                .await
15823                .expect("GOODBYE releases authority even if the old socket remains open");
15824        }
15825
15826        #[tokio::test]
15827        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
15828            let registry = Arc::new(Registry::default());
15829            let supervisor_handle = SupervisorHandle::new();
15830            let supervisor =
15831                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
15832                    .with_handle(supervisor_handle.clone())
15833                    .with_daemon_incarnation("incarnation-7".to_string());
15834            // Configured with enabled: false, so the supervisor lists the
15835            // module without spawning a process for it.
15836            supervisor
15837                .supervise_configured(
15838                    ModuleSpec {
15839                        module_id: OWNER.to_string(),
15840                        program: PathBuf::from("/nonexistent/prefrontal-core"),
15841                        args: Vec::new(),
15842                        env: Vec::new(),
15843                        reserved: false,
15844                        reserved_prefixes: Vec::new(),
15845                        protocol: ModuleProtocol::Subc,
15846                        overlap: Default::default(),
15847                    },
15848                    false,
15849                )
15850                .unwrap();
15851            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
15852            let handler =
15853                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
15854            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
15855
15856            // Configured but not yet synced: a reader waits for the owner.
15857            let ModuleControlResponseToModule::ScopeDescribe {
15858                status,
15859                daemon_incarnation,
15860                owner_synced,
15861                owner_configured,
15862                scope,
15863                ..
15864            } = describe(&handler, &reader, OWNER, "s").await
15865            else {
15866                panic!("not a describe reply");
15867            };
15868            assert_eq!(status, ScopeStatus::NotLive);
15869            assert_eq!(daemon_incarnation, "incarnation-7");
15870            assert!(!owner_synced);
15871            assert!(owner_configured);
15872            assert!(scope.is_none());
15873
15874            // Not a supervised module: the owner will never sync, and a reader
15875            // refuses rather than waits.
15876            let ModuleControlResponseToModule::ScopeDescribe {
15877                status,
15878                owner_configured,
15879                ..
15880            } = describe(&handler, &reader, "ghost", "s").await
15881            else {
15882                panic!("not a describe reply");
15883            };
15884            assert_eq!(status, ScopeStatus::NotLive);
15885            assert!(!owner_configured);
15886
15887            // Live, with the stamp fields and the computed owner_authorized.
15888            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
15889            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
15890            let ModuleControlResponseToModule::ScopeDescribe {
15891                status,
15892                scope_epoch,
15893                owner_synced,
15894                scope,
15895                ..
15896            } = describe(&handler, &reader, OWNER, "s").await
15897            else {
15898                panic!("not a describe reply");
15899            };
15900            assert_eq!(status, ScopeStatus::Live);
15901            assert_eq!(scope_epoch, Some(4));
15902            assert!(owner_synced);
15903            let stamp = scope.expect("a live scope carries its stamp");
15904            assert!(
15905                stamp.owner_authorized,
15906                "prefrontal-core is the default authority"
15907            );
15908            assert_eq!(stamp.kind, ScopeKind::Head);
15909        }
15910
15911        #[tokio::test]
15912        async fn scope_authority_owners_decides_owner_authorized() {
15913            let supervisor = SupervisorHandle::new();
15914            supervisor.set_spawn_nonce("broca", "b1".to_string());
15915            let handler = ControlHandler::new(Arc::new(Registry::default()))
15916                .with_supervisor(supervisor)
15917                .with_scope_authority_owners(vec!["broca".to_string()]);
15918            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
15919            let mut gated = head("s", 1);
15920            gated.attributes.agent_id = Some("agent".to_string());
15921            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
15922            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
15923                describe(&handler, &broca, "broca", "s").await
15924            else {
15925                panic!("not a describe reply");
15926            };
15927            assert!(scope.unwrap().owner_authorized);
15928        }
15929
15930        /// With route admission, the stamp, the commit re-check and drains in
15931        /// place, the feature is advertised: the module ops in HELLO_ACK, and
15932        /// `scopes/v1` in HELLO_ACK and `server.describe`.
15933        #[tokio::test]
15934        async fn scope_ops_and_the_scopes_capability_are_advertised() {
15935            let handler = ControlHandler::new(Arc::new(Registry::default()));
15936            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
15937            let ack = hello_via_sink(
15938                &handler,
15939                &ctx,
15940                &mut rx,
15941                hello_frame("m", PROTOCOL_VERSION, 1),
15942            )
15943            .await;
15944            let ack = parse_ack(&ack);
15945            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
15946                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
15947            }
15948            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
15949
15950            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
15951            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15952            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15953            let reply = handler
15954                .handle_control_frame(&client, frame)
15955                .await
15956                .unwrap()
15957                .pop()
15958                .unwrap();
15959            let ClientControlResponse::ServerDescribe { capabilities, .. } =
15960                serde_json::from_slice(&reply.body).unwrap()
15961            else {
15962                panic!("not a server.describe reply");
15963            };
15964            assert!(
15965                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
15966                "{capabilities:?}"
15967            );
15968        }
15969
15970        // ---- route admission, stamps, commit re-check and drains ----------
15971
15972        const PLEXUS: &str = "plexus";
15973        const OTHER: &str = "other";
15974        const AFT: &str = "aft";
15975        const BROCA: &str = "broca";
15976        const MAGIC: &str = "magic-context";
15977
15978        fn nonce(module_id: &str) -> String {
15979            format!("nonce-{module_id}")
15980        }
15981
15982        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15983            let (tx, rx) = mpsc::channel(64);
15984            (
15985                RouteCtx {
15986                    connection_id: ConnectionId::new(connection),
15987                    egress: FrameSink::new(tx),
15988                },
15989                rx,
15990            )
15991        }
15992
15993        /// A daemon with a configured owner (prefrontal-core) registered on its
15994        /// own module connection, two routable targets (plexus, other), and
15995        /// launch nonces minted for the modules that open routes as carriers.
15996        struct Rig {
15997            handler: ControlHandler,
15998            forwarding: Arc<ForwardingTable>,
15999            owner: RouteCtx,
16000            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16001            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16002            generation: u64,
16003            next_connection: u64,
16004            _supervisor: Supervisor,
16005        }
16006
16007        async fn rig() -> Rig {
16008            rig_with_flow_support(true).await
16009        }
16010
16011        async fn rig_with_flow_support(flow_support: bool) -> Rig {
16012            rig_with_scope_capabilities(flow_support, false).await
16013        }
16014
16015        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16016            let mut provides = Vec::new();
16017            if flow_support {
16018                provides.push("flow-scopes/v1");
16019            }
16020            if run_support {
16021                provides.push("agent-run-scopes/v1");
16022            }
16023            rig_with_target_capabilities(&provides).await
16024        }
16025
16026        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16027            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16028            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16029            // Decoder versions alone cannot admit scoped calls: only a capability
16030            // declaration promises the required run or flow behaviour.
16031            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16032            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16033            Frame::build(
16034                FrameType::Hello,
16035                control_flags(),
16036                0,
16037                0,
16038                connection,
16039                serde_json::to_vec(&body).unwrap(),
16040            )
16041            .unwrap()
16042        }
16043
16044        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16045            let registry = Arc::new(Registry::default());
16046            let forwarding = Arc::new(ForwardingTable::default());
16047            let supervisor_handle = SupervisorHandle::new();
16048            let supervisor =
16049                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16050                    .with_handle(supervisor_handle.clone());
16051            supervisor
16052                .supervise_configured(
16053                    ModuleSpec {
16054                        module_id: OWNER.to_string(),
16055                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16056                        args: Vec::new(),
16057                        env: Vec::new(),
16058                        reserved: false,
16059                        reserved_prefixes: Vec::new(),
16060                        protocol: ModuleProtocol::Subc,
16061                        overlap: Default::default(),
16062                    },
16063                    false,
16064                )
16065                .unwrap();
16066            for module_id in [OWNER, AFT, BROCA, MAGIC] {
16067                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16068            }
16069            let handler =
16070                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16071                    .with_supervisor(supervisor_handle);
16072            let (owner, mut owner_rx) = wide_ctx(1);
16073            hello_via_sink(
16074                &handler,
16075                &owner,
16076                &mut owner_rx,
16077                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16078            )
16079            .await;
16080            let mut modules = BTreeMap::new();
16081            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16082                let (ctx, mut rx) = wide_ctx(connection);
16083                let hello = hello_with_capabilities(module_id, connection, provides);
16084                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16085                modules.insert(module_id.to_string(), (ctx, rx));
16086            }
16087            Rig {
16088                handler,
16089                forwarding,
16090                owner,
16091                _owner_rx: owner_rx,
16092                modules,
16093                generation: 0,
16094                next_connection: 100,
16095                _supervisor: supervisor,
16096            }
16097        }
16098
16099        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16100            ScopeCarrier::new(Principal::Reserved {
16101                module_id: module_id.to_string(),
16102            })
16103            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16104        }
16105
16106        /// The scope most tests open under: aft carries to any module, broca
16107        /// only to plexus and other, and the owner delegates as agent-1.
16108        fn session(scope_epoch: u64) -> ScopeRecord {
16109            let mut record = head("s", scope_epoch);
16110            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16111            record.attributes.agent_id = Some("agent-1".to_string());
16112            record.attributes.delegates = true;
16113            record
16114        }
16115
16116        impl Rig {
16117            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16118                self.generation += 1;
16119                sync(&self.handler, &self.owner, self.generation, scopes)
16120                    .await
16121                    .expect("the owner's sync is accepted");
16122            }
16123
16124            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16125                ScopeSelector::new(
16126                    Principal::Reserved {
16127                        module_id: OWNER.to_string(),
16128                    },
16129                    scope_ref.to_string(),
16130                    scope_epoch,
16131                )
16132            }
16133
16134            fn open_frame(
16135                &mut self,
16136                opener: Option<&str>,
16137                target: &str,
16138                scope: Option<ScopeSelector>,
16139            ) -> (
16140                RouteCtx,
16141                mpsc::Receiver<crate::router::OutboundFrame>,
16142                Frame,
16143            ) {
16144                self.next_connection += 1;
16145                let (ctx, rx) = wide_ctx(self.next_connection);
16146                let root = unique_project_root("scoped-open");
16147                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16148                    target: RouteTarget::ToolProvider {
16149                        module_id: target.to_string(),
16150                    },
16151                    identity: BindIdentity::new(
16152                        root.path().to_path_buf(),
16153                        "unit".to_string(),
16154                        "session".to_string(),
16155                    ),
16156                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
16157                        module_id: module_id.to_string(),
16158                        launch_nonce: nonce(module_id),
16159                    }),
16160                    consumer_capabilities: None,
16161                    role_versions: None,
16162                    admission_facts: None,
16163                    scope,
16164                })
16165                .unwrap();
16166                let frame = Frame::build(
16167                    FrameType::Request,
16168                    control_flags(),
16169                    0,
16170                    0,
16171                    self.next_connection,
16172                    body,
16173                )
16174                .unwrap();
16175                (ctx, rx, frame)
16176            }
16177
16178            /// Open and expect a refusal before anything is relayed.
16179            async fn refused(
16180                &mut self,
16181                opener: Option<&str>,
16182                target: &str,
16183                scope: Option<ScopeSelector>,
16184            ) -> String {
16185                self.refusal_body(opener, target, scope).await["code"]
16186                    .as_str()
16187                    .unwrap()
16188                    .to_string()
16189            }
16190
16191            async fn refusal_body(
16192                &mut self,
16193                opener: Option<&str>,
16194                target: &str,
16195                scope: Option<ScopeSelector>,
16196            ) -> Value {
16197                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16198                let replies = tokio::time::timeout(
16199                    Duration::from_secs(2),
16200                    self.handler.handle_control_frame(&ctx, frame),
16201                )
16202                .await
16203                .expect("the open must be refused before waiting for a bind ack")
16204                .unwrap();
16205                assert_eq!(replies.len(), 1, "{replies:?}");
16206                assert_eq!(replies[0].header.ty, FrameType::Error);
16207                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16208                assert!(
16209                    module_rx.try_recv().is_err(),
16210                    "a refused open relays nothing"
16211                );
16212                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16213                parse_error(&replies[0])
16214            }
16215
16216            /// Start an open and return its task and the bind the target got.
16217            async fn relayed(
16218                &mut self,
16219                opener: Option<&str>,
16220                target: &str,
16221                scope: Option<ScopeSelector>,
16222            ) -> Relayed {
16223                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16224                let handler = self.handler.clone();
16225                let task_ctx = ctx.clone();
16226                let task = tokio::spawn(async move {
16227                    handler
16228                        .handle_control_frame(&task_ctx, frame)
16229                        .await
16230                        .unwrap()
16231                });
16232                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16233                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16234                    .await
16235                    .expect("the target receives the relayed route.bind")
16236                    .unwrap()
16237                    .frame;
16238                Relayed {
16239                    target: target.to_string(),
16240                    client: ctx,
16241                    client_rx: rx,
16242                    task,
16243                    bind,
16244                }
16245            }
16246
16247            async fn ack(&self, relayed: &Relayed) {
16248                let (module, _) = &self.modules[&relayed.target];
16249                self.handler
16250                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16251                    .await
16252                    .unwrap();
16253            }
16254
16255            /// Open, ack and return the bound route.
16256            async fn bound(
16257                &mut self,
16258                opener: Option<&str>,
16259                target: &str,
16260                scope: Option<ScopeSelector>,
16261            ) -> Bound {
16262                let relayed = self.relayed(opener, target, scope).await;
16263                self.ack(&relayed).await;
16264                let Relayed {
16265                    target,
16266                    client,
16267                    mut client_rx,
16268                    task,
16269                    bind,
16270                } = relayed;
16271                assert!(
16272                    task.await.unwrap().is_empty(),
16273                    "the open is answered by commit"
16274                );
16275                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
16276                Bound {
16277                    target,
16278                    client,
16279                    client_rx,
16280                    channel,
16281                    epoch,
16282                    bind,
16283                }
16284            }
16285
16286            fn live(&self, route: &Bound) -> bool {
16287                matches!(
16288                    self.forwarding
16289                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16290                        .unwrap(),
16291                    DataRoute::Client(DataRouteState::Bound(_))
16292                )
16293            }
16294        }
16295
16296        struct Relayed {
16297            target: String,
16298            client: RouteCtx,
16299            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16300            task: tokio::task::JoinHandle<Vec<Frame>>,
16301            bind: Frame,
16302        }
16303
16304        struct Bound {
16305            target: String,
16306            client: RouteCtx,
16307            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16308            channel: u16,
16309            epoch: u32,
16310            bind: Frame,
16311        }
16312
16313        impl Bound {
16314            /// The reason of the `route.closed` this client was sent, after
16315            /// checking it also got a GOODBYE on exactly this route.
16316            fn closed_reason(&mut self) -> RouteCloseReason {
16317                let mut reason = None;
16318                let mut goodbye = false;
16319                while let Ok(outbound) = self.client_rx.try_recv() {
16320                    let frame = outbound.frame;
16321                    match frame.header.ty {
16322                        FrameType::Goodbye => {
16323                            assert_eq!(
16324                                (frame.header.channel, frame.header.epoch),
16325                                (self.channel, self.epoch)
16326                            );
16327                            goodbye = true;
16328                        }
16329                        FrameType::Push => {
16330                            let ClientControlPush::RouteClosed {
16331                                reason: r,
16332                                module_id,
16333                                ..
16334                            } = serde_json::from_slice(&frame.body).unwrap()
16335                            else {
16336                                panic!("unexpected push");
16337                            };
16338                            assert_eq!(module_id, self.target);
16339                            reason = Some(r);
16340                        }
16341                        other => panic!("unexpected frame {other:?}"),
16342                    }
16343                }
16344                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
16345                reason.expect("the client is told why the route closed")
16346            }
16347
16348            fn untouched(&mut self) -> bool {
16349                self.client_rx.try_recv().is_err()
16350            }
16351
16352            fn stamp(&self) -> Option<ScopeStamp> {
16353                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
16354                    ModuleControlRequest::RouteBind { scope, .. } => scope,
16355                    other => panic!("expected a route.bind, got {other:?}"),
16356                }
16357            }
16358        }
16359
16360        #[tokio::test]
16361        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
16362        ) {
16363            let mut rig = rig().await;
16364            let mut record = session(1);
16365            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
16366            record.child_owners = vec![Principal::Reserved {
16367                module_id: MAGIC.to_string(),
16368            }];
16369            rig.sync(vec![record]).await;
16370            let scope = || Some(rig_selector("s", Some(1)));
16371
16372            // Admitted: the owner, a bare carrier to any module, a targeted
16373            // carrier to its listed module.
16374            rig.bound(Some(OWNER), PLEXUS, scope()).await;
16375            rig.bound(Some(AFT), OTHER, scope()).await;
16376            rig.bound(Some(BROCA), PLEXUS, scope()).await;
16377
16378            // Refused scope_not_carrier: a targeted carrier to an unlisted
16379            // module, a module that is not listed at all (a child owner is not
16380            // a carrier), and a direct key-holder.
16381            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16382                assert_eq!(
16383                    rig.refused(opener, target, scope()).await,
16384                    error_codes::SCOPE_NOT_CARRIER,
16385                    "{opener:?} -> {target}"
16386                );
16387            }
16388        }
16389
16390        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16391            ScopeSelector::new(
16392                Principal::Reserved {
16393                    module_id: OWNER.to_string(),
16394                },
16395                scope_ref.to_string(),
16396                scope_epoch,
16397            )
16398        }
16399
16400        #[tokio::test]
16401        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16402            let mut rig = rig().await;
16403            rig.sync(vec![session(1)]).await;
16404            for opener in [OWNER, AFT] {
16405                assert_eq!(
16406                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16407                        .await,
16408                    error_codes::SCOPE_EPOCH_REQUIRED,
16409                    "{opener}"
16410                );
16411            }
16412        }
16413
16414        #[tokio::test]
16415        async fn admission_separates_not_synced_not_live_and_ended() {
16416            let mut rig = rig().await;
16417            // Before the configured owner's first sync: retryable.
16418            let code = rig
16419                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16420                .await;
16421            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16422            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16423
16424            // An owner that is not configured will never sync: terminal.
16425            let ghost = ScopeSelector::new(
16426                Principal::Reserved {
16427                    module_id: "ghost".to_string(),
16428                },
16429                "s".to_string(),
16430                Some(1),
16431            );
16432            assert_eq!(
16433                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16434                error_codes::SCOPE_NOT_LIVE
16435            );
16436
16437            rig.sync(vec![session(2)]).await;
16438            assert_eq!(
16439                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16440                    .await,
16441                error_codes::SCOPE_NOT_LIVE
16442            );
16443            for epoch in [1, 3] {
16444                assert_eq!(
16445                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16446                        .await,
16447                    error_codes::SCOPE_ENDED,
16448                    "epoch {epoch}"
16449                );
16450            }
16451            // Control: the live epoch is admitted.
16452            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16453                .await;
16454        }
16455
16456        #[tokio::test]
16457        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16458            let mut rig = rig().await;
16459            rig.sync(vec![session(1)]).await;
16460            let route = rig
16461                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16462                .await;
16463            let stamp = route.stamp().expect("a scoped bind carries the stamp");
16464            assert_eq!(stamp.scope_ref, "s");
16465            assert_eq!(stamp.scope_epoch, 1);
16466            assert_eq!(stamp.kind, ScopeKind::Head);
16467            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16468            assert!(stamp.attributes.delegates);
16469            assert!(stamp.owner_authorized);
16470            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16471            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16472
16473            // broca owns a scope of its own on its own module connection; it is
16474            // not in scope_authority_owners, so its stamp is not authorized.
16475            let (broca, mut broca_rx) = wide_ctx(50);
16476            hello_via_sink(
16477                &rig.handler,
16478                &broca,
16479                &mut broca_rx,
16480                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
16481            )
16482            .await;
16483            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
16484                .await
16485                .unwrap();
16486            let own = ScopeSelector::new(
16487                Principal::Reserved {
16488                    module_id: BROCA.to_string(),
16489                },
16490                "b".to_string(),
16491                Some(1),
16492            );
16493            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
16494            assert!(!route.stamp().unwrap().owner_authorized);
16495        }
16496
16497        #[tokio::test]
16498        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16499            let mut rig = rig().await;
16500            let mut record = head("s", 1);
16501            record.carriers = vec![carrier(AFT, None)];
16502            let flow_id = "Flow:run-7/step_2!~";
16503            record.attributes.flow_id = Some(flow_id.to_string());
16504            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16505                .await
16506                .unwrap();
16507            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16508                panic!("not a sync reply");
16509            };
16510            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
16511            let route = rig
16512                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16513                .await;
16514            assert!(rig.live(&route), "the stamped bind committed");
16515            let stamp = route.stamp().expect("a flow scope carries a stamp");
16516            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
16517            assert_eq!(stamp.attributes.agent_id, None);
16518            assert!(!stamp.attributes.delegates);
16519            assert!(stamp.owner_authorized);
16520            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16521            assert_eq!(unscoped.stamp(), None);
16522        }
16523
16524        #[tokio::test]
16525        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
16526            let mut rig = rig_with_flow_support(false).await;
16527            let mut record = session(1);
16528            record.attributes.flow_id = Some("flow:7".to_string());
16529            rig.sync(vec![record]).await;
16530            for opener in [OWNER, AFT] {
16531                let body = rig
16532                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16533                    .await;
16534                assert_eq!(body["code"], "target_flow_unsupported");
16535                let message = body["message"].as_str().unwrap();
16536                for required in [PLEXUS, "flow-scopes/v1"] {
16537                    assert!(message.contains(required), "{message}");
16538                }
16539            }
16540        }
16541
16542        #[tokio::test]
16543        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
16544            let mut rig = rig_with_flow_support(true).await;
16545            let mut record = session(1);
16546            record.attributes.flow_id = Some("flow:7".to_string());
16547            rig.sync(vec![record]).await;
16548            for opener in [OWNER, AFT] {
16549                let route = rig
16550                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16551                    .await;
16552                assert!(rig.live(&route));
16553                assert_eq!(
16554                    route.stamp().unwrap().attributes.flow_id.as_deref(),
16555                    Some("flow:7")
16556                );
16557            }
16558        }
16559
16560        #[tokio::test]
16561        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
16562            use std::future::Future;
16563
16564            let mut rig = rig_with_flow_support(true).await;
16565            let mut record = session(1);
16566            record.attributes.flow_id = Some("flow:7".to_string());
16567            rig.sync(vec![record]).await;
16568            let (client, mut client_rx, frame) =
16569                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
16570            // Hold the route-open response permit so admission sees the first
16571            // target but relay reservation cannot capture an endpoint yet.
16572            for _ in 0..64 {
16573                client.egress.try_send(route_bind_ack(1)).unwrap();
16574            }
16575            let handler = rig.handler.clone();
16576            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16577            std::future::poll_fn(|cx| {
16578                assert!(open.as_mut().poll(cx).is_pending());
16579                std::task::Poll::Ready(())
16580            })
16581            .await;
16582            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16583
16584            let old_connection = rig.modules[PLEXUS].0.connection_id;
16585            rig.handler.cleanup_connection(old_connection).unwrap();
16586            let (replacement, mut replacement_rx) = wide_ctx(200);
16587            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
16588            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16589            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16590            let hello = Frame::build(
16591                FrameType::Hello,
16592                control_flags(),
16593                0,
16594                0,
16595                200,
16596                serde_json::to_vec(&body).unwrap(),
16597            )
16598            .unwrap();
16599            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
16600
16601            client_rx.try_recv().unwrap();
16602            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16603                .await
16604                .expect("the replacement is refused without waiting for a bind ack")
16605                .unwrap();
16606            assert_eq!(replies.len(), 1);
16607            let body = parse_error(&replies[0]);
16608            assert_eq!(body["code"], "target_flow_unsupported");
16609            for required in [PLEXUS, "flow-scopes/v1"] {
16610                assert!(body["message"].as_str().unwrap().contains(required));
16611            }
16612            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
16613            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16614            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16615            assert!(rig
16616                .handler
16617                .registry
16618                .get_module_by_connection(replacement.connection_id)
16619                .unwrap()
16620                .is_some());
16621        }
16622
16623        #[tokio::test]
16624        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
16625            use std::future::Future;
16626            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16627            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16628            const NEXT_HASH: &str =
16629                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16630            let mut rig = rig().await;
16631            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
16632            record.carriers.push(ScopeCarrier::new(Principal::Direct));
16633            rig.sync(vec![record.clone()]).await;
16634            let (client, mut rx, frame) = rig.open_frame(
16635                None,
16636                PLEXUS,
16637                Some(
16638                    rig.selector("s", Some(1))
16639                        .with_bind_proof(Some(PROOF.into())),
16640                ),
16641            );
16642            // Hold egress reservation while the scope owner changes its bind-proof
16643            // hash, so a proof valid at admission must be checked again at relay.
16644            for _ in 0..64 {
16645                client.egress.try_send(route_bind_ack(1)).unwrap();
16646            }
16647            let handler = rig.handler.clone();
16648            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16649            std::future::poll_fn(|cx| {
16650                assert!(open.as_mut().poll(cx).is_pending());
16651                std::task::Poll::Ready(())
16652            })
16653            .await;
16654            record.bind_proof_sha256 = Some(NEXT_HASH.into());
16655            rig.sync(vec![record]).await;
16656            rx.try_recv().unwrap();
16657            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16658                .await
16659                .unwrap()
16660                .unwrap();
16661            assert_eq!(
16662                parse_error(&replies[0])["code"],
16663                error_codes::SCOPE_BIND_PROOF_MISMATCH
16664            );
16665            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
16666            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16667            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16668        }
16669
16670        #[tokio::test]
16671        async fn scope_bind_proof_reserved_openers_ignore_proof() {
16672            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16673            let mut rig = rig().await;
16674            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16675                .await;
16676            for opener in [OWNER, AFT] {
16677                for proof in [None, Some("malformed+proof=")] {
16678                    let route = rig
16679                        .bound(
16680                            Some(opener),
16681                            PLEXUS,
16682                            Some(
16683                                rig.selector("s", Some(1))
16684                                    .with_bind_proof(proof.map(str::to_string)),
16685                            ),
16686                        )
16687                        .await;
16688                    assert!(rig.live(&route));
16689                }
16690            }
16691        }
16692
16693        #[tokio::test]
16694        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
16695            use std::future::Future;
16696            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16697            const NEXT_HASH: &str =
16698                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16699            let mut rig = rig().await;
16700            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16701                .await;
16702            let (client, mut rx, frame) =
16703                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
16704            for _ in 0..64 {
16705                client.egress.try_send(route_bind_ack(1)).unwrap();
16706            }
16707            let handler = rig.handler.clone();
16708            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16709            std::future::poll_fn(|cx| {
16710                assert!(open.as_mut().poll(cx).is_pending());
16711                std::task::Poll::Ready(())
16712            })
16713            .await;
16714            rig.sync(vec![
16715                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
16716            ])
16717            .await;
16718            rx.try_recv().unwrap();
16719            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16720            let bind = std::future::poll_fn(|cx| {
16721                assert!(open.as_mut().poll(cx).is_pending());
16722                module_rx.poll_recv(cx)
16723            })
16724            .await
16725            .unwrap()
16726            .frame;
16727            handler
16728                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
16729                .await
16730                .unwrap();
16731            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16732                .await
16733                .unwrap()
16734                .unwrap();
16735            assert_eq!(replies.len(), 1);
16736            // Proof rechecking allowed the relay. Independent tag settlement
16737            // still asks the opener to retry after any same-epoch content change.
16738            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
16739            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16740            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
16741            let route = rig
16742                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
16743                .await;
16744            assert!(rig.live(&route));
16745        }
16746
16747        #[tokio::test]
16748        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
16749        {
16750            let mut rig = rig_with_flow_support(false).await;
16751            rig.sync(vec![session(1)]).await;
16752            let route = rig
16753                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16754                .await;
16755            assert!(rig.live(&route));
16756            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
16757            let mut record = session(1);
16758            record.attributes.flow_id = Some("flow:7".to_string());
16759            rig.sync(vec![record]).await;
16760            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
16761            assert!(rig.live(&unscoped));
16762            assert_eq!(unscoped.stamp(), None);
16763        }
16764
16765        #[tokio::test]
16766        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
16767            let mut rig = rig().await;
16768            let mut record = session(1);
16769            record.attributes.flow_id = Some("flow:7".to_string());
16770            rig.sync(vec![record.clone()]).await;
16771            let before = rig.forwarding.published_scope_tag(OWNER, "s");
16772            let mut owner_route = rig
16773                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16774                .await;
16775            let mut carrier_route = rig
16776                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
16777                .await;
16778            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16779            rig.sync(vec![record.clone()]).await;
16780            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16781            assert!(rig.live(&owner_route) && owner_route.untouched());
16782            assert!(rig.live(&carrier_route) && carrier_route.untouched());
16783
16784            record.attributes.flow_id = Some("flow:8".to_string());
16785            rig.sync(vec![record]).await;
16786            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
16787            let before = before.unwrap();
16788            assert_eq!(after.scope_epoch, before.scope_epoch);
16789            assert!(after.version > before.version);
16790            for route in [&mut owner_route, &mut carrier_route] {
16791                assert!(!rig.live(route));
16792                assert_eq!(
16793                    route.closed_reason(),
16794                    RouteCloseReason::ScopeDelegationChanged
16795                );
16796            }
16797            assert!(rig.live(&unscoped) && unscoped.untouched());
16798            // Each provider also receives a GOODBYE for its drained route;
16799            // consume it before expecting the next route.bind on that sink.
16800            for target in [PLEXUS, OTHER] {
16801                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
16802                let goodbye = module_rx
16803                    .try_recv()
16804                    .expect("the provider sees the drain")
16805                    .frame;
16806                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16807                assert!(module_rx.try_recv().is_err());
16808            }
16809            let rebound = rig
16810                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16811                .await;
16812            assert_eq!(
16813                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
16814                Some("flow:8")
16815            );
16816        }
16817
16818        /// The owner's sync lands between admission and the module's ack. The
16819        /// open is refused by name, the module's other routes stay up, and the
16820        /// reserved pair is released. Changed content is retryable; an ended
16821        /// scope is not.
16822        #[tokio::test]
16823        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
16824            let mut rig = rig().await;
16825            rig.sync(vec![session(1)]).await;
16826            let mut cotenant = rig
16827                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16828                .await;
16829
16830            let mut changed = session(1);
16831            changed.child_owners.push(Principal::Reserved {
16832                module_id: MAGIC.to_string(),
16833            });
16834            let mut ended = None;
16835            for (code, next) in [
16836                (error_codes::SCOPE_CHANGED, vec![changed]),
16837                (error_codes::SCOPE_ENDED, Vec::new()),
16838            ] {
16839                let relayed = rig
16840                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16841                    .await;
16842                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
16843                ended = Some(next.is_empty());
16844                rig.sync(next).await;
16845                rig.ack(&relayed).await;
16846                let replies = relayed.task.await.unwrap();
16847                assert_eq!(replies.len(), 1, "{replies:?}");
16848                assert_eq!(parse_error(&replies[0])["code"], code);
16849                assert_eq!(
16850                    subc_protocol::error_codes::is_retryable_route_open(code),
16851                    code == error_codes::SCOPE_CHANGED
16852                );
16853                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16854                // The module is told to drop just the binding it created.
16855                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16856                // Collected, because ending the scope also closes the co-tenant
16857                // route, whose GOODBYE comes first.
16858                let mut goodbyes = Vec::new();
16859                while let Ok(outbound) = plexus_rx.try_recv() {
16860                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
16861                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
16862                }
16863                assert!(
16864                    goodbyes.contains(&(bind_channel, bind_epoch)),
16865                    "{goodbyes:?}"
16866                );
16867                assert!(rig
16868                    .handler
16869                    .registry
16870                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
16871                    .unwrap()
16872                    .is_some());
16873            }
16874            assert_eq!(ended, Some(true));
16875            // The co-tenant stayed up through the change, and closed only when
16876            // the scope ended, by the drain rule rather than by the commit.
16877            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
16878        }
16879
16880        /// Each row of the drain table on one set of routes: the owner's, a
16881        /// bare carrier's, and a targeted carrier's to each of its targets.
16882        #[tokio::test]
16883        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
16884            struct Case {
16885                name: &'static str,
16886                change: fn(&mut ScopeRecord),
16887                /// Closed routes by index: owner->plexus, aft->plexus,
16888                /// broca->plexus, broca->other.
16889                closed: [Option<RouteCloseReason>; 4],
16890            }
16891            use RouteCloseReason::*;
16892            let cases = [
16893                Case {
16894                    name: "a carrier entry removed",
16895                    change: |r| {
16896                        r.carriers.retain(|c| {
16897                            c.principal
16898                                != Principal::Reserved {
16899                                    module_id: AFT.to_string(),
16900                                }
16901                        })
16902                    },
16903                    closed: [None, Some(ScopeCarrierRemoved), None, None],
16904                },
16905                Case {
16906                    name: "a target removed from a carrier",
16907                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
16908                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
16909                },
16910                Case {
16911                    name: "a bare carrier narrowed to targets",
16912                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
16913                    closed: [None, Some(ScopeCarrierRemoved), None, None],
16914                },
16915                Case {
16916                    name: "delegates turned off",
16917                    change: |r| r.attributes.delegates = false,
16918                    closed: [Some(ScopeDelegationChanged); 4],
16919                },
16920                Case {
16921                    name: "agent_id changed",
16922                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
16923                    closed: [Some(ScopeDelegationChanged); 4],
16924                },
16925                Case {
16926                    name: "a carrier added, child owners changed, the record re-sent",
16927                    change: |r| {
16928                        r.carriers.push(carrier(MAGIC, None));
16929                        r.child_owners.push(Principal::Reserved {
16930                            module_id: MAGIC.to_string(),
16931                        });
16932                    },
16933                    closed: [None; 4],
16934                },
16935                Case {
16936                    name: "a target added",
16937                    change: |r| {
16938                        r.carriers[1]
16939                            .targets
16940                            .as_mut()
16941                            .unwrap()
16942                            .push("third".to_string())
16943                    },
16944                    closed: [None; 4],
16945                },
16946                Case {
16947                    name: "delegates turned on",
16948                    change: |r| r.attributes.delegates = true,
16949                    closed: [None; 4],
16950                },
16951            ];
16952            for case in cases {
16953                let mut rig = rig().await;
16954                rig.sync(vec![session(1)]).await;
16955                let scope = || Some(rig_selector("s", Some(1)));
16956                let mut routes = [
16957                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
16958                    rig.bound(Some(AFT), PLEXUS, scope()).await,
16959                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
16960                    rig.bound(Some(BROCA), OTHER, scope()).await,
16961                ];
16962                let mut record = session(1);
16963                (case.change)(&mut record);
16964                rig.sync(vec![record]).await;
16965                for (index, expected) in case.closed.iter().enumerate() {
16966                    let route = &mut routes[index];
16967                    match expected {
16968                        Some(reason) => {
16969                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
16970                            assert_eq!(
16971                                route.closed_reason(),
16972                                *reason,
16973                                "{}: route {index}",
16974                                case.name
16975                            );
16976                        }
16977                        None => {
16978                            assert!(rig.live(route), "{}: route {index} closed", case.name);
16979                            assert!(
16980                                route.untouched(),
16981                                "{}: route {index} was told something",
16982                                case.name
16983                            );
16984                        }
16985                    }
16986                }
16987            }
16988        }
16989
16990        #[tokio::test]
16991        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
16992            // Removed, and replaced by a higher epoch.
16993            for next in [Vec::new(), vec![session(2)]] {
16994                let mut rig = rig().await;
16995                rig.sync(vec![session(1)]).await;
16996                let mut route = rig
16997                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16998                    .await;
16999                rig.sync(next).await;
17000                assert!(!rig.live(&route));
17001                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17002            }
17003
17004            // A child whose parent ends: its routes close as parent-ended, the
17005            // child stays live, and routes under the parent close as ended.
17006            let mut rig = rig().await;
17007            let mut child = session(1);
17008            child.scope_ref = "child".to_string();
17009            child.kind = ScopeKind::Worker;
17010            child.parent = Some(ScopeParent::new(
17011                Principal::Reserved {
17012                    module_id: OWNER.to_string(),
17013                },
17014                "s".to_string(),
17015                1,
17016            ));
17017            rig.sync(vec![session(1), child.clone()]).await;
17018            let mut child_route = rig
17019                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17020                .await;
17021            assert_eq!(
17022                child_route.stamp().unwrap().parent_state,
17023                Some(ParentState::Linked)
17024            );
17025            rig.sync(vec![child]).await;
17026            assert!(!rig.live(&child_route));
17027            assert_eq!(
17028                child_route.closed_reason(),
17029                RouteCloseReason::ScopeParentEnded
17030            );
17031        }
17032
17033        #[tokio::test]
17034        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17035        ) {
17036            let mut rig = rig().await;
17037            rig.sync(vec![session(1)]).await;
17038            let mut route = rig
17039                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17040                .await;
17041            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17042            rig.sync(vec![session(1)]).await;
17043            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17044            assert!(rig.live(&route) && route.untouched());
17045
17046            // A call in flight on the route when another carrier is added. A
17047            // forwarded REQUEST holds one credit on the route's flow until the
17048            // module answers; the router takes it exactly like this.
17049            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17050                .forwarding
17051                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17052                .unwrap()
17053            else {
17054                panic!("the route is bound");
17055            };
17056            binding.flow.acquire_tagged(9, false).await.unwrap();
17057            let mut widened = session(1);
17058            widened.carriers.push(carrier(MAGIC, None));
17059            rig.sync(vec![widened]).await;
17060            assert!(rig.live(&route) && route.untouched());
17061            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17062            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17063            // The call's credit is still held on an open flow, so its answer
17064            // will be delivered: closing the route would have closed the flow.
17065            assert_eq!(binding.flow.in_flight(), 1);
17066            binding
17067                .flow
17068                .acquire_tagged(10, false)
17069                .await
17070                .expect("the flow is still open");
17071        }
17072
17073        /// A swap's superseded endpoint keeps its routes until drained; ending
17074        /// the scope closes them there too.
17075        #[tokio::test]
17076        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17077            let mut rig = rig().await;
17078            rig.sync(vec![session(1)]).await;
17079            let mut on_incumbent = rig
17080                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17081                .await;
17082
17083            // Swap plexus: register a candidate and cut over, leaving the
17084            // incumbent superseded with the route still on it.
17085            let (candidate, _candidate_rx) = wide_ctx(9);
17086            let registration = rig
17087                .handler
17088                .registry
17089                .register_candidate_with_control_ops(
17090                    manifest(PLEXUS, PROTOCOL_VERSION),
17091                    PROTOCOL_VERSION,
17092                    candidate.connection_id,
17093                    module_baseline_control_ops(),
17094                )
17095                .unwrap();
17096            rig.forwarding
17097                .register_candidate_module_connection(
17098                    candidate.connection_id,
17099                    PLEXUS.to_string(),
17100                    PROTOCOL_VERSION,
17101                    manifest_concurrency(&registration.manifest),
17102                    candidate.egress.clone(),
17103                )
17104                .unwrap();
17105            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17106            rig.handler
17107                .registry
17108                .promote_candidate(PLEXUS)
17109                .unwrap()
17110                .unwrap();
17111            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17112
17113            rig.sync(Vec::new()).await;
17114            assert!(!rig.live(&on_incumbent));
17115            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17116            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17117            let goodbye = incumbent_rx
17118                .try_recv()
17119                .expect("the superseded endpoint is told")
17120                .frame;
17121            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17122        }
17123    }
17124}
17125
17126#[cfg(test)]
17127mod concurrency_default_exposure_tests {
17128    use super::*;
17129
17130    fn hello_body(role_json: &str) -> Vec<u8> {
17131        format!(
17132            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17133        )
17134        .into_bytes()
17135    }
17136
17137    fn manifest_from(body: &[u8]) -> ModuleManifest {
17138        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17139        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17140            .expect("manifest parses")
17141    }
17142
17143    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17144
17145    #[test]
17146    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17147        let body = hello_body(&format!(
17148            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17149        ));
17150        let manifest = manifest_from(&body);
17151        // Precondition: serde really resolved it to the default, so the typed
17152        // manifest alone cannot answer the question this probe exists for.
17153        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17154        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17155    }
17156
17157    #[test]
17158    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17159        let body = hello_body(&format!(
17160            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17161        ));
17162        let manifest = manifest_from(&body);
17163        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17164        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17165    }
17166
17167    #[test]
17168    fn non_management_roles_are_never_reported() {
17169        let body = hello_body(
17170            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17171        );
17172        let manifest = manifest_from(&body);
17173        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17174    }
17175}