Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_event_declarations,
25        validate_hello_rich_ui_descriptor, validate_hello_self_signal_declarations,
26        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
27        ProviderRole,
28    },
29    scope::{
30        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
31        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
32        SCOPE_SYNC_OP,
33    },
34    session::{
35        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
36        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
37        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
38        ROLE_VERSIONS_FIELD,
39    },
40    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
41    Priority, RouteTarget, PROTOCOL_VERSION,
42};
43use tokio::time::{timeout_at, Instant};
44use tracing::{debug, info, warn};
45
46use crate::{
47    caller_identity::CallerIdentity,
48    capability_requirements::{
49        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
50        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
51        RuntimeModule,
52    },
53    child_roster::DaemonShutdownFlag,
54    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
55    forwarding::{
56        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
57        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
58        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
59    },
60    observability::{
61        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
62    },
63    privileged_prompt::{prompt_text, PrivilegedOperation},
64    provenance::{
65        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
66    },
67    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
68    router::{RouteCtx, RouterError},
69    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
70    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
71    stderr_tail::{CaptureState, TailEntry},
72    supervise::{
73        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
74        SupervisorHandle, SwapHelloAdmission,
75    },
76    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
77};
78
79/// Lowest envelope version this subc build will negotiate.
80///
81/// Module HELLO negotiation is exact: peers must use the daemon's locked
82/// protocol version. Older and newer peers receive `version_unsupported` and
83/// are not registered.
84pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
85
86const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
87const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
88const CAP_PING_PONG: &str = "ping_pong_v1";
89const CAP_SESSION_ATTACH: &str = "session_attach_v1";
90const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
91
92const SUBC_CONTROL_OPS: &[&str] = &[
93    ops::ACTIVITY_RECORD,
94    ops::ACTIVITY_QUERY,
95    ops::ACTIVITY_SUBSCRIBE,
96    ops::SERVER_DESCRIBE,
97    ops::CATALOG_LIST,
98    ops::ROUTE_OPEN,
99    ops::ROUTE_POLL,
100    ops::ROUTE_CLOSING,
101    ops::ROUTE_CLOSED,
102    ops::SUPERVISOR_LIST,
103    ops::SUPERVISOR_RESTART,
104    ops::SUPERVISOR_SWAP,
105    ops::SUPERVISOR_RELOAD,
106    ops::SUPERVISOR_RESCAN,
107    ops::SUPERVISOR_RELEASE_RESERVED,
108    ops::SUPERVISOR_SET_ENABLED,
109    ops::SUPERVISOR_HEALTH_PROBE,
110    ops::SUPERVISOR_HEALTH,
111    ops::SUPERVISOR_STDERR_TAIL,
112    ops::SUPERVISOR_TERMINALS,
113    ops::SUPERVISOR_ROUTES,
114    ops::SUPERVISOR_PROVENANCE,
115    ops::SUPERVISOR_SPAWN_SNAPSHOT,
116    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
117];
118
119/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
120/// announced by a capability in `server.describe` instead, and clients check
121/// that capability before sending it. `ck daemon --json` prints `subc_ops`
122/// verbatim and scripts read that output, so a new name there would change it.
123/// A malformed body for one of these is still refused `invalid_control_body`
124/// rather than as an unknown op.
125const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
126
127mod rescan_checked;
128use rescan_checked::{RescanMode, RescanPlan};
129
130const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
131    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
132    "supervisor.live_roots",
133    SCOPE_SYNC_OP,
134    SCOPE_APPLY_OP,
135    SCOPE_DESCRIBE_OP,
136    "operator.confirm",
137];
138
139/// Module-originated ops the daemon answers but does not advertise in
140/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
141/// belongs to is incomplete, so no module is told it works before it does.
142const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
143
144const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
145
146/// How long subc waits for a module to ack a relayed route.bind before returning
147/// `module_timeout`. The ack waits on the module's own configure, which for AFT
148/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
149/// and DB-open work — on a cold page cache or a large repo that legitimately
150/// exceeds a couple of seconds. The default is generous because rejecting a VALID
151/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
152/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
153pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
154
155/// How many CONSECUTIVE full-budget relay timeouts against one target module
156/// open that module's bind-relay breaker.
157///
158/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
159/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
160/// so three consecutive full-budget timeouts take ~36s to observe: every client
161/// whose open contributed to opening the breaker had already given up on its
162/// own. That is what makes opening the breaker unable to turn a call that would
163/// have succeeded into a refusal — it can only make an already-failing module
164/// fail faster.
165///
166/// Two would be reachable inside one default deadline. One would convict a
167/// module on a single cold-cache bind, which is exactly the valid-but-slow case
168/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
169pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
170
171/// How long a module's bind-relay breaker stays open before exactly one
172/// `route.open` is let through as a probe.
173///
174/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
175/// re-pays a full-budget stall almost continuously, and the breaker stops being
176/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
177/// deadline: a client that starts retrying after the module recovers has to get
178/// a probe opportunity inside its own deadline, or the breaker converts a
179/// recovered module into a failed call — the failure it exists to prevent,
180/// pointed the other way.
181///
182/// 20s sits between those with room on both sides, and it caps what a wedged
183/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
184/// rather than one per `route.open` per connection. The stall that motivated
185/// this, with its measurements, is written up in
186/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
187/// waited the whole budget out.
188pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
189
190const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
191const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
192
193fn reload_verdict(
194    configured: &Path,
195    spawned_from: Option<&Path>,
196    image: subc_control::RunningImageAgreement,
197) -> PendingReloadVerdict {
198    let path = match spawned_from {
199        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
200            ReloadPathAgreement::Match
201        }
202        Some(spawned_from) => ReloadPathAgreement::Mismatch {
203            configured: configured.to_path_buf(),
204            spawned_from: spawned_from.to_path_buf(),
205        },
206        None => ReloadPathAgreement::Unavailable {
207            reason: if matches!(
208                image,
209                subc_control::RunningImageAgreement::Unavailable {
210                    reason: subc_control::RunningImageUnavailableReason::NotRunning
211                }
212            ) {
213                ReloadPathUnavailableReason::NotRunning
214            } else {
215                ReloadPathUnavailableReason::SpawnedPathUnavailable
216            },
217        },
218    };
219    PendingReloadVerdict {
220        path,
221        image,
222        spec: None,
223    }
224}
225
226fn launch_spec_agreement(
227    configured: &crate::supervise::ModuleSpec,
228    running: bool,
229    spawned: Option<&crate::supervise::LaunchSpecDigest>,
230) -> subc_control::LaunchSpecAgreement {
231    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
232    if !running {
233        return Agreement::Unavailable {
234            reason: Reason::NotRunning,
235        };
236    }
237    match spawned {
238        None => Agreement::Unavailable {
239            reason: Reason::SpawnedSpecUnavailable,
240        },
241        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
242            Agreement::Match
243        }
244        Some(_) => Agreement::Mismatch,
245    }
246}
247
248fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
249    if configured == spawned_from {
250        return true;
251    }
252    #[cfg(windows)]
253    {
254        // Windows spawn records the resolved verbatim path (\\?\...), while
255        // configuration keeps its original spelling. Resolve both with the OS
256        // instead of stripping prefixes or folding case, which could conflate
257        // distinct names in case-sensitive directories. Different resolved
258        // paths still differ even if their files have identical content.
259        match (
260            std::fs::canonicalize(configured),
261            std::fs::canonicalize(spawned_from),
262        ) {
263            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
264            _ => false,
265        }
266    }
267    #[cfg(not(windows))]
268    {
269        false
270    }
271}
272
273#[derive(Clone)]
274struct DaemonProvenanceFacts {
275    build: DaemonBuildProvenance,
276    pid: Option<u32>,
277    started_at_ms: Option<u64>,
278    start_clock: Option<crate::clock::StartClock>,
279    executable_path: Option<PathBuf>,
280    executable_identity: Option<SpawnedFileIdentity>,
281    process_start_time: Option<u64>,
282    probe: ExecutableIdentityProbe,
283}
284
285impl Default for DaemonProvenanceFacts {
286    fn default() -> Self {
287        Self {
288            build: DaemonBuildProvenance {
289                build_git_sha: None,
290                build_lock_digest: None,
291            },
292            pid: None,
293            started_at_ms: None,
294            start_clock: None,
295            executable_path: None,
296            executable_identity: None,
297            process_start_time: None,
298            probe: ExecutableIdentityProbe::default(),
299        }
300    }
301}
302
303#[derive(Debug, Clone)]
304struct SupervisorRescanContext {
305    supervisor: Supervisor,
306    config_path: PathBuf,
307    configured_port: Option<u16>,
308    storage_config: Option<crate::daemon_config::StorageConfig>,
309    admission_facts_carrier_module_id: Option<String>,
310    admission_facts_targets: Option<Vec<String>>,
311    scope_authority_owners: Vec<String>,
312    privileged_operator_prompt: PrivilegedOperatorPrompt,
313    /// Counts executed rescans that set out to change the supervised modules'
314    /// staged or applied configuration, counted before the first change so a
315    /// rescan that fails partway still counts. With the daemon incarnation it
316    /// names the state a `supervisor.rescan_checked` preview was computed
317    /// against.
318    generation: Arc<std::sync::atomic::AtomicU64>,
319}
320
321/// Refusal labels passed to `observe_route_open_refusal` that mean the target
322/// module is not serving right now, and so open or extend an outage in the
323/// route outage tracker. Every one of them is only reachable after the target
324/// was found in the registry, which is what keeps an arbitrary client-chosen
325/// id from ever creating tracker state.
326///
327/// Deliberately absent: `not_registered` and `removed` (the id may be
328/// anything a client sent, and a removed module is gone on purpose),
329/// `protocol_none` (such a module never serves routes, so nothing is out),
330/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
331/// capability and admission-facts refusals (they refuse the caller, not a
332/// module outage), and `relay_reservation_failed` (its code ranges over
333/// capacity limits as well as a vanished connection). Capacity, breaker,
334/// relay-timeout and module-rejection refusals do not pass through that
335/// function at all; the breaker logs its own transitions.
336///
337/// The two not-serving refusals that bypass that function record themselves
338/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
339/// `required_capability_unprovided` is not tracked: the module itself is up,
340/// and the outage belongs to the missing provider.
341const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
342    "reloading",
343    "supervisor_not_live",
344    "registration_not_active",
345    "no_forwarding_connection",
346    "relay_send_failed",
347];
348
349/// Real channel-0 control handler for subc itself.
350#[derive(Clone)]
351pub struct ControlHandler {
352    registry: Arc<Registry>,
353    activity: Option<crate::activity::ActivityLedger>,
354    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
355    forwarding: Arc<ForwardingTable>,
356    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
357    supervisor: SupervisorHandle,
358    subc_capabilities: Arc<[String]>,
359    /// Daemon-wide route.bind relay budget. Used as the fallback when the
360    /// target module has no per-module override in
361    /// `route_bind_relay_timeouts`.
362    route_bind_relay_timeout: Duration,
363    /// Per-module route.bind relay budget overrides, keyed by module id. When
364    /// `handle_route_open` resolves the deadline for a target module, a
365    /// per-module entry wins over the daemon-wide value above.
366    route_bind_relay_timeouts: BTreeMap<String, Duration>,
367    /// Per-target-module bind-relay breaker state. Shared with the forwarding
368    /// table, which is where a new module connection resets it.
369    route_bind_breakers: RouteBindBreakers,
370    /// Live relay admissions keyed by target module. Shared through the
371    /// forwarding table so cloned or separately built handlers enforce one cap.
372    route_bind_concurrency: RouteBindConcurrency,
373    /// Start and end of each module's not-serving period as seen by
374    /// `route.open`, so an outage gets one line at each edge instead of only
375    /// the per-refusal INFO lines. Taken from the forwarding table, so every
376    /// handler built over one table shares it.
377    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
378    /// Consecutive relay timeouts that open a module's breaker.
379    route_bind_breaker_threshold: u32,
380    /// How long a breaker stays open before one probe is admitted.
381    route_bind_breaker_cooldown: Duration,
382    health_probe_timeout: Duration,
383    /// Central storage policy. When set, each registering module receives its
384    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
385    storage_config: Option<crate::daemon_config::StorageConfig>,
386    /// The machine id established at boot, served on every HELLO_ACK and on
387    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
388    /// changes the file, never this value. `None` serves no id.
389    machine_id: Option<crate::machine_id::MachineId>,
390    admission_facts_carrier_module_id: Option<String>,
391    admission_facts_targets: Option<Vec<String>>,
392    /// Scope records with their sync authorities and tombstones; see
393    /// `crate::scopes`. Shared by clones of this handler, so every connection
394    /// reads and writes one table.
395    scopes: Arc<RwLock<ScopeTable>>,
396    /// Wakes the expiry loop when an accepted scope change may have added a
397    /// deadline, so the loop can wait without ticking while no live scope has
398    /// one. A permit is stored when nothing waits, so no wake is lost.
399    scope_deadline_added: Arc<tokio::sync::Notify>,
400    /// The configured `scope_authority_owners`, kept so a rescan can report a
401    /// changed value as needing a daemon restart; rescan never applies it.
402    scope_authority_owners: Vec<String>,
403    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
404    privileged_operator_prompt: PrivilegedOperatorPrompt,
405    /// The local and peer address of each accepted TCP client connection, read
406    /// from the daemon's own socket, so a privileged request can name its caller.
407    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
408    #[cfg(test)]
409    caller_lookup: Option<TestCallerLookup>,
410    /// The launch nonce each module connection presented at HELLO, which is how
411    /// a `scope.sync` is matched to the owner's current launch.
412    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
413    rescan: Option<SupervisorRescanContext>,
414    connected_clients: ConnectedClients,
415    counters: DaemonCounters,
416    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
417    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
418    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
419    /// their usual severity (see [`log_requirement_events`]). A handler built
420    /// without one has a flag that is never set.
421    daemon_shutdown: DaemonShutdownFlag,
422    daemon_provenance: DaemonProvenanceFacts,
423    #[cfg(test)]
424    control_dispatch_delay: Option<Duration>,
425    #[cfg(test)]
426    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
427    /// Holds `route.open` after the route.bind is queued to the module and
428    /// before it is marked enqueued, so a test can make the module's reply (or
429    /// a client teardown) land exactly inside that window.
430    #[cfg(test)]
431    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
432    /// Holds a HELLO after its registry entry is reserved and before its
433    /// forwarding endpoint is installed, so a test can read the catalog and
434    /// open routes while the module is registered but not yet routable. A
435    /// blocking barrier because the HELLO handler is synchronous; the test
436    /// runs the HELLO on its own thread.
437    #[cfg(test)]
438    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
439}
440
441impl fmt::Debug for ControlHandler {
442    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
443        f.debug_struct("ControlHandler")
444            .field("registry", &self.registry)
445            .field("forwarding", &self.forwarding)
446            .field("process_liveness", &self.process_liveness.is_some())
447            .field("supervisor", &self.supervisor)
448            .field("subc_capabilities", &self.subc_capabilities)
449            .finish()
450    }
451}
452
453struct RouteOpenRequest {
454    target: RouteTarget,
455    identity: BindIdentity,
456    consumer_identity: Option<ConsumerIdentity>,
457    consumer_capabilities: Option<Vec<String>>,
458    role_versions: Option<BTreeMap<String, String>>,
459    admission_facts: Option<serde_json::Value>,
460    scope: Option<ScopeSelector>,
461}
462
463#[derive(Clone, Copy)]
464enum RouteOpenPhase {
465    Admission,
466    ProjectRoot,
467    EgressReservation,
468    BindSend,
469    BindAckWait,
470}
471
472/// Store elapsed time for each route-open phase, including attempts that end
473/// in an early refusal or backend error. Admission also includes rechecks and
474/// commit bookkeeping performed between the other timed phases.
475struct RouteOpenTimings {
476    module_id: String,
477    principal: String,
478    admission: Duration,
479    project_root: Duration,
480    egress_reservation: Duration,
481    bind_send: Duration,
482    bind_ack_wait: Duration,
483    phase: Option<RouteOpenPhase>,
484    phase_started_at: Instant,
485}
486
487impl RouteOpenTimings {
488    fn new() -> Self {
489        Self {
490            module_id: String::new(),
491            // Refusals before attestation must not claim a verified principal.
492            principal: "unresolved".to_string(),
493            admission: Duration::ZERO,
494            project_root: Duration::ZERO,
495            egress_reservation: Duration::ZERO,
496            bind_send: Duration::ZERO,
497            bind_ack_wait: Duration::ZERO,
498            phase: Some(RouteOpenPhase::Admission),
499            phase_started_at: Instant::now(),
500        }
501    }
502
503    fn finish(&mut self) {
504        let elapsed = self.phase_started_at.elapsed();
505        match self.phase.take() {
506            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
507            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
508            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
509            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
510            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
511            None => {}
512        }
513    }
514
515    fn enter(&mut self, phase: RouteOpenPhase) {
516        self.finish();
517        self.phase = Some(phase);
518        self.phase_started_at = Instant::now();
519    }
520}
521
522struct RouteBindReservationGuard {
523    forwarding: Arc<ForwardingTable>,
524    endpoint: ModuleEndpointId,
525    relay_corr: u64,
526    armed: bool,
527}
528
529struct ModuleControlRpcGuard {
530    forwarding: Arc<ForwardingTable>,
531    endpoint: ModuleEndpointId,
532    corr: u64,
533    armed: bool,
534}
535
536impl ModuleControlRpcGuard {
537    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
538        Self {
539            forwarding,
540            endpoint,
541            corr,
542            armed: true,
543        }
544    }
545
546    fn disarm(&mut self) {
547        self.armed = false;
548    }
549}
550
551impl Drop for ModuleControlRpcGuard {
552    fn drop(&mut self) {
553        if self.armed {
554            let _ = self
555                .forwarding
556                .cancel_module_control_rpc(self.endpoint, self.corr);
557        }
558    }
559}
560
561impl RouteBindReservationGuard {
562    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
563        Self {
564            forwarding,
565            endpoint,
566            relay_corr,
567            armed: true,
568        }
569    }
570
571    fn release_and_disarm(&mut self) {
572        if !self.armed {
573            return;
574        }
575        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
576            self.endpoint,
577            self.relay_corr,
578            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
579        ) {
580            send_goodbye_target_best_effort(
581                &self.forwarding.counters(),
582                &target,
583                "canceled route.bind",
584            );
585        }
586        self.armed = false;
587    }
588
589    fn disarm(&mut self) {
590        self.armed = false;
591    }
592}
593
594impl Drop for RouteBindReservationGuard {
595    fn drop(&mut self) {
596        self.release_and_disarm();
597    }
598}
599
600/// Per-target-module circuit breaker around the `route.bind` relay.
601///
602/// The connection reader is serial per connection, so a module whose `on_bind`
603/// sits on the ack blocks every LATER frame on the connections that call it,
604/// including calls to unrelated modules. This does not make any module's bind
605/// fast; it stops the daemon paying the full budget again and again for a
606/// condition it has already observed.
607///
608/// State is keyed by TARGET MODULE and shared by every connection: a wedged
609/// module wedges everyone, so what one connection learned should protect the
610/// rest.
611///
612/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
613/// relay to that module has actually timed out, and is removed again when a
614/// relay is accepted or the module reconnects, so it cannot grow with traffic
615/// or with modules that behave.
616///
617/// # Why a `std` mutex here is not the head-of-line defect again
618///
619/// Acquisition never awaits. The critical section is a hash lookup plus a few
620/// integer updates, with no I/O and no `.await` inside it, so a reader task
621/// cannot be descheduled behind it the way it can behind
622/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
623/// primitive, held for the same kind of work, as the refusal counter this very
624/// path already increments.
625///
626/// It is also NOT on the data-plane splice path: only `route.open` and module
627/// registration touch it, so bound-route frames gain no state check and no
628/// contention.
629#[derive(Debug, Clone, Default)]
630pub(crate) struct RouteBindBreakers {
631    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
632}
633
634#[derive(Debug, Clone, Default)]
635pub(crate) struct RouteBindConcurrency {
636    modules: Arc<Mutex<HashMap<String, usize>>>,
637}
638
639struct RouteBindConcurrencyGuard {
640    concurrency: RouteBindConcurrency,
641    module_id: String,
642}
643
644impl RouteBindConcurrency {
645    /// Admit without waiting. Waiting here would move the bind stall from the
646    /// module reply to a semaphore and restore reader head-of-line blocking.
647    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
648        let mut modules = self
649            .modules
650            .lock()
651            .expect("route.bind concurrency mutex poisoned");
652        let in_flight = modules.entry(module_id.to_string()).or_default();
653        if *in_flight >= limit {
654            return Err(*in_flight);
655        }
656        *in_flight += 1;
657        Ok(RouteBindConcurrencyGuard {
658            concurrency: self.clone(),
659            module_id: module_id.to_string(),
660        })
661    }
662}
663
664impl Drop for RouteBindConcurrencyGuard {
665    fn drop(&mut self) {
666        let mut modules = self
667            .concurrency
668            .modules
669            .lock()
670            .expect("route.bind concurrency mutex poisoned");
671        let remove = {
672            let in_flight = modules
673                .get_mut(&self.module_id)
674                .expect("admitted route.bind has a concurrency entry");
675            *in_flight -= 1;
676            *in_flight == 0
677        };
678        if remove {
679            modules.remove(&self.module_id);
680        }
681    }
682}
683
684#[derive(Debug, Default)]
685struct ModuleBreakerState {
686    /// Relay timeouts observed with no accepted relay in between.
687    consecutive_timeouts: u32,
688    /// `Some` while the breaker is open: the instant the cooldown expires and
689    /// the next arrival may probe. `None` means closed.
690    cooldown_until: Option<Instant>,
691    /// A half-open probe has been admitted and has not settled yet. This is
692    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
693    /// that read the cooldown, so concurrent opens arriving at the moment the
694    /// cooldown expires cannot all decide that they are the probe.
695    probe_in_flight: Option<Arc<()>>,
696}
697
698/// What the breaker decided for one `route.open`, before any relay work.
699enum RouteBindAdmission<'a> {
700    Admitted {
701        guard: RouteBindBreakerGuard<'a>,
702        /// This open is the single half-open probe, so the transition is worth
703        /// one log line.
704        probe: bool,
705    },
706    Refused {
707        consecutive_timeouts: u32,
708        /// What is left of the cooldown. Zero when the refusal is because the
709        /// one probe is already in flight rather than because the cooldown has
710        /// not elapsed.
711        retry_in: Duration,
712        probe_in_flight: bool,
713    },
714}
715
716/// An outstanding admission, which must be told how its relay settled.
717///
718/// `Drop` settles it as inconclusive, so an early return between admission and
719/// the relay -- or the whole handler being cancelled when the client
720/// disconnects -- releases a half-open probe slot instead of leaving the
721/// breaker wedged half-open with no further probes.
722struct RouteBindBreakerGuard<'a> {
723    breakers: RouteBindBreakers,
724    module_id: &'a str,
725    probe_token: Option<Arc<()>>,
726    settled: bool,
727}
728
729impl RouteBindBreakerGuard<'_> {
730    /// The module answered within the budget and took the bind. THE ONLY
731    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
732    /// breaker, which is a transition worth logging.
733    fn record_accepted(&mut self) -> bool {
734        self.settled = true;
735        self.breakers.record_accepted(self.module_id)
736    }
737
738    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
739    /// COUNTS TOWARD OPENING.
740    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
741        self.settled = true;
742        self.breakers.record_timeout(
743            self.module_id,
744            self.probe_token.as_ref(),
745            threshold,
746            cooldown,
747        )
748    }
749
750    /// Everything else: the module REJECTED the bind, its connection went away
751    /// mid-relay, or the waiter was cancelled.
752    ///
753    /// None of these is evidence that a module is slow, and each already has
754    /// its own refusal with its own code. A module that rejects a bind in
755    /// microseconds is healthy and must never be convicted for it; a module
756    /// that died has said nothing about the module that replaces it. So these
757    /// neither increment nor reset the count -- they only release a probe slot.
758    fn record_inconclusive(&mut self) {
759        self.settled = true;
760        self.breakers
761            .record_inconclusive(self.module_id, self.probe_token.as_ref());
762    }
763}
764
765impl Drop for RouteBindBreakerGuard<'_> {
766    fn drop(&mut self) {
767        if !self.settled {
768            self.breakers
769                .record_inconclusive(self.module_id, self.probe_token.as_ref());
770        }
771    }
772}
773
774/// The breaker moved to open, reported so the caller can log it outside the
775/// lock. Opening is rare and load-bearing; the refusals that follow are
776/// frequent and are counted rather than logged.
777struct BreakerOpened {
778    consecutive_timeouts: u32,
779    /// True when a failed probe re-opened an already-open breaker, which reads
780    /// very differently in a log from a first opening.
781    reopened_after_probe: bool,
782}
783
784impl RouteBindBreakers {
785    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
786        self.modules
787            .lock()
788            .expect("route.bind breaker mutex poisoned")
789    }
790
791    /// Decide whether this `route.open` may attempt its relay. Takes the map
792    /// lock and nothing else, and never awaits.
793    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
794        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
795            probe: probe_token.is_some(),
796            guard: RouteBindBreakerGuard {
797                breakers: self.clone(),
798                module_id,
799                probe_token,
800                settled: false,
801            },
802        };
803
804        let mut modules = self.lock();
805        let Some(state) = modules.get_mut(module_id) else {
806            return admitted(None);
807        };
808        let Some(cooldown_until) = state.cooldown_until else {
809            return admitted(None);
810        };
811        if state.probe_in_flight.is_some() {
812            return RouteBindAdmission::Refused {
813                consecutive_timeouts: state.consecutive_timeouts,
814                retry_in: Duration::ZERO,
815                probe_in_flight: true,
816            };
817        }
818        let now = Instant::now();
819        if now < cooldown_until {
820            return RouteBindAdmission::Refused {
821                consecutive_timeouts: state.consecutive_timeouts,
822                retry_in: cooldown_until - now,
823                probe_in_flight: false,
824            };
825        }
826        let token = Arc::new(());
827        state.probe_in_flight = Some(Arc::clone(&token));
828        admitted(Some(token))
829    }
830
831    fn record_accepted(&self, module_id: &str) -> bool {
832        self.lock()
833            .remove(module_id)
834            .is_some_and(|state| state.cooldown_until.is_some())
835    }
836
837    fn record_timeout(
838        &self,
839        module_id: &str,
840        probe_token: Option<&Arc<()>>,
841        threshold: u32,
842        cooldown: Duration,
843    ) -> Option<BreakerOpened> {
844        let mut modules = self.lock();
845        let state = modules.entry(module_id.to_string()).or_default();
846        let was_open = state.cooldown_until.is_some();
847        let was_probe = Self::owns_probe(state, probe_token);
848        if was_probe {
849            state.probe_in_flight = None;
850        }
851        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
852        if state.consecutive_timeouts < threshold {
853            return None;
854        }
855        state.cooldown_until = Some(Instant::now() + cooldown);
856        Some(BreakerOpened {
857            consecutive_timeouts: state.consecutive_timeouts,
858            reopened_after_probe: was_open && was_probe,
859        })
860    }
861
862    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
863        // A relay can finish after the breaker was reset or after it opened
864        // again and started a new probe. Only the guard whose token matches the
865        // active probe may release it, so a late relay never frees a newer probe.
866        state
867            .probe_in_flight
868            .as_ref()
869            .zip(token)
870            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
871    }
872
873    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
874        if let Some(state) = self.lock().get_mut(module_id) {
875            if Self::owns_probe(state, probe_token) {
876                state.probe_in_flight = None;
877            }
878        }
879    }
880
881    /// Discard what was learned about a module, because the process it was
882    /// learned about is gone. Returns the discarded count when it was non-zero.
883    ///
884    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
885    /// `module_id` is a configuration identity that outlives any particular
886    /// child; what the breaker observed was the process behind the module
887    /// connection of the moment. When a new connection registers under that id
888    /// the verdict's subject no longer exists, so the verdict is stale by
889    /// construction rather than merely likely to be wrong. Keeping it would
890    /// apply a dead process's record to a live one, which is the same defect
891    /// class this breaker exists to stop the daemon committing.
892    ///
893    /// A half-open probe in flight is discarded with the rest: it was a
894    /// question about the old process.
895    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
896        self.lock()
897            .remove(module_id)
898            .map(|state| state.consecutive_timeouts)
899            .filter(|discarded| *discarded > 0)
900    }
901
902    /// Open breakers, for the `server.describe` counters object. `None` when
903    /// none is open, so the key stays absent rather than present-and-empty.
904    ///
905    /// This is the operator's answer to "is this module refusing instantly or
906    /// is it fine?", which look identical from a client that retries and then
907    /// succeeds.
908    fn open_snapshot(&self) -> Option<serde_json::Value> {
909        let now = Instant::now();
910        let modules = self.lock();
911        let open = modules
912            .iter()
913            .filter_map(|(module_id, state)| {
914                let cooldown_until = state.cooldown_until?;
915                Some((
916                    module_id.clone(),
917                    serde_json::json!({
918                        "consecutive_timeouts": state.consecutive_timeouts,
919                        "cooldown_remaining_ms":
920                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
921                        "probe_in_flight": state.probe_in_flight.is_some(),
922                    }),
923                ))
924            })
925            .collect::<serde_json::Map<String, serde_json::Value>>();
926        (!open.is_empty()).then_some(serde_json::Value::Object(open))
927    }
928}
929
930impl ControlHandler {
931    pub fn new(registry: Arc<Registry>) -> Self {
932        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
933    }
934
935    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
936        let counters = forwarding.counters();
937        // Taken from the forwarding table rather than created here, so that the
938        // breaker a `route.open` consults is the same one a module's
939        // registration resets, however many handlers are built over one table.
940        let route_bind_breakers = forwarding.route_bind_breakers();
941        let route_bind_concurrency = forwarding.route_bind_concurrency();
942        let route_outages = forwarding.route_outages();
943        Self {
944            registry,
945            activity: None,
946            activity_streams: Arc::new(Mutex::new(HashMap::new())),
947            forwarding,
948            process_liveness: None,
949            supervisor: SupervisorHandle::new(),
950            subc_capabilities: Arc::from([
951                CAP_MANIFEST_REGISTRATION.to_string(),
952                CAP_CHANNEL_LIFECYCLE.to_string(),
953                CAP_PING_PONG.to_string(),
954                CAP_SESSION_ATTACH.to_string(),
955                CAP_ADMISSION_FACTS_RELAY.to_string(),
956                CAP_SCOPES_V1.to_string(),
957                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
958                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
959                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
960                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
961                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
962                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
963                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
964                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
965                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
966            ]),
967            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
968            route_bind_relay_timeouts: BTreeMap::new(),
969            route_bind_breakers,
970            route_bind_concurrency,
971            route_outages,
972            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
973            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
974            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
975            storage_config: None,
976            machine_id: None,
977            admission_facts_carrier_module_id: None,
978            admission_facts_targets: None,
979            scopes: Arc::new(RwLock::new(ScopeTable::new(
980                crate::daemon_config::default_scope_authority_owners(),
981            ))),
982            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
983            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
984            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
985            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
986            #[cfg(test)]
987            caller_lookup: None,
988            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
989            rescan: None,
990            connected_clients: ConnectedClients::new(),
991            counters,
992            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
993            daemon_shutdown: DaemonShutdownFlag::default(),
994            daemon_provenance: DaemonProvenanceFacts::default(),
995            #[cfg(test)]
996            control_dispatch_delay: None,
997            #[cfg(test)]
998            provenance_probe_override: None,
999            #[cfg(test)]
1000            route_bind_relay_sent_hold: None,
1001            #[cfg(test)]
1002            hello_registry_to_forwarding_hold: None,
1003        }
1004    }
1005
1006    /// Set the central storage policy: registering modules then receive their
1007    /// resolved storage descriptor in HELLO_ACK.
1008    pub fn with_storage_config(
1009        mut self,
1010        storage_config: Option<crate::daemon_config::StorageConfig>,
1011    ) -> Self {
1012        self.storage_config = storage_config;
1013        self
1014    }
1015
1016    /// Open the ledger on its own writer thread; an unavailable sink does not
1017    /// prevent the daemon from serving routing or supervisor requests.
1018    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1019        self.activity = Some(crate::activity::ActivityLedger::open(
1020            run_dir.join("activity.jsonl"),
1021            incarnation,
1022        ));
1023        self.supervisor.set_activity(self.activity.clone());
1024        self
1025    }
1026
1027    /// Set the machine id served to every registering module (HELLO_ACK) and on
1028    /// `server.describe`.
1029    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1030        self.machine_id = machine_id;
1031        self
1032    }
1033
1034    /// Configure the exact reserved module and target ids permitted to relay
1035    /// opaque admission facts. Config-file loading validates this authority;
1036    /// this builder keeps the same policy available to embedded test daemons.
1037    pub fn with_admission_facts_config(
1038        mut self,
1039        carrier_module_id: Option<String>,
1040        targets: Option<Vec<String>>,
1041    ) -> Self {
1042        self.admission_facts_carrier_module_id = carrier_module_id;
1043        self.admission_facts_targets = targets;
1044        self
1045    }
1046
1047    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1048    /// Replaces the scope table with an empty one under the new list, so call it
1049    /// while building the handler, before any module can sync.
1050    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1051        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1052        self.scope_authority_owners = owners;
1053        self
1054    }
1055
1056    /// Set the startup policy before installing the rescan context or serving requests.
1057    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1058        self.privileged_operator_prompt = mode;
1059        self
1060    }
1061
1062    /// Override the route.bind relay timeout. Used by tests that assert the
1063    /// timeout path so they don't block on the production-safe default.
1064    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1065        self.route_bind_relay_timeout = timeout;
1066        self
1067    }
1068
1069    /// Install per-module route.bind relay budget overrides. A module id
1070    /// listed here wins over the daemon-wide default set via
1071    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1072    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1073    /// the same `Duration` the bind path will use.
1074    pub fn with_route_bind_relay_timeouts(
1075        mut self,
1076        timeouts: impl IntoIterator<Item = (String, Duration)>,
1077    ) -> Self {
1078        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1079        self
1080    }
1081
1082    /// Resolve the route.bind relay budget for a specific target module id.
1083    /// Per-module overrides win; the daemon-wide value (set via
1084    /// `with_route_bind_relay_timeout` or the built-in default) is the
1085    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1086    /// the same resolution `handle_route_open` will use.
1087    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1088        self.route_bind_relay_timeouts
1089            .get(module_id)
1090            .copied()
1091            .unwrap_or(self.route_bind_relay_timeout)
1092    }
1093
1094    /// Override the per-module bind-relay breaker policy.
1095    ///
1096    /// Used by tests, which cannot spend three production budgets opening a
1097    /// breaker or twenty seconds waiting for its cooldown. The production
1098    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1099    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1100    /// reasoning for the numbers.
1101    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1102        self.route_bind_breaker_threshold = threshold.max(1);
1103        self.route_bind_breaker_cooldown = cooldown;
1104        self
1105    }
1106
1107    #[cfg(test)]
1108    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1109        self.health_probe_timeout = timeout;
1110        self
1111    }
1112
1113    #[cfg(test)]
1114    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1115        self.control_dispatch_delay = Some(delay);
1116        self
1117    }
1118
1119    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1120    /// announce it is parked, once to be released. Build the barrier for two
1121    /// parties, the test being the other one.
1122    #[cfg(test)]
1123    pub(crate) fn with_route_bind_relay_sent_hold(
1124        mut self,
1125        hold: Arc<tokio::sync::Barrier>,
1126    ) -> Self {
1127        self.route_bind_relay_sent_hold = Some(hold);
1128        self
1129    }
1130
1131    /// A HELLO waits on `hold` twice between reserving its registry entry and
1132    /// installing its forwarding endpoint: once to announce it is parked, once
1133    /// to be released. Build the barrier for two parties, the test being the
1134    /// other one, and run the HELLO on a thread of its own.
1135    #[cfg(test)]
1136    pub(crate) fn with_hello_registry_to_forwarding_hold(
1137        mut self,
1138        hold: Arc<std::sync::Barrier>,
1139    ) -> Self {
1140        self.hello_registry_to_forwarding_hold = Some(hold);
1141        self
1142    }
1143
1144    #[cfg(test)]
1145    fn hold_between_registry_and_forwarding(&self) {
1146        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1147            hold.wait();
1148            hold.wait();
1149        }
1150    }
1151
1152    pub fn with_process_liveness(
1153        mut self,
1154        process_liveness: Arc<dyn ModuleProcessLiveness>,
1155    ) -> Self {
1156        self.process_liveness = Some(process_liveness);
1157        self
1158    }
1159
1160    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1161        self.supervisor = supervisor;
1162        self.supervisor.set_activity(self.activity.clone());
1163        self
1164    }
1165
1166    pub fn with_daemon_provenance(
1167        mut self,
1168        pid: u32,
1169        started_at_ms: u64,
1170        executable_path: Option<PathBuf>,
1171        build_git_sha: Option<String>,
1172        build_lock_digest: Option<String>,
1173    ) -> Self {
1174        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1175        let process_start_time = process_start_time(pid);
1176        self.daemon_provenance = DaemonProvenanceFacts {
1177            build: DaemonBuildProvenance {
1178                build_git_sha,
1179                build_lock_digest,
1180            },
1181            pid: Some(pid),
1182            started_at_ms: Some(started_at_ms),
1183            start_clock: None,
1184            executable_path,
1185            executable_identity,
1186            process_start_time,
1187            probe: ExecutableIdentityProbe::default(),
1188        };
1189        self
1190    }
1191
1192    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1193        self.daemon_provenance.start_clock = Some(clock);
1194        self
1195    }
1196
1197    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1198    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1199        self.scopes
1200            .write()
1201            .unwrap_or_else(|poisoned| poisoned.into_inner())
1202            .set_wall_clock(Arc::new(clock));
1203        self
1204    }
1205
1206    #[cfg(test)]
1207    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1208        self.provenance_probe_override = Some(result);
1209        self
1210    }
1211
1212    /// Install the configured module set and its reserved capability bindings.
1213    /// Bindings are configuration-scoped and may point at a provider that has not
1214    /// been installed yet, so this does not require the bound module to exist.
1215    pub fn with_capability_config(
1216        self,
1217        modules: impl IntoIterator<Item = (String, bool)>,
1218        reserved_capabilities: BTreeMap<String, String>,
1219    ) -> Self {
1220        self.capability_evaluator
1221            .configure(modules, reserved_capabilities);
1222        self
1223    }
1224
1225    /// Share the supervisor's daemon-shutdown flag with this handler, so
1226    /// requirement changes after the daemon has begun its announced shutdown
1227    /// are not reported as faults.
1228    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1229        self.daemon_shutdown = flag;
1230        self
1231    }
1232
1233    pub fn with_supervisor_rescan(
1234        mut self,
1235        supervisor: Supervisor,
1236        config_path: impl Into<PathBuf>,
1237        configured_port: Option<u16>,
1238    ) -> Self {
1239        self.rescan = Some(SupervisorRescanContext {
1240            supervisor,
1241            config_path: config_path.into(),
1242            configured_port,
1243            storage_config: self.storage_config.clone(),
1244            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1245            admission_facts_targets: self.admission_facts_targets.clone(),
1246            scope_authority_owners: self.scope_authority_owners.clone(),
1247            privileged_operator_prompt: self.privileged_operator_prompt,
1248            generation: Arc::default(),
1249        });
1250        self
1251    }
1252
1253    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1254        self.connected_clients = connected_clients;
1255        self
1256    }
1257
1258    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1259        Arc::clone(&self.forwarding)
1260    }
1261
1262    pub(crate) fn counters(&self) -> DaemonCounters {
1263        self.counters.clone()
1264    }
1265
1266    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1267    /// requirement event without depending on an operator polling a status command.
1268    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1269        tokio::spawn(async move {
1270            loop {
1271                self.capability_evaluator
1272                    .wait_for_change_or_deadline()
1273                    .await;
1274                self.refresh_capability_requirements();
1275            }
1276        });
1277    }
1278
1279    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1280    /// deadline passed during sleep is enforced on the first tick after wake.
1281    /// While no live scope carries a deadline the loop waits for an accepted
1282    /// scope change instead of ticking, so a daemon without deadlines never
1283    /// wakes for expiry.
1284    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1285        tokio::spawn(async move {
1286            loop {
1287                let has_deadlines = self
1288                    .scopes
1289                    .read()
1290                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1291                    .has_deadlines();
1292                if !has_deadlines {
1293                    self.scope_deadline_added.notified().await;
1294                    continue;
1295                }
1296                tokio::time::sleep(Duration::from_secs(1)).await;
1297                if let Err(error) = self.sweep_expired_scopes() {
1298                    warn!(%error, "scope expiry sweep failed");
1299                }
1300            }
1301        });
1302    }
1303
1304    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1305        let mut table = self
1306            .scopes
1307            .write()
1308            .unwrap_or_else(|poisoned| poisoned.into_inner());
1309        let swept = table.sweep_expired();
1310        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1311        drop(table);
1312        self.close_scope_drained_routes(drained);
1313        Ok(())
1314    }
1315
1316    fn publish_scope_changes(
1317        &self,
1318        changes: &[crate::scopes::ScopeTagChange],
1319        expired: &[crate::scopes::ScopeExpired],
1320    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1321        if changes.is_empty() {
1322            return Ok(Vec::new());
1323        }
1324        let drained = self
1325            .forwarding
1326            .publish_scope_changes(changes)
1327            .map_err(RouterError::Forwarding)?;
1328        let mut counts = HashMap::new();
1329        for route in &drained {
1330            *counts
1331                .entry((
1332                    route.scope.owner.as_str(),
1333                    route.scope.scope_ref.as_str(),
1334                    route.scope.tag.scope_epoch,
1335                ))
1336                .or_insert(0usize) += 1;
1337        }
1338        for scope in expired {
1339            let routes_closed = counts
1340                .get(&(
1341                    scope.owner.as_str(),
1342                    scope.scope_ref.as_str(),
1343                    scope.scope_epoch,
1344                ))
1345                .copied()
1346                .unwrap_or(0);
1347            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1348                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1349                routes_closed, "scope expired");
1350        }
1351        Ok(drained)
1352    }
1353
1354    fn runtime_capability_snapshot(
1355        &self,
1356    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1357        let runtime = self
1358            .supervisor
1359            .list()
1360            .into_iter()
1361            .map(|module| {
1362                let status = module.status().map_err(|err| {
1363                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1364                })?;
1365                Ok(RuntimeModule {
1366                    module_id: status.module_id,
1367                    state: status.state,
1368                    enabled: status.enabled,
1369                })
1370            })
1371            .collect::<Result<Vec<_>, RouterError>>()?;
1372        // Include every registration, published or not. The capability
1373        // evaluator counts a module as providing its capabilities as soon as it
1374        // has registered, even before `catalog.list` shows it, so a HELLO in
1375        // progress does not change other modules' required-capability
1376        // verdicts. See `Registry::list_registered_modules`.
1377        let registrations = self.registry.list_registered_modules().map_err(|err| {
1378            RouterError::backend(
1379                0,
1380                0,
1381                format!("failed to list capability registrations: {err}"),
1382            )
1383        })?;
1384        let registrations = registrations
1385            .into_iter()
1386            .map(|registration| RegisteredModule {
1387                module_id: registration.manifest.module_id,
1388                module_version: registration.manifest.module_version,
1389                capabilities: registration.manifest.capabilities,
1390            })
1391            .collect();
1392        Ok((runtime, registrations))
1393    }
1394
1395    /// The capability side effects of a module becoming the active registration
1396    /// for its id: cache its manifest (warning if its claims drifted), run the
1397    /// deny census when its declarations call for one, and recompute the
1398    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1399    /// candidate's does not, and the supervisor does it at promotion instead,
1400    /// through [`crate::supervise::SwapPromotionObserver`].
1401    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1402        let cached_registration = RegisteredModule {
1403            module_id: registration.manifest.module_id.clone(),
1404            module_version: registration.manifest.module_version.clone(),
1405            capabilities: registration.manifest.capabilities.clone(),
1406        };
1407        if self.capability_evaluator.record_hello(&cached_registration) {
1408            warn!(
1409                module_id = %cached_registration.module_id,
1410                "capability claims drifted from the cached manifest"
1411            );
1412        }
1413        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1414            self.enforce_capability_denies();
1415        }
1416        self.refresh_capability_requirements();
1417    }
1418
1419    /// Point the shared supervisor handle at this handler for swap promotions.
1420    /// Called wherever a handler is put behind the `Arc` the router serves, so
1421    /// it can be held weakly.
1422    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1423        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1424            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1425        self.supervisor.set_swap_promotion_observer(observer);
1426    }
1427
1428    pub fn refresh_capability_requirements(&self) {
1429        match self.runtime_capability_snapshot() {
1430            Ok((runtime, registrations)) => {
1431                log_requirement_events(
1432                    self.capability_evaluator
1433                        .evaluate_now(&runtime, &registrations),
1434                    &self.daemon_shutdown,
1435                );
1436            }
1437            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1438        }
1439    }
1440
1441    /// Reconcile only live, attested route bindings after a capability deny edge
1442    /// or target claim was added. This is deliberately a control-plane census:
1443    /// the opaque forwarding hot path must not grow a per-frame capability check.
1444    fn enforce_capability_denies(&self) {
1445        let registrations = match self.registry.list_registered_modules() {
1446            Ok(snapshot) => snapshot,
1447            Err(err) => {
1448                warn!(error = %err, "failed to read registrations for capability deny census");
1449                return;
1450            }
1451        };
1452        let manifests = registrations
1453            .into_iter()
1454            .map(|registration| {
1455                (
1456                    registration.manifest.module_id.clone(),
1457                    registration.manifest,
1458                )
1459            })
1460            .collect::<BTreeMap<_, _>>();
1461        let census = match self.forwarding.route_census(None) {
1462            Ok(census) => census,
1463            Err(err) => {
1464                warn!(error = %err, "failed to read route census for capability deny enforcement");
1465                return;
1466            }
1467        };
1468
1469        for (target_module_id, routes) in census {
1470            let Some(target_manifest) = manifests.get(&target_module_id) else {
1471                continue;
1472            };
1473            let mut closed_routes = Vec::new();
1474            let mut module_goodbyes = Vec::new();
1475            for route in routes {
1476                let Principal::Reserved {
1477                    module_id: opening_module_id,
1478                } = &route.principal
1479                else {
1480                    continue;
1481                };
1482                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1483                    continue;
1484                };
1485                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1486                    continue;
1487                };
1488
1489                match self.forwarding.release_client_route(
1490                    route.goodbye_target.connection_id,
1491                    route.goodbye_target.channel,
1492                    route.goodbye_target.epoch,
1493                ) {
1494                    Ok(RouteRelease::Removed(module_goodbye)) => {
1495                        warn!(
1496                            opening_module_id,
1497                            target_module_id,
1498                            capability,
1499                            "force-closing route because an attested capability deny edge now matches"
1500                        );
1501                        closed_routes.push(route);
1502                        module_goodbyes.push(module_goodbye);
1503                    }
1504                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1505                    Err(err) => warn!(
1506                        opening_module_id,
1507                        target_module_id,
1508                        capability,
1509                        error = %err,
1510                        "failed to force-close capability-denied route"
1511                    ),
1512                }
1513            }
1514
1515            if closed_routes.is_empty() {
1516                continue;
1517            }
1518            send_route_control_pushes(
1519                &self.forwarding,
1520                closed_routes,
1521                ClientControlPush::RouteClosed {
1522                    module_id: target_module_id,
1523                    channels: Vec::new(),
1524                    reason: RouteCloseReason::CapabilityDenied,
1525                    drained: false,
1526                    abandoned: 0,
1527                    excluded_subscriptions: 0,
1528                    terminal: Some(false),
1529                },
1530            );
1531            self.emit_route_goodbyes(module_goodbyes);
1532        }
1533    }
1534
1535    /// Why a registered module is not accepting new route binds, or `None` when
1536    /// it is. This is the module's effective readiness: its declared readiness
1537    /// first, then every `need: required` capability it declares evaluating to
1538    /// `provided`. `route.open` and `catalog.list` both read it here so the
1539    /// catalog never reports a module routable that `route.open` would refuse.
1540    fn not_ready_reason(
1541        &self,
1542        registration: &crate::registry::ModuleRegistration,
1543    ) -> Option<NotReadyReason> {
1544        if !registration.ready {
1545            return Some(NotReadyReason {
1546                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1547                capability: None,
1548            });
1549        }
1550        self.first_unprovided_required_capability(registration)
1551            .map(|capability| NotReadyReason {
1552                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1553                capability: Some(capability),
1554            })
1555    }
1556
1557    /// The lexicographically first capability this registration declares
1558    /// `need: required` whose evaluator verdict is not `provided`.
1559    ///
1560    /// The verdicts are the capability evaluator's own; nothing here decides
1561    /// what "provided" means. The evaluator counts a capability provided as
1562    /// soon as a module claiming it has REGISTERED, not once that module is
1563    /// ready. That distinction is what keeps two modules that require each
1564    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1565    /// is ready", each would wait for the other to become ready first and
1566    /// neither ever would. Do not tighten it to readiness.
1567    ///
1568    /// A required capability with no verdict at all means this registration's
1569    /// HELLO or catalog.update landed after the last recompute; recompute once
1570    /// rather than let a missing verdict read as either answer. If it is still
1571    /// missing (the recompute itself failed) the capability counts as
1572    /// unprovided: the refusal is retryable, and routing a module whose
1573    /// required provider is unknown is the outcome this check exists to stop.
1574    fn first_unprovided_required_capability(
1575        &self,
1576        registration: &crate::registry::ModuleRegistration,
1577    ) -> Option<String> {
1578        let required = registration
1579            .manifest
1580            .capabilities
1581            .iter()
1582            .flat_map(|declarations| declarations.requires.iter())
1583            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1584            .map(|requirement| requirement.capability.as_str())
1585            .collect::<BTreeSet<_>>();
1586        if required.is_empty() {
1587            return None;
1588        }
1589        let module_id = registration.manifest.module_id.as_str();
1590        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1591        if required
1592            .iter()
1593            .any(|capability| verdict(capability).is_none())
1594        {
1595            self.refresh_capability_requirements();
1596        }
1597        required
1598            .into_iter()
1599            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1600            .map(str::to_string)
1601    }
1602
1603    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1604        self.capability_evaluator
1605            .statuses()
1606            .into_iter()
1607            .map(capability_requirement_status)
1608            .collect()
1609    }
1610
1611    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1612    /// registration-release watch. The signal is what the supervisor waits on
1613    /// before spawning a replacement, so it must only fire once forwarding
1614    /// teardown is also done (see [`Self::cleanup_connection`] /
1615    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1616    /// admitted but its forwarding endpoint could not be installed.
1617    fn deregister_connection(
1618        &self,
1619        connection_id: ConnectionId,
1620        reason: RegistrationEndReason,
1621    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1622        self.registry
1623            .deregister_connection_with_reason(connection_id, reason)
1624    }
1625
1626    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1627        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1628            return None;
1629        }
1630        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1631            parse_client_control_request(&frame.body)
1632        else {
1633            return None;
1634        };
1635        Some(target_module_id(&target).to_string())
1636    }
1637
1638    pub(crate) fn route_open_capacity_refusal(
1639        &self,
1640        ctx: &RouteCtx,
1641        frame: &Frame,
1642        target_module_id: &str,
1643        in_flight: usize,
1644        limit: usize,
1645    ) -> Result<Frame, RouterError> {
1646        self.route_open_admission_refusal_frame(
1647            ctx,
1648            frame,
1649            target_module_id,
1650            "open_admission_full",
1651            (in_flight, limit),
1652            format!(
1653                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1654            ),
1655        )
1656    }
1657
1658    fn route_open_target_capacity_refusal(
1659        &self,
1660        ctx: &RouteCtx,
1661        frame: &Frame,
1662        target_module_id: &str,
1663        in_flight: usize,
1664    ) -> Result<Frame, RouterError> {
1665        self.route_open_admission_refusal_frame(
1666            ctx,
1667            frame,
1668            target_module_id,
1669            "target_binds_full",
1670            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1671            format!(
1672                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1673            ),
1674        )
1675    }
1676
1677    /// Admission pressure clears as existing binds settle, so its refusal must
1678    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1679    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1680    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1681    /// cannot currently reach its target; `module_timeout` would falsely claim
1682    /// that a wait expired. A new, cleaner code would be terminal to deployed
1683    /// clients, so it requires a client-tolerance rollout before daemon emission.
1684    fn route_open_admission_refusal_frame(
1685        &self,
1686        ctx: &RouteCtx,
1687        frame: &Frame,
1688        target_module_id: &str,
1689        reason: &'static str,
1690        (in_flight, limit): (usize, usize),
1691        message: impl Into<String>,
1692    ) -> Result<Frame, RouterError> {
1693        let code = error_codes::TARGET_UNAVAILABLE;
1694        self.counters.increment_route_open_refused(code);
1695        info!(
1696            target: "control",
1697            code,
1698            reason,
1699            module_id = ?target_module_id,
1700            connection_id = ctx.connection_id.get(),
1701            in_flight,
1702            limit,
1703            "route.open refused"
1704        );
1705        control_error_frame(frame, code, message.into())
1706    }
1707
1708    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1709    ///
1710    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1711    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1712    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1713    #[cfg(test)]
1714    pub fn handle_control(
1715        &self,
1716        connection_id: ConnectionId,
1717        frame: Frame,
1718    ) -> Result<Vec<Frame>, RouterError> {
1719        match frame.header.ty {
1720            FrameType::Ping => Ok(vec![pong(&frame)?]),
1721            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1722            FrameType::Goodbye => self.handle_goodbye(connection_id),
1723            ty => Ok(vec![control_error_frame(
1724                &frame,
1725                "unsupported_control_frame",
1726                format!("unsupported channel-0 frame {ty:?}"),
1727            )?]),
1728        }
1729    }
1730
1731    pub async fn handle_control_frame(
1732        &self,
1733        ctx: &RouteCtx,
1734        frame: Frame,
1735    ) -> Result<Vec<Frame>, RouterError> {
1736        self.handle_control_frame_timed(ctx, frame, None).await
1737    }
1738
1739    pub(crate) fn record_caller_endpoints(
1740        &self,
1741        connection: ConnectionId,
1742        endpoints: Option<(SocketAddr, SocketAddr)>,
1743    ) {
1744        let mut callers = self
1745            .caller_endpoints
1746            .lock()
1747            .unwrap_or_else(|p| p.into_inner());
1748        callers.remove(&connection);
1749        if let Some(endpoints) = endpoints {
1750            callers.insert(connection, endpoints);
1751        }
1752    }
1753
1754    #[cfg(test)]
1755    pub(crate) fn with_caller_lookup_for_test(
1756        mut self,
1757        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1758    ) -> Self {
1759        self.caller_lookup = Some(Arc::new(lookup));
1760        self
1761    }
1762
1763    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1764        let endpoints = self
1765            .caller_endpoints
1766            .lock()
1767            .unwrap_or_else(|p| p.into_inner())
1768            .get(&connection)
1769            .copied();
1770        #[cfg(test)]
1771        let lookup = self.caller_lookup.clone();
1772        tokio::task::spawn_blocking(move || {
1773            #[cfg(test)]
1774            if let Some(lookup) = lookup {
1775                return lookup(endpoints);
1776            }
1777            endpoints
1778                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1779                .unwrap_or_default()
1780        })
1781        .await
1782        .unwrap_or_default()
1783    }
1784
1785    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1786        frame.header.channel == 0
1787            && frame.header.ty == FrameType::Request
1788            && self
1789                .forwarding
1790                .module_endpoint_for_connection(ctx.connection_id)
1791                .is_ok_and(|endpoint| endpoint.is_none())
1792            && parse_client_control_request(&frame.body)
1793                .ok()
1794                .is_some_and(|request| privileged_operation(&request).is_some())
1795    }
1796
1797    pub(crate) async fn handle_control_frame_timed(
1798        &self,
1799        ctx: &RouteCtx,
1800        frame: Frame,
1801        dispatch_started_at: Option<StdInstant>,
1802    ) -> Result<Vec<Frame>, RouterError> {
1803        match frame.header.ty {
1804            FrameType::Ping => Ok(vec![pong(&frame)?]),
1805            FrameType::Hello => {
1806                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1807            }
1808            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1809            FrameType::Cancel => {
1810                // A Cancel on channel 0 names a waiting operator.confirm or a
1811                // subscription. Successful cancellation has no ordinary reply.
1812                if self
1813                    .forwarding
1814                    .operator_confirms()
1815                    .cancel(ctx.connection_id, frame.header.corr)
1816                    || self
1817                        .supervisor
1818                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1819                    || self.cancel_activity_subscription(ctx, &frame)?
1820                {
1821                    Ok(Vec::new())
1822                } else {
1823                    Ok(vec![control_error_frame(
1824                        &frame,
1825                        "unknown_subscription",
1826                        "no subscription has this correlation id",
1827                    )?])
1828                }
1829            }
1830            FrameType::Request => {
1831                // This additive operation is not part of the existing exhaustive
1832                // module-control enum. Probe the op before decoding that enum.
1833                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1834                if op
1835                    .as_ref()
1836                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1837                {
1838                    return self.handle_activity_record(ctx, frame).await;
1839                }
1840                if op.as_ref().is_some_and(|probe| {
1841                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1842                }) {
1843                    return self.handle_activity_read(ctx, frame).await;
1844                }
1845                if op
1846                    .as_ref()
1847                    .is_some_and(|probe| probe.op == "operator.confirm")
1848                {
1849                    return self.handle_operator_confirm(ctx, frame);
1850                }
1851                if op
1852                    .as_ref()
1853                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1854                {
1855                    return self.handle_scope_served_by(ctx.connection_id, frame);
1856                }
1857                if self
1858                    .forwarding
1859                    .module_endpoint_for_connection(ctx.connection_id)
1860                    .map_err(RouterError::Forwarding)?
1861                    .is_some()
1862                {
1863                    if !is_known_module_request_op(&frame.body) {
1864                        return Ok(vec![control_error_frame(
1865                            &frame,
1866                            "unsupported_control_frame",
1867                            "module-originated channel-0 REQUEST is not supported",
1868                        )?]);
1869                    }
1870                    let request = match parse_module_control_request_from_module(&frame.body) {
1871                        Ok(request) => request,
1872                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1873                            return Ok(vec![control_error_frame(
1874                                &frame,
1875                                "unsupported_control_frame",
1876                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1877                            )?])
1878                        }
1879                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1880                            return Ok(vec![control_error_frame(
1881                                &frame,
1882                                "invalid_control_body",
1883                                format!("malformed module control body: {err}"),
1884                            )?])
1885                        }
1886                    };
1887                    let op = module_control_request_op(&request);
1888                    let corr = frame.header.corr;
1889                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1890                    let result =
1891                        self.handle_module_control_request(ctx.connection_id, frame, request);
1892                    log_slow_control_dispatch(
1893                        dispatch_started_at,
1894                        op,
1895                        ctx.connection_id,
1896                        corr,
1897                        None,
1898                    );
1899                    return result;
1900                }
1901
1902                if is_known_module_request_op(&frame.body) {
1903                    return Ok(vec![control_error_frame(
1904                        &frame,
1905                        "not_registered",
1906                        "catalog.update requires an active module registration owned by this connection",
1907                    )?]);
1908                }
1909
1910                let request = match parse_client_control_request(&frame.body) {
1911                    Ok(request) => request,
1912                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1913                        return Ok(vec![control_error_frame(
1914                            &frame,
1915                            "unknown_control_op",
1916                            format!("unknown client control op: {err}"),
1917                        )?])
1918                    }
1919                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1920                        return Ok(vec![control_error_frame(
1921                            &frame,
1922                            "invalid_control_body",
1923                            format!("malformed client control body: {err}"),
1924                        )?])
1925                    }
1926                };
1927                let op = client_control_request_op(&request);
1928                let corr = frame.header.corr;
1929                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1930                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1931                #[cfg(test)]
1932                if let Some(delay) = self.control_dispatch_delay {
1933                    tokio::time::sleep(delay).await;
1934                }
1935                let result = self
1936                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1937                    .await;
1938                if let Some(timings) = route_open_timings.as_mut() {
1939                    timings.finish();
1940                }
1941                log_slow_control_dispatch(
1942                    dispatch_started_at,
1943                    op,
1944                    ctx.connection_id,
1945                    corr,
1946                    route_open_timings.as_ref(),
1947                );
1948                result
1949            }
1950            FrameType::Push => {
1951                let Some(endpoint) = self
1952                    .forwarding
1953                    .module_endpoint_for_connection(ctx.connection_id)
1954                    .map_err(RouterError::Forwarding)?
1955                else {
1956                    return Ok(vec![control_error_frame(
1957                        &frame,
1958                        "unsupported_control_frame",
1959                        "client-originated channel-0 PUSH is not supported",
1960                    )?]);
1961                };
1962                self.handle_status_update(endpoint, frame)
1963            }
1964            FrameType::Response | FrameType::Error
1965                if self
1966                    .forwarding
1967                    .module_endpoint_for_connection(ctx.connection_id)
1968                    .map_err(RouterError::Forwarding)?
1969                    .is_some() =>
1970            {
1971                self.handle_module_relay_response(ctx.connection_id, frame)
1972            }
1973            ty => Ok(vec![control_error_frame(
1974                &frame,
1975                "unsupported_control_frame",
1976                format!("unsupported channel-0 frame {ty:?}"),
1977            )?]),
1978        }
1979    }
1980
1981    pub fn cleanup_connection(
1982        &self,
1983        connection_id: ConnectionId,
1984    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1985        self.caller_endpoints
1986            .lock()
1987            .unwrap_or_else(|p| p.into_inner())
1988            .remove(&connection_id);
1989        self.forwarding
1990            .operator_confirms()
1991            .connection_closed(connection_id);
1992        self.cleanup_connection_with_end_reason(
1993            connection_id,
1994            RegistrationEndReason::ConnectionClosed,
1995        )
1996    }
1997
1998    fn cleanup_connection_with_end_reason(
1999        &self,
2000        connection_id: ConnectionId,
2001        requested_reason: RegistrationEndReason,
2002    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2003        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2004            self.registry
2005                .get_module_by_connection(connection_id)?
2006                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2007                .and_then(|module| module.registration_end_reason().ok().flatten())
2008                .unwrap_or(requested_reason)
2009        } else {
2010            requested_reason
2011        };
2012        let crash_closed = self
2013            .registry
2014            .get_module_by_connection(connection_id)?
2015            .and_then(|registration| {
2016                self.forwarding
2017                    .module_endpoint_for_connection(connection_id)
2018                    .ok()
2019                    .flatten()
2020                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2021                    .map(|routes| (registration.manifest.module_id, routes))
2022            });
2023        let crash_closed = crash_closed.map(|(module_id, routes)| {
2024            let terminal = match self.supervisor.get(&module_id) {
2025                None => false,
2026                Some(module) => match module.will_recover_after_connection_loss() {
2027                    Ok(will_recover) => !will_recover,
2028                    Err(err) => {
2029                        warn!(
2030                            %module_id,
2031                            error = %err,
2032                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2033                        );
2034                        false
2035                    }
2036                },
2037            };
2038            // The forwarding table gates all providers at the start of daemon
2039            // shutdown, before their connections are closed. An ordinary
2040            // module disconnect still reports crash if that gate is not set.
2041            let (reason, drained) = match self
2042                .forwarding
2043                .daemon_drain_result_for_connection(connection_id)
2044            {
2045                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2046                Ok(None) => (RouteCloseReason::Crash, false),
2047                Err(err) => {
2048                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2049                    (RouteCloseReason::Crash, false)
2050                }
2051            };
2052            (module_id, routes, reason, drained, terminal)
2053        });
2054        let registrations = self.deregister_connection(connection_id, end_reason);
2055        let cleanup = if crash_closed.is_some() {
2056            self.forwarding.cleanup_connection_counted(connection_id)
2057        } else {
2058            // A module connection's teardown needs the count of abandoned
2059            // route.bind relays for its route.closed notice below. Any other
2060            // connection, such as a client's, sends no such notice and needs
2061            // only its routes released, so it uses the route-only wrapper and
2062            // reports zero.
2063            self.forwarding
2064                .cleanup_connection(connection_id)
2065                .map(|released| crate::forwarding::ConnectionCleanup {
2066                    released,
2067                    abandoned_relays: 0,
2068                })
2069        };
2070        // The route.closed push waits for forwarding teardown because only
2071        // teardown knows how many pending route.bind relays it aborted. It still
2072        // goes out before the GOODBYEs for the released routes, and its targets
2073        // were captured above, before teardown removed those routes.
2074        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2075            let abandoned = cleanup
2076                .as_ref()
2077                .map_or(0, |cleanup| cleanup.abandoned_relays);
2078            send_route_control_pushes(
2079                &self.forwarding,
2080                routes,
2081                ClientControlPush::RouteClosed {
2082                    module_id,
2083                    channels: Vec::new(),
2084                    reason,
2085                    drained,
2086                    abandoned,
2087                    excluded_subscriptions: 0,
2088                    terminal: Some(terminal),
2089                },
2090            );
2091        }
2092        if let Ok(cleanup) = cleanup {
2093            self.emit_route_goodbyes(cleanup.released);
2094        }
2095        // Signal the registration-release watch only now that BOTH registry and
2096        // forwarding teardown are done, so a supervisor waiting to spawn a
2097        // replacement never observes release while old routes still exist.
2098        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2099            crate::supervise::notify_registration_release();
2100            self.capability_evaluator.wake_deadline_loop();
2101            self.refresh_capability_requirements();
2102        }
2103        self.supervisor.remove_spawn_subscribers(connection_id);
2104        self.activity_streams
2105            .lock()
2106            .unwrap_or_else(|p| p.into_inner())
2107            .retain(|(id, _), task| {
2108                if *id == connection_id {
2109                    task.abort();
2110                    false
2111                } else {
2112                    true
2113                }
2114            });
2115        // Sync authority dies with its connection, so the owner's next
2116        // connection can take it; the owner's scopes stay as they are.
2117        self.hello_launch_nonces
2118            .lock()
2119            .unwrap_or_else(|poisoned| poisoned.into_inner())
2120            .forget(connection_id);
2121        self.scopes
2122            .write()
2123            .unwrap_or_else(|poisoned| poisoned.into_inner())
2124            .release_connection(connection_id);
2125        registrations
2126    }
2127
2128    pub(crate) fn handle_route_goodbye(
2129        &self,
2130        connection_id: ConnectionId,
2131        route_channel: u16,
2132        route_epoch: u32,
2133    ) -> Result<bool, RouterError> {
2134        debug!(
2135            connection_id = connection_id.get(),
2136            route_channel, route_epoch, "handling route GOODBYE"
2137        );
2138        let RouteRelease::Removed(released_route) = self
2139            .forwarding
2140            .release_client_route(connection_id, route_channel, route_epoch)
2141            .map_err(RouterError::Forwarding)?
2142        else {
2143            return Ok(false);
2144        };
2145        self.emit_route_goodbyes(vec![released_route]);
2146        Ok(true)
2147    }
2148
2149    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2150        for released in released_routes {
2151            let frame = match Frame::build_with_version(
2152                released.negotiated_ver,
2153                FrameType::Goodbye,
2154                control_flags(),
2155                released.channel,
2156                released.epoch,
2157                0,
2158                Vec::new(),
2159            ) {
2160                Ok(frame) => frame,
2161                Err(err) => {
2162                    warn!(
2163                        route_channel = released.channel,
2164                        error = %err,
2165                        "failed to build route GOODBYE frame"
2166                    );
2167                    continue;
2168                }
2169            };
2170            if !released.close_on_delivery_failure() {
2171                crate::forwarding::send_module_route_goodbye(
2172                    &self.counters,
2173                    &released.sink,
2174                    frame,
2175                    released.module_id.as_deref(),
2176                    "client route released",
2177                );
2178                continue;
2179            }
2180            if let Err(err) = released.sink.try_send(frame) {
2181                warn!(
2182                    target_connection_id = released.connection_id.get(),
2183                    route_channel = released.channel,
2184                    error = %err,
2185                    "route GOODBYE was not delivered to client; closing target connection"
2186                );
2187                if self
2188                    .forwarding
2189                    .escalate_client_delivery_failure(
2190                        released.connection_id,
2191                        released.channel,
2192                        released.epoch,
2193                        CloseReason::new(
2194                            "route_goodbye_delivery_failed",
2195                            format!(
2196                                "failed to enqueue route GOODBYE for channel {}: {err}",
2197                                released.channel
2198                            ),
2199                        ),
2200                        crate::forwarding::UndeliveredFrame {
2201                            module_id: released.module_id.as_deref(),
2202                            sink: &released.sink,
2203                        },
2204                    )
2205                    .unwrap_or(false)
2206                {
2207                    self.counters.increment_goodbye_relay_client_failed();
2208                }
2209            }
2210        }
2211    }
2212
2213    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2214    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2215    /// own commit failed after the module had already accepted). Without this, a
2216    /// module that accepts late keeps a binding subc has torn down, so a later
2217    /// frame on that module channel could misdeliver if the channel is reused.
2218    ///
2219    /// Never closes the shared module connection on failure: a dropped notification
2220    /// only wastes a bounded amount of warm module-side state, which the module's
2221    /// own idle reaper reclaims. Only call this once the route.bind relay was
2222    /// actually enqueued to the module — if the relay send itself failed, the
2223    /// module never created a binding and there is nothing to tear down.
2224    fn send_abandoned_route_bind_goodbye(
2225        &self,
2226        module_sink: &crate::FrameSink,
2227        negotiated_ver: u8,
2228        module_channel: u16,
2229        module_epoch: u32,
2230    ) {
2231        let frame = match Frame::build_with_version(
2232            negotiated_ver,
2233            FrameType::Goodbye,
2234            control_flags(),
2235            module_channel,
2236            module_epoch,
2237            0,
2238            Vec::new(),
2239        ) {
2240            Ok(frame) => frame,
2241            Err(err) => {
2242                warn!(
2243                    route_channel = module_channel,
2244                    error = %err,
2245                    "failed to build GOODBYE for abandoned route.bind"
2246                );
2247                return;
2248            }
2249        };
2250        crate::forwarding::send_module_route_goodbye(
2251            &self.counters,
2252            module_sink,
2253            frame,
2254            None,
2255            "abandoned route.bind",
2256        );
2257    }
2258
2259    fn handle_hello(
2260        &self,
2261        connection_id: ConnectionId,
2262        sink: Option<crate::FrameSink>,
2263        frame: Frame,
2264    ) -> Result<Vec<Frame>, RouterError> {
2265        debug!(
2266            connection_id = connection_id.get(),
2267            corr = frame.header.corr,
2268            "handling HELLO"
2269        );
2270        // A module connection has one identity for its entire lifetime. A second
2271        // registration would leave the old registry owner behind while replacing
2272        // its forwarding endpoint and launch nonce.
2273        if self
2274            .registry
2275            .get_module_by_connection(connection_id)
2276            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2277            .is_some()
2278        {
2279            return Ok(vec![control_error_frame(
2280                &frame,
2281                "invalid_hello",
2282                "connection is already registered as a module",
2283            )?]);
2284        }
2285        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2286            Ok(value) => value,
2287            Err(err) => {
2288                return Ok(vec![control_error_frame(
2289                    &frame,
2290                    "invalid_hello",
2291                    format!("malformed HELLO body: {err}"),
2292                )?])
2293            }
2294        };
2295        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2296            return Ok(vec![control_error_frame(
2297                &frame,
2298                "invalid_capability_grammar",
2299                err.to_string(),
2300            )?]);
2301        }
2302        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2303            return Ok(vec![control_error_frame(
2304                &frame,
2305                "invalid_manifest",
2306                err.to_string(),
2307            )?]);
2308        }
2309        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2310            return Ok(vec![control_error_frame(
2311                &frame,
2312                "invalid_event_declaration",
2313                err.to_string(),
2314            )?]);
2315        }
2316        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2317            return Ok(vec![control_error_frame(
2318                &frame,
2319                "invalid_manifest",
2320                err.to_string(),
2321            )?]);
2322        }
2323        if let Some(provenance) = hello_value
2324            .get("manifest")
2325            .and_then(|manifest| manifest.get("provenance"))
2326        {
2327            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2328                return Ok(vec![control_error_frame(
2329                    &frame,
2330                    "invalid_manifest",
2331                    format!("malformed manifest provenance: {err}"),
2332                )?]);
2333            }
2334        }
2335        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2336            Ok(hello) => hello,
2337            Err(err) => {
2338                return Ok(vec![control_error_frame(
2339                    &frame,
2340                    "invalid_hello",
2341                    format!("malformed HELLO body: {err}"),
2342                )?])
2343            }
2344        };
2345
2346        if hello.protocol_ver != hello.manifest.protocol_ver {
2347            return Ok(vec![control_error_frame(
2348                &frame,
2349                "invalid_manifest",
2350                format!(
2351                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2352                    hello.protocol_ver, hello.manifest.protocol_ver
2353                ),
2354            )?]);
2355        }
2356
2357        if hello.manifest.module_id.trim().is_empty() {
2358            return Ok(vec![control_error_frame(
2359                &frame,
2360                "invalid_manifest",
2361                "manifest module_id must not be empty",
2362            )?]);
2363        }
2364
2365        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2366            Ok(negotiated_ver) => negotiated_ver,
2367            Err(message) => {
2368                return Ok(vec![control_error_frame(
2369                    &frame,
2370                    "version_unsupported",
2371                    message,
2372                )?])
2373            }
2374        };
2375
2376        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2377        // swap is open for this id, the only HELLO admitted as a second process
2378        // is the one carrying the candidate's launch nonce (the swap token), and
2379        // it registers into the candidate slot rather than being refused as a
2380        // duplicate. Run after the reserved gate, a reserved module's candidate
2381        // would be refused `reserved_module` for presenting a nonce that gate
2382        // does not know. See `SupervisorHandle::swap_hello_admission`.
2383        let swap_admission = self
2384            .supervisor
2385            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2386        if swap_admission == SwapHelloAdmission::Refused {
2387            warn!(
2388                module_id = %hello.manifest.module_id,
2389                connection_id = connection_id.get(),
2390                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2391            );
2392            return Ok(vec![control_error_frame(
2393                &frame,
2394                "swap_token_invalid",
2395                format!(
2396                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2397                    hello.manifest.module_id
2398                ),
2399            )?]);
2400        }
2401        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2402
2403        // A swap candidate proves its identity with the candidate nonce above.
2404        // Other registrations receive this catalog fact only when reserved-module
2405        // policy required and accepted a nonce; unreserved registrations require
2406        // no nonce check.
2407        let nonce_verified = if swap_candidate {
2408            true
2409        } else {
2410            match self.supervisor.reserved_hello_verification(
2411                &hello.manifest.module_id,
2412                hello.launch_nonce.as_deref(),
2413            ) {
2414                Ok(verified) => verified,
2415                Err(rejection) => {
2416                    let message = match rejection {
2417                        ReservedHelloRejection::Exact { module_id } => format!(
2418                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2419                        ),
2420                        ReservedHelloRejection::Prefix {
2421                            prefix,
2422                            owner_module_id,
2423                        } => format!(
2424                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2425                            hello.manifest.module_id
2426                        ),
2427                    };
2428                    return Ok(vec![control_error_frame(
2429                        &frame,
2430                        "reserved_module",
2431                        message,
2432                    )?]);
2433                }
2434            }
2435        };
2436
2437        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2438            &hello.manifest.module_id,
2439            hello.manifest.capabilities.as_ref(),
2440        );
2441        if let Some(refusal) = reserved_capability_refusals.first() {
2442            let capability = refusal.capability.clone();
2443            let bound_module = refusal.claimants[0].clone();
2444            log_duplicate_claim_events(reserved_capability_refusals);
2445            return Ok(vec![control_error_frame(
2446                &frame,
2447                "reserved_capability",
2448                format!(
2449                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2450                    capability, bound_module, hello.manifest.module_id
2451                ),
2452            )?]);
2453        }
2454
2455        // A connection that already opened client routes must not also register as
2456        // a module: cleanup would then release only one side and leak the other.
2457        if self
2458            .forwarding
2459            .connection_has_client_routes(connection_id)
2460            .map_err(RouterError::Forwarding)?
2461        {
2462            return Ok(vec![control_error_frame(
2463                &frame,
2464                "invalid_hello",
2465                "connection has open client routes and cannot also register as a module",
2466            )?]);
2467        }
2468
2469        // Kept for scope sync authority, which goes only to the connection that
2470        // presented the module's current launch nonce. Recorded before the
2471        // registration is attempted: a connection whose registration then fails
2472        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2473        self.hello_launch_nonces
2474            .lock()
2475            .unwrap_or_else(|poisoned| poisoned.into_inner())
2476            .record(connection_id, hello.launch_nonce.as_deref());
2477        let control_ops = effective_module_control_ops(hello.control_ops);
2478        // Built before anything is registered so an encoding failure leaves no
2479        // registry or forwarding state behind.
2480        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2481        if swap_candidate {
2482            return self.register_swap_candidate(
2483                connection_id,
2484                sink,
2485                &frame,
2486                hello.manifest,
2487                negotiated_ver,
2488                control_ops,
2489                hello_ack,
2490            );
2491        }
2492        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2493        // is refused, but `catalog.list` does not show the module and the
2494        // catalog generation does not move until it is published below, once
2495        // its forwarding endpoint exists. Listing it now would advertise a
2496        // module that `route.open` still refuses for having no forwarding
2497        // connection.
2498        let registration = match self
2499            .registry
2500            .register_unpublished_verified_with_control_ops(
2501                hello.manifest,
2502                negotiated_ver,
2503                connection_id,
2504                control_ops,
2505                nonce_verified,
2506            ) {
2507            Ok(registration) => registration,
2508            Err(RegistryError::DuplicateModuleId { module_id }) => {
2509                return Ok(vec![control_error_frame(
2510                    &frame,
2511                    "duplicate_module_id",
2512                    format!(
2513                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2514                    ),
2515                )?])
2516            }
2517            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2518                return Ok(vec![control_error_frame(
2519                    &frame,
2520                    "invalid_module_id",
2521                    err.to_string(),
2522                )?])
2523            }
2524            Err(err) => {
2525                return Ok(vec![control_error_frame(
2526                    &frame,
2527                    "registry_error",
2528                    err.to_string(),
2529                )?])
2530            }
2531        };
2532        #[cfg(test)]
2533        self.hold_between_registry_and_forwarding();
2534
2535        let reply = if let Some(sink) = sink {
2536            // The forwarding table's module store is also the daemon-to-module
2537            // control-RPC lane, so every HELLO gets a live endpoint even when the
2538            // manifest has no routable provider role. Non-routable modules still
2539            // cannot receive route.bind in production: `handle_route_open` checks
2540            // the registry manifest with `target_has_required_role` before the
2541            // only production call to `begin_route_bind_relay_for` below that
2542            // route.open path. The remaining direct relay callers are unit tests
2543            // and benchmark harnesses that construct forwarding state explicitly.
2544            //
2545            // The HELLO_ACK is queued by the forwarding table itself, before the
2546            // endpoint becomes visible, and is NOT returned as a reply. A module
2547            // reads HELLO_ACK first and exits on anything else; a reply is only
2548            // written after this handler returns, by which time a route.open on
2549            // another connection could already have queued a route.bind request
2550            // for this module ahead of it.
2551            let concurrency = manifest_concurrency(&registration.manifest);
2552            if let Err(err) = self.forwarding.register_module_connection_acked(
2553                connection_id,
2554                registration.manifest.module_id.clone(),
2555                negotiated_ver,
2556                concurrency,
2557                sink,
2558                hello_ack,
2559            ) {
2560                // Forwarding registration failed, so there is no forwarding
2561                // state to tear down. Remove the registry entry, which was never
2562                // published and so never listed or counted in the catalog
2563                // generation, and signal the release watch directly.
2564                if matches!(
2565                    self.deregister_connection(
2566                        connection_id,
2567                        RegistrationEndReason::RegistrationFailed,
2568                    ),
2569                    Ok(r) if !r.is_empty()
2570                ) {
2571                    crate::supervise::notify_registration_release();
2572                }
2573                return Ok(vec![control_error_frame(
2574                    &frame,
2575                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2576                        "invalid_hello"
2577                    } else {
2578                        forwarding_error_code(&err)
2579                    },
2580                    err.to_string(),
2581                )?]);
2582            }
2583            Vec::new()
2584        } else {
2585            // No sink means no forwarding endpoint, so nothing can be routed
2586            // ahead of the ack; it goes out as the reply.
2587            vec![hello_ack]
2588        };
2589
2590        // The module is routable now (or, without a sink, as routable as it
2591        // will ever be), so the catalog may list it. Publishing bumps the
2592        // catalog generation in the same registry critical section that makes
2593        // the module listed.
2594        if !self
2595            .registry
2596            .publish_registration(connection_id)
2597            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2598        {
2599            debug!(
2600                module_id = %registration.manifest.module_id,
2601                connection_id = connection_id.get(),
2602                "registration was removed before it could be published"
2603            );
2604        }
2605
2606        // Exposure over assumption: Concurrency's serde default is pinned to the
2607        // pre-field behavior (ModuleManaged), so a management surface that is
2608        // genuinely Serial and just never declared it inherits concurrent
2609        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2610        // turns "no module has been bitten yet" into the checkable claim "no
2611        // module is exposed" -- one read of the boot log instead of a fleet
2612        // audit. Detected from the raw HELLO bytes because the serde default
2613        // deliberately erases the absent/declared distinction from the type.
2614        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2615            info!(
2616                module_id = %registration.manifest.module_id,
2617                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2618            );
2619        }
2620
2621        self.apply_registration_capabilities(&registration);
2622
2623        self.supervisor
2624            .record_activity_registration(&registration.manifest.module_id);
2625
2626        info!(
2627            module_id = %registration.manifest.module_id,
2628            module_version = %registration.manifest.module_version,
2629            negotiated_ver,
2630            routable_provider = manifest_provides_routable_role(&registration.manifest),
2631            connection_id = connection_id.get(),
2632            "module registered"
2633        );
2634
2635        Ok(reply)
2636    }
2637
2638    /// Register a HELLO the swap gate admitted into the candidate slot of the
2639    /// registry and of forwarding, where it is reachable over its own
2640    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2641    /// nothing routes to it until the supervisor cuts over.
2642    ///
2643    /// Registry first (unpublished), then forwarding, then publication, the
2644    /// same order as an ordinary HELLO; a forwarding failure removes the
2645    /// registry entry again. The swap supervisor treats a published, ready
2646    /// candidate as registered and promotes it, and promotion needs the
2647    /// candidate's forwarding endpoint, so the candidate is published only
2648    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2649    /// recompute capability verdicts or run the capability deny census
2650    /// (`enforce_capability_denies`): both are about the module in the active
2651    /// slot, which `route.open` can target. A candidate sits in the candidate
2652    /// slot, so it neither provides capabilities to other modules nor can be
2653    /// routed to until promotion moves it into the active slot.
2654    #[allow(clippy::too_many_arguments)]
2655    fn register_swap_candidate(
2656        &self,
2657        connection_id: ConnectionId,
2658        sink: Option<crate::FrameSink>,
2659        frame: &Frame,
2660        manifest: ModuleManifest,
2661        negotiated_ver: u8,
2662        control_ops: Vec<String>,
2663        hello_ack: Frame,
2664    ) -> Result<Vec<Frame>, RouterError> {
2665        let module_id = manifest.module_id.clone();
2666        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2667            manifest,
2668            negotiated_ver,
2669            connection_id,
2670            control_ops,
2671        ) {
2672            Ok(registration) => registration,
2673            Err(RegistryError::DuplicateModuleId { module_id }) => {
2674                return Ok(vec![control_error_frame(
2675                    frame,
2676                    "duplicate_module_id",
2677                    format!(
2678                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2679                    ),
2680                )?])
2681            }
2682            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2683                return Ok(vec![control_error_frame(
2684                    frame,
2685                    "invalid_module_id",
2686                    err.to_string(),
2687                )?])
2688            }
2689            Err(err) => {
2690                return Ok(vec![control_error_frame(
2691                    frame,
2692                    "registry_error",
2693                    err.to_string(),
2694                )?])
2695            }
2696        };
2697        #[cfg(test)]
2698        self.hold_between_registry_and_forwarding();
2699        let reply = if let Some(sink) = sink {
2700            // Same ordering as an ordinary HELLO: the forwarding table queues
2701            // the HELLO_ACK before the candidate endpoint is inserted, because
2702            // a module exits if its first frame after HELLO is anything else.
2703            let concurrency = manifest_concurrency(&registration.manifest);
2704            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2705                connection_id,
2706                module_id.clone(),
2707                negotiated_ver,
2708                concurrency,
2709                sink,
2710                hello_ack,
2711            ) {
2712                if matches!(
2713                    self.deregister_connection(
2714                        connection_id,
2715                        RegistrationEndReason::RegistrationFailed,
2716                    ),
2717                    Ok(r) if !r.is_empty()
2718                ) {
2719                    crate::supervise::notify_registration_release();
2720                }
2721                return Ok(vec![control_error_frame(
2722                    frame,
2723                    forwarding_error_code(&err),
2724                    err.to_string(),
2725                )?]);
2726            }
2727            Vec::new()
2728        } else {
2729            vec![hello_ack]
2730        };
2731        if !self
2732            .registry
2733            .publish_registration(connection_id)
2734            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2735        {
2736            debug!(
2737                module_id = %module_id,
2738                connection_id = connection_id.get(),
2739                "swap candidate registration was removed before it could be published"
2740            );
2741        }
2742        self.supervisor.mark_swap_candidate_admitted(&module_id);
2743        info!(
2744            module_id = %module_id,
2745            module_version = %registration.manifest.module_version,
2746            negotiated_ver,
2747            ready = registration.ready,
2748            connection_id = connection_id.get(),
2749            "swap candidate registered; not routable until cutover"
2750        );
2751        Ok(reply)
2752    }
2753
2754    fn build_hello_ack(
2755        &self,
2756        frame: &Frame,
2757        negotiated_ver: u8,
2758        module_id: &str,
2759    ) -> Result<Frame, RouterError> {
2760        let ack = ModuleHelloAckBody {
2761            negotiated_ver,
2762            subc_ops: module_subc_ops(),
2763            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2764            storage: self
2765                .storage_config
2766                .as_ref()
2767                .map(|cfg| cfg.descriptor_for(module_id)),
2768            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2769        };
2770        let body = serde_json::to_vec(&ack).map_err(|err| {
2771            RouterError::backend(
2772                0,
2773                frame.header.corr,
2774                format!("failed to encode HELLO_ACK: {err}"),
2775            )
2776        })?;
2777
2778        Frame::build_with_version(
2779            negotiated_ver,
2780            FrameType::HelloAck,
2781            control_flags(),
2782            0,
2783            0,
2784            frame.header.corr,
2785            body,
2786        )
2787        .map_err(RouterError::FrameBuild)
2788    }
2789
2790    async fn handle_client_control_request(
2791        &self,
2792        ctx: &RouteCtx,
2793        frame: Frame,
2794        request: ClientControlRequest,
2795        route_open_timings: &mut Option<RouteOpenTimings>,
2796    ) -> Result<Vec<Frame>, RouterError> {
2797        if let Some(operation) = privileged_operation(&request) {
2798            let caller = self.resolve_caller(ctx.connection_id).await;
2799            let op = client_control_request_op(&request);
2800            let text = prompt_text(operation, &caller);
2801            match self.privileged_operator_prompt {
2802                PrivilegedOperatorPrompt::Log => {
2803                    info!(target: "subc_daemon::privileged_prompt", operation = op,
2804                        target = ?operation, caller = %caller.description(), summary = %text,
2805                        "privileged request would prompt the operator");
2806                }
2807                PrivilegedOperatorPrompt::Enforce => {
2808                    let key =
2809                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2810                    let outcome = self
2811                        .forwarding
2812                        .operator_confirms()
2813                        .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2814                        .await;
2815                    if outcome.code != "confirmed" {
2816                        return Ok(vec![outcome.frame(&frame)]);
2817                    }
2818                }
2819            }
2820        }
2821        match request {
2822            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2823            ClientControlRequest::CatalogList { module_id } => {
2824                self.handle_catalog_list(frame, module_id)
2825            }
2826            ClientControlRequest::RouteOpen {
2827                target,
2828                identity,
2829                consumer_identity,
2830                consumer_capabilities,
2831                role_versions,
2832                admission_facts,
2833                scope,
2834            } => {
2835                self.handle_route_open(
2836                    ctx,
2837                    frame,
2838                    RouteOpenRequest {
2839                        target,
2840                        identity,
2841                        consumer_identity,
2842                        consumer_capabilities,
2843                        role_versions,
2844                        admission_facts,
2845                        scope,
2846                    },
2847                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2848                )
2849                .await
2850            }
2851            ClientControlRequest::RoutePoll {
2852                route_channel,
2853                route_epoch,
2854                kind,
2855            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2856            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2857            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2858                self.handle_supervisor_spawn_snapshot(frame)
2859            }
2860            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2861                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2862            }
2863            ClientControlRequest::SupervisorRestart {
2864                module_id,
2865                drain_timeout_ms,
2866            } => {
2867                self.log_supervisor_request_received(
2868                    ctx,
2869                    frame.header.corr,
2870                    ops::SUPERVISOR_RESTART,
2871                    Some(&module_id),
2872                    None,
2873                )?;
2874                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2875                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2876                    .await
2877            }
2878            ClientControlRequest::SupervisorSwap {
2879                module_id,
2880                ready_timeout_ms,
2881            } => {
2882                self.log_supervisor_request_received(
2883                    ctx,
2884                    frame.header.corr,
2885                    ops::SUPERVISOR_SWAP,
2886                    Some(&module_id),
2887                    None,
2888                )?;
2889                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2890                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2891                    .await
2892            }
2893            ClientControlRequest::SupervisorReload { module_id } => {
2894                self.log_supervisor_request_received(
2895                    ctx,
2896                    frame.header.corr,
2897                    ops::SUPERVISOR_RELOAD,
2898                    Some(&module_id),
2899                    None,
2900                )?;
2901                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2902                self.handle_supervisor_reload(frame, module_id, requester)
2903                    .await
2904            }
2905            ClientControlRequest::SupervisorRescan { preview } => {
2906                if !preview {
2907                    self.log_supervisor_request_received(
2908                        ctx,
2909                        frame.header.corr,
2910                        ops::SUPERVISOR_RESCAN,
2911                        None,
2912                        None,
2913                    )?;
2914                }
2915                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2916                self.handle_supervisor_rescan(
2917                    frame,
2918                    if preview {
2919                        RescanMode::Preview
2920                    } else {
2921                        RescanMode::Execute
2922                    },
2923                    requester,
2924                )
2925                .await
2926            }
2927            ClientControlRequest::SupervisorRescanChecked {
2928                preview,
2929                candidate,
2930                expect,
2931            } => {
2932                self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2933                    .await
2934            }
2935            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2936                self.log_supervisor_request_received(
2937                    ctx,
2938                    frame.header.corr,
2939                    ops::SUPERVISOR_RELEASE_RESERVED,
2940                    Some(&module_id),
2941                    None,
2942                )?;
2943                self.handle_supervisor_release_reserved(frame, module_id)
2944                    .await
2945            }
2946            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2947                self.log_supervisor_request_received(
2948                    ctx,
2949                    frame.header.corr,
2950                    ops::SUPERVISOR_SET_ENABLED,
2951                    Some(&module_id),
2952                    Some(enabled),
2953                )?;
2954                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2955                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2956                    .await
2957            }
2958            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2959                self.handle_supervisor_health_probe(frame, module_id).await
2960            }
2961            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2962            ClientControlRequest::SupervisorRoutes { module_id } => {
2963                self.handle_supervisor_routes(frame, module_id)
2964            }
2965            ClientControlRequest::SupervisorProvenance { module_id } => {
2966                self.handle_supervisor_provenance(frame, module_id).await
2967            }
2968            ClientControlRequest::SupervisorStderrTail {
2969                module_id,
2970                max_lines,
2971                max_bytes,
2972            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2973            ClientControlRequest::SupervisorTerminals { module_id } => {
2974                self.handle_supervisor_terminals(frame, module_id).await
2975            }
2976        }
2977    }
2978
2979    fn log_supervisor_request_received(
2980        &self,
2981        ctx: &RouteCtx,
2982        corr: u64,
2983        op: &'static str,
2984        module_id: Option<&str>,
2985        enabled: Option<bool>,
2986    ) -> Result<(), RouterError> {
2987        let caller = self.supervisor_requester(ctx, corr)?;
2988        let caller = principal_label(&caller);
2989        // `Option` fields are recorded only when present, so a request with no
2990        // module id (rescan) or no enabled flag simply omits that field.
2991        info!(
2992            target: "control",
2993            op,
2994            module_id,
2995            enabled,
2996            connection_id = ctx.connection_id.get(),
2997            caller = %caller,
2998            "supervisor request received"
2999        );
3000        Ok(())
3001    }
3002
3003    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3004        Ok(self
3005            .registry
3006            .get_module_by_connection(ctx.connection_id)
3007            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3008            .map(|registration| Principal::Reserved {
3009                module_id: registration.manifest.module_id,
3010            })
3011            .unwrap_or(Principal::Direct))
3012    }
3013
3014    fn handle_module_control_request(
3015        &self,
3016        connection_id: ConnectionId,
3017        frame: Frame,
3018        request: ModuleControlRequestFromModule,
3019    ) -> Result<Vec<Frame>, RouterError> {
3020        match request {
3021            ModuleControlRequestFromModule::CatalogUpdate {
3022                provides,
3023                capabilities,
3024                ready,
3025            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3026            ModuleControlRequestFromModule::LiveRoots {} => {
3027                let registered = self
3028                    .registry
3029                    .get_module_by_connection(connection_id)
3030                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3031                let Some(registration) = registered else {
3032                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3033                };
3034                let response = self
3035                    .forwarding
3036                    .live_roots(&registration.manifest.module_id)
3037                    .map_err(RouterError::Forwarding)?;
3038                Ok(vec![control_response_body_frame(
3039                    &frame,
3040                    &response,
3041                    "ModuleControlResponseToModule::LiveRoots",
3042                )?])
3043            }
3044            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3045                self.handle_scope_sync(connection_id, frame, generation, scopes)
3046            }
3047            ModuleControlRequestFromModule::ScopeApply {
3048                generation,
3049                upsert,
3050                end,
3051            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3052            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3053                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3054            }
3055        }
3056    }
3057
3058    fn handle_scope_served_by(
3059        &self,
3060        connection_id: ConnectionId,
3061        frame: Frame,
3062    ) -> Result<Vec<Frame>, RouterError> {
3063        let registered = self
3064            .registry
3065            .get_module_by_connection(connection_id)
3066            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3067        let Some(registered) = registered else {
3068            return Ok(vec![control_error_frame(
3069                &frame,
3070                "not_registered",
3071                "scope.served_by requires an active module registration owned by this connection",
3072            )?]);
3073        };
3074        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3075        {
3076            Ok(request) => request,
3077            Err(err) => {
3078                return Ok(vec![control_error_frame(
3079                    &frame,
3080                    "invalid_control_body",
3081                    format!("malformed scope.served_by body: {err}"),
3082                )?])
3083            }
3084        };
3085        let answer = self
3086            .scopes
3087            .read()
3088            .unwrap_or_else(|poisoned| poisoned.into_inner())
3089            .served_by(
3090                &registered.manifest.module_id,
3091                &request.scope_ref,
3092                request.scope_epoch,
3093                &request.module_id,
3094            );
3095        let reply = subc_control::ScopeServedByReply::new(
3096            answer,
3097            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3098        );
3099        Ok(vec![control_response_body_frame(
3100            &frame,
3101            &reply,
3102            "ScopeServedByReply",
3103        )?])
3104    }
3105
3106    fn handle_operator_confirm(
3107        &self,
3108        ctx: &RouteCtx,
3109        frame: Frame,
3110    ) -> Result<Vec<Frame>, RouterError> {
3111        use crate::operator_confirm::{audit, Outcome};
3112        let registration = self
3113            .registry
3114            .get_module_by_connection(ctx.connection_id)
3115            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3116        let Some(registration) = registration else {
3117            let outcome = Outcome::refusal("not_registered");
3118            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3119            return Ok(vec![outcome.frame(&frame)]);
3120        };
3121        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3122            Ok(request) => request,
3123            Err(_) => {
3124                let outcome = Outcome::refusal("invalid_control_body");
3125                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3126                return Ok(vec![outcome.frame(&frame)]);
3127            }
3128        };
3129        let module_id = registration.manifest.module_id;
3130        // Read the launch nonce (under its own lock) before taking the forwarding
3131        // table's lock below: holding forwarding while waiting on another daemon
3132        // lock risks a lock-order deadlock with paths that take them the other way.
3133        let nonce = self
3134            .hello_launch_nonces
3135            .lock()
3136            .unwrap_or_else(|p| p.into_inner())
3137            .nonce(ctx.connection_id)
3138            .map(str::to_owned);
3139        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3140            self.supervisor
3141                .spawned_consumer_authorized(&module_id, nonce)
3142        });
3143        let confirms = self.forwarding.operator_confirms();
3144        self.forwarding
3145            .with_operator_route(
3146                ctx.connection_id,
3147                request.route_channel,
3148                request.route_epoch,
3149                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3150            )
3151            .map_err(RouterError::Forwarding)
3152    }
3153
3154    /// Resolve folders before taking the scope-table lock: a slow filesystem
3155    /// must not hold up admissions for unrelated scoped routes.
3156    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3157        let mut scopes = ScopeRecords::from(scopes);
3158        for (index, record) in scopes.records.iter_mut().enumerate() {
3159            if let Some(refusal) = Self::canonicalize_workdir(record) {
3160                scopes.refusals.insert(index, refusal);
3161            }
3162        }
3163        scopes
3164    }
3165
3166    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3167        let workdir = record.workdir.as_mut()?;
3168        let path = std::path::Path::new(workdir);
3169        if !path.is_absolute() {
3170            return Some((
3171                error_codes::INVALID_WORKDIR,
3172                "workdir must be absolute".to_string(),
3173            ));
3174        }
3175        match ProjectRootId::from_path_allowing_missing(path) {
3176            Ok(root) => match root.as_path().to_str() {
3177                Some(canonical) => {
3178                    *workdir = canonical.to_string();
3179                    None
3180                }
3181                None => Some((
3182                    error_codes::INVALID_WORKDIR,
3183                    "canonical workdir is not UTF-8".to_string(),
3184                )),
3185            },
3186            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3187        }
3188    }
3189
3190    /// `scope.sync`: the owner is the module registered on this connection.
3191    /// A connection with no registration (every client connection, `direct`
3192    /// included) is refused `not_registered` before the table is consulted.
3193    fn handle_scope_sync(
3194        &self,
3195        connection_id: ConnectionId,
3196        frame: Frame,
3197        generation: u64,
3198        scopes: Vec<ScopeRecord>,
3199    ) -> Result<Vec<Frame>, RouterError> {
3200        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3201    }
3202
3203    fn handle_scope_change(
3204        &self,
3205        connection_id: ConnectionId,
3206        frame: Frame,
3207        generation: u64,
3208        scopes: Vec<ScopeRecord>,
3209        end: Option<Vec<ScopeEnd>>,
3210    ) -> Result<Vec<Frame>, RouterError> {
3211        let is_apply = end.is_some();
3212        let Some(registration) = self
3213            .registry
3214            .get_module_by_connection(connection_id)
3215            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3216        else {
3217            return Ok(vec![control_error_frame(
3218                &frame,
3219                "not_registered",
3220                "scope.sync and scope.apply require an active module registration owned by this connection",
3221            )?]);
3222        };
3223        let owner = registration.manifest.module_id;
3224        let scopes = Self::prepare_scope_records(scopes);
3225        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3226        let is_current_launch = |connection: ConnectionId| {
3227            self.hello_launch_nonces
3228                .lock()
3229                .unwrap_or_else(|poisoned| poisoned.into_inner())
3230                .presented(connection, current_nonce.as_deref())
3231        };
3232        // Lock order is the scope table, then the forwarding table: the new
3233        // tags are published, and the routes the change closes are selected,
3234        // while the scope table is still write-locked, so no admission can read
3235        // a record whose tag is not yet published.
3236        let mut table = self
3237            .scopes
3238            .write()
3239            .unwrap_or_else(|poisoned| poisoned.into_inner());
3240        let outcome = match end {
3241            Some(end) => table.apply(
3242                &owner,
3243                connection_id,
3244                is_current_launch,
3245                generation,
3246                scopes,
3247                end,
3248            ),
3249            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3250        };
3251        let drained = match &outcome {
3252            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3253            Err(_) => Vec::new(),
3254        };
3255        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3256        drop(table);
3257        if has_deadlines {
3258            self.scope_deadline_added.notify_one();
3259        }
3260        match outcome {
3261            Ok(applied) => {
3262                let counts = ScopeOutcomeCounts::of(&applied.results);
3263                info!(
3264                    owner = %owner,
3265                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3266                    generation,
3267                    records = applied.results.len(),
3268                    created = counts.created,
3269                    replaced = counts.replaced,
3270                    updated = counts.updated,
3271                    unchanged = counts.unchanged,
3272                    refused = counts.refused,
3273                    ended = applied.ended.len(),
3274                    tag_changes = applied.tag_changes.len(),
3275                    routes_closed = drained.len(),
3276                    "scope change accepted"
3277                );
3278                // An accepted scope change can still refuse individual records, and the
3279                // owner is the only party that sees the reply. Name them here so
3280                // an operator can tell a refused session from a missing one
3281                // without the owner's logs. Capped so a call that refuses
3282                // thousands cannot flood the log; the count above is complete.
3283                for refused in applied
3284                    .results
3285                    .iter()
3286                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3287                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3288                {
3289                    warn!(
3290                        owner = %owner,
3291                        generation,
3292                        scope_ref = %refused.scope_ref,
3293                        scope_epoch = refused.scope_epoch,
3294                        code = refused.code.as_deref().unwrap_or(""),
3295                        "scope record refused"
3296                    );
3297                }
3298                self.close_scope_drained_routes(drained);
3299                let response = if is_apply {
3300                    ModuleControlResponseToModule::ScopeApply {
3301                        generation,
3302                        results: applied.results,
3303                        end_results: applied.end_results,
3304                        ended: applied.ended,
3305                    }
3306                } else {
3307                    ModuleControlResponseToModule::ScopeSync {
3308                        generation,
3309                        results: applied.results,
3310                        ended: applied.ended,
3311                    }
3312                };
3313                Ok(vec![control_response_body_frame(
3314                    &frame,
3315                    &response,
3316                    "scope change response",
3317                )?])
3318            }
3319            Err(refusal) => {
3320                info!(
3321                    owner = %owner,
3322                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3323                    generation,
3324                    code = refusal.code,
3325                    "scope change refused"
3326                );
3327                Ok(vec![control_error_frame(
3328                    &frame,
3329                    refusal.code,
3330                    refusal.message,
3331                )?])
3332            }
3333        }
3334    }
3335
3336    /// Tell both ends of each route a scope change closed. The module gets a
3337    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3338    /// the client's route handle. The client also gets `route.closed` with the
3339    /// scope reason, one push per module and reason, so it can tell a revoked
3340    /// route from an ordinary close and not reopen it.
3341    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3342        if drained.is_empty() {
3343            return;
3344        }
3345        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3346            BTreeMap::new();
3347        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3348        for route in drained {
3349            warn!(
3350                module_id = %route.module_id,
3351                reason = ?route.reason,
3352                client_connection_id = route.client.connection_id.get(),
3353                route_channel = route.client.channel,
3354                "closing route because its scope changed"
3355            );
3356            pushes
3357                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3358                .or_insert_with(|| (route.reason, Vec::new()))
3359                .1
3360                .push(EndpointRoute {
3361                    goodbye_target: route.client.clone(),
3362                    principal: Principal::Unverified,
3363                    bound_at: Instant::now(),
3364                    draining: false,
3365                    drain_reason: None,
3366                });
3367            goodbyes.push(route.module);
3368            goodbyes.push(route.client);
3369        }
3370        for ((module_id, _), (reason, routes)) in pushes {
3371            send_route_control_pushes(
3372                &self.forwarding,
3373                routes,
3374                ClientControlPush::RouteClosed {
3375                    module_id,
3376                    channels: Vec::new(),
3377                    reason,
3378                    drained: false,
3379                    abandoned: 0,
3380                    excluded_subscriptions: 0,
3381                    terminal: Some(false),
3382                },
3383            );
3384        }
3385        self.emit_route_goodbyes(goodbyes);
3386    }
3387
3388    /// `scope.describe`: any registered module may read any scope, because a
3389    /// provider must read the scope a route it serves is stamped with.
3390    fn handle_scope_describe(
3391        &self,
3392        connection_id: ConnectionId,
3393        frame: Frame,
3394        owner: Principal,
3395        scope_ref: String,
3396    ) -> Result<Vec<Frame>, RouterError> {
3397        let registered = self
3398            .registry
3399            .get_module_by_connection(connection_id)
3400            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3401        let Some(registered) = registered else {
3402            return Ok(vec![control_error_frame(
3403                &frame,
3404                "not_registered",
3405                "scope.describe requires an active module registration owned by this connection",
3406            )?]);
3407        };
3408        let mut description = self
3409            .scopes
3410            .read()
3411            .unwrap_or_else(|poisoned| poisoned.into_inner())
3412            .describe(&owner, &scope_ref);
3413        // Describe readers also decode strict ScopeAttributes. Attribution is
3414        // safe to omit unless the requesting module declares it understands it.
3415        if !registered
3416            .manifest
3417            .capabilities
3418            .as_ref()
3419            .is_some_and(|caps| {
3420                caps.provides
3421                    .iter()
3422                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3423            })
3424        {
3425            if let Some(stamp) = description.stamp.as_mut() {
3426                stamp.attributes.on_behalf_of_agent = None;
3427            }
3428        }
3429        let owner_configured = match &owner {
3430            // Ask whether the owner is configured (`is_configured`), not
3431            // whether it is on the roster (`get(..).is_some()`): a supervised
3432            // module's process can register and describe a scope before the
3433            // supervisor has put it on the roster.
3434            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3435            _ => false,
3436        };
3437        let response = ModuleControlResponseToModule::ScopeDescribe {
3438            status: description.status,
3439            scope_epoch: description.scope_epoch,
3440            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3441            owner_synced: description.owner_synced,
3442            owner_configured,
3443            scope: description.stamp,
3444        };
3445        Ok(vec![control_response_body_frame(
3446            &frame,
3447            &response,
3448            "ModuleControlResponseToModule::ScopeDescribe",
3449        )?])
3450    }
3451
3452    fn handle_catalog_update(
3453        &self,
3454        connection_id: ConnectionId,
3455        frame: Frame,
3456        provides: Vec<ProviderRole>,
3457        capabilities: Option<CapabilityDeclarations>,
3458        ready: Option<bool>,
3459    ) -> Result<Vec<Frame>, RouterError> {
3460        self.refresh_capability_requirements();
3461        let Some(registration) = self
3462            .registry
3463            .get_module_by_connection(connection_id)
3464            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3465        else {
3466            return Ok(vec![control_error_frame(
3467                &frame,
3468                "not_registered",
3469                "catalog.update requires an active module registration owned by this connection",
3470            )?]);
3471        };
3472
3473        if let Some(message) =
3474            catalog_update_frozen_field_message(&registration.manifest, &provides)
3475        {
3476            return Ok(vec![control_error_frame(
3477                &frame,
3478                "catalog_update_frozen_field",
3479                message,
3480            )?]);
3481        }
3482
3483        let mut candidate = registration.manifest.clone();
3484        candidate.provides = provides.clone();
3485        candidate.capabilities = capabilities
3486            .clone()
3487            .or_else(|| registration.manifest.capabilities.clone());
3488        if let Some(rich_ui) = &candidate.rich_ui {
3489            if let Err(err) = rich_ui.validate() {
3490                return Ok(vec![control_error_frame(
3491                    &frame,
3492                    "invalid_manifest",
3493                    err.to_string(),
3494                )?]);
3495            }
3496        }
3497        if let Err(err) = candidate.validate_capability_grammar() {
3498            return Ok(vec![control_error_frame(
3499                &frame,
3500                "invalid_capability_grammar",
3501                err.to_string(),
3502            )?]);
3503        }
3504
3505        // Updates must honor the same reserved owner as initial registration;
3506        // otherwise an empty HELLO could acquire the claim after admission.
3507        let mut conflicts = self
3508            .capability_evaluator
3509            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3510        if let Some(conflict) = conflicts.first() {
3511            let message = format!(
3512                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3513                conflict.capability, conflict.claimants[0], candidate.module_id
3514            );
3515            for conflict in &mut conflicts {
3516                conflict.source = DuplicateClaimSource::CatalogUpdate;
3517            }
3518            log_duplicate_claim_events(conflicts);
3519            return Ok(vec![control_error_frame(
3520                &frame,
3521                "reserved_capability",
3522                message,
3523            )?]);
3524        }
3525
3526        let updated = self
3527            .registry
3528            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3529            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3530        if updated.is_none() {
3531            return Ok(vec![control_error_frame(
3532                &frame,
3533                "not_registered",
3534                "catalog.update requires an active module registration owned by this connection",
3535            )?]);
3536        }
3537        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3538            log_duplicate_claim_events(
3539                self.capability_evaluator
3540                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3541            );
3542        }
3543        if capability_census_trigger(
3544            registration.manifest.capabilities.as_ref(),
3545            updated
3546                .as_ref()
3547                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3548        ) {
3549            self.enforce_capability_denies();
3550        }
3551        self.refresh_capability_requirements();
3552
3553        let response = ModuleControlResponseToModule::CatalogUpdate {};
3554        control_response_body_frame(
3555            &frame,
3556            &response,
3557            "ModuleControlResponseToModule::CatalogUpdate",
3558        )
3559        .map(|frame| vec![frame])
3560    }
3561
3562    pub(crate) fn with_activity_ledger_config(
3563        mut self,
3564        run_dir: &Path,
3565        incarnation: String,
3566        config: crate::activity::ActivityConfig,
3567    ) -> Self {
3568        self.activity = Some(crate::activity::ActivityLedger::open_config(
3569            run_dir.join("activity.jsonl"),
3570            incarnation,
3571            config,
3572        ));
3573        self.supervisor.set_activity(self.activity.clone());
3574        self
3575    }
3576
3577    pub(crate) async fn configure_activity(
3578        &self,
3579        config: crate::activity::ActivityConfig,
3580    ) -> Result<(), &'static str> {
3581        match &self.activity {
3582            Some(ledger) => ledger.configure(config).await,
3583            None => Ok(()),
3584        }
3585    }
3586
3587    fn cancel_activity_subscription(
3588        &self,
3589        ctx: &RouteCtx,
3590        frame: &Frame,
3591    ) -> Result<bool, RouterError> {
3592        let task = self
3593            .activity_streams
3594            .lock()
3595            .unwrap_or_else(|p| p.into_inner())
3596            .remove(&(ctx.connection_id, frame.header.corr));
3597        let Some(task) = task else {
3598            return Ok(false);
3599        };
3600        task.abort();
3601        let end = Frame::build_with_version(
3602            response_version(frame),
3603            FrameType::StreamEnd,
3604            control_flags(),
3605            0,
3606            0,
3607            frame.header.corr,
3608            Vec::new(),
3609        )
3610        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3611        let sink = ctx.egress.clone();
3612        tokio::spawn(async move {
3613            let _ = sink.send(end).await;
3614        });
3615        Ok(true)
3616    }
3617
3618    async fn handle_activity_read(
3619        &self,
3620        ctx: &RouteCtx,
3621        frame: Frame,
3622    ) -> Result<Vec<Frame>, RouterError> {
3623        use subc_control::{
3624            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3625            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3626        };
3627        #[derive(Deserialize)]
3628        struct Identity {
3629            #[serde(default)]
3630            consumer_identity: Option<ConsumerIdentity>,
3631        }
3632        // A registered module connection already identifies its caller. Ignore
3633        // consumer_identity there so a module cannot borrow a mirror's identity.
3634        // Only separate consumer connections use the Direct/nonce trust boundary.
3635        let principal = match self
3636            .forwarding
3637            .module_id_for_connection(ctx.connection_id)
3638            .map_err(RouterError::Forwarding)?
3639        {
3640            Some(module_id) => Ok(Principal::Reserved { module_id }),
3641            None => {
3642                let identity: Identity = match serde_json::from_slice(&frame.body) {
3643                    Ok(identity) => identity,
3644                    Err(_) => {
3645                        return Ok(vec![control_error_frame(
3646                            &frame,
3647                            ACTIVITY_MALFORMED,
3648                            "malformed activity read",
3649                        )?])
3650                    }
3651                };
3652                self.route_open_principal(&frame, identity.consumer_identity)?
3653            }
3654        };
3655        let admitted = matches!(principal, Ok(Principal::Direct))
3656            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3657        if !admitted {
3658            return Ok(vec![control_error_frame(
3659                &frame,
3660                ACTIVITY_READ_FORBIDDEN,
3661                "activity reads require an operator or trusted mirror principal",
3662            )?]);
3663        }
3664        let Some(ledger) = &self.activity else {
3665            return Ok(vec![control_error_frame(
3666                &frame,
3667                ACTIVITY_UNAVAILABLE,
3668                "activity ledger unavailable",
3669            )?]);
3670        };
3671        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3672            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3673        {
3674            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3675                Ok(request) => ledger.query(request).await,
3676                Err(_) => Err(ACTIVITY_MALFORMED),
3677            };
3678            return match result {
3679                Ok(reply) => Ok(vec![control_response_body_frame(
3680                    &frame,
3681                    &reply,
3682                    ops::ACTIVITY_QUERY,
3683                )?]),
3684                Err(code) => Ok(vec![control_error_frame(
3685                    &frame,
3686                    code,
3687                    "activity query refused",
3688                )?]),
3689            };
3690        }
3691        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3692            Ok(request) => request,
3693            Err(_) => {
3694                return Ok(vec![control_error_frame(
3695                    &frame,
3696                    ACTIVITY_MALFORMED,
3697                    "malformed activity subscription",
3698                )?])
3699            }
3700        };
3701        let mut subscription = match ledger.subscribe(request.since).await {
3702            Ok(subscription) => subscription,
3703            Err(code) => {
3704                return Ok(vec![control_error_frame(
3705                    &frame,
3706                    code,
3707                    "activity subscription refused",
3708                )?])
3709            }
3710        };
3711        let key = (ctx.connection_id, frame.header.corr);
3712        let sink = ctx.egress.clone();
3713        let mut streams = self
3714            .activity_streams
3715            .lock()
3716            .unwrap_or_else(|p| p.into_inner());
3717        streams.retain(|_, task| !task.is_finished());
3718        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3719            return Ok(vec![control_error_frame(
3720                &frame,
3721                "duplicate_subscription",
3722                "activity subscription correlation id already in use",
3723            )?]);
3724        }
3725        let task = tokio::spawn(async move {
3726            loop {
3727                let entry = if let Some(entry) = subscription.replay.pop_front() {
3728                    Some(entry)
3729                } else {
3730                    subscription.receiver.recv().await
3731                };
3732                let Some(entry) = entry else {
3733                    break;
3734                };
3735                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3736                    return;
3737                };
3738                let Ok(next) = Frame::build_with_version(
3739                    response_version(&frame),
3740                    FrameType::StreamData,
3741                    control_flags(),
3742                    0,
3743                    0,
3744                    frame.header.corr,
3745                    body,
3746                ) else {
3747                    return;
3748                };
3749                if sink.send(next).await.is_err() {
3750                    return;
3751                }
3752            }
3753            // Dropping a full subscriber's sender drains its accepted entries
3754            // before the terminal error, preserving their order on the wire.
3755            if let Some(code) = subscription.terminal_code() {
3756                if let Ok(end) =
3757                    control_error_frame(&frame, code, "activity subscriber fell behind")
3758                {
3759                    let _ = sink.send(end).await;
3760                }
3761            }
3762        });
3763        streams.insert(key, task.abort_handle());
3764        Ok(Vec::new())
3765    }
3766
3767    async fn handle_activity_record(
3768        &self,
3769        ctx: &RouteCtx,
3770        frame: Frame,
3771    ) -> Result<Vec<Frame>, RouterError> {
3772        use subc_control::{
3773            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3774            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3775        };
3776        let result = async {
3777            let module = self
3778                .forwarding
3779                .module_id_for_connection(ctx.connection_id)
3780                .ok()
3781                .flatten()
3782                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3783            let request: ActivityRecordRequest =
3784                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3785            crate::activity::validate(&module, &request)?;
3786            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3787            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3788            let at_ms = std::time::SystemTime::now()
3789                .duration_since(std::time::UNIX_EPOCH)
3790                .unwrap_or_default()
3791                .as_millis() as u64;
3792            ledger.record(module, request, actor, at_ms).await
3793        }
3794        .await;
3795        match result {
3796            Ok(entry) => Ok(vec![control_response_body_frame(
3797                &frame,
3798                &ActivityRecordResponse { seq: entry.seq },
3799                "activity.record",
3800            )?]),
3801            Err(code) => Ok(vec![control_error_frame(
3802                &frame,
3803                code,
3804                "activity recording refused",
3805            )?]),
3806        }
3807    }
3808
3809    fn activity_actor(
3810        &self,
3811        connection: ConnectionId,
3812        module: &str,
3813        on: Option<&subc_control::ActivityOn>,
3814    ) -> Result<subc_control::ActivityActor, &'static str> {
3815        use crate::forwarding::{DataRoute, DataRouteState};
3816        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3817        let principal = |id: &str| Principal::Reserved {
3818            module_id: id.to_owned(),
3819        };
3820        let mut actor = ActivityActor {
3821            principal: principal(module),
3822            scope_owner: None,
3823            agent_id: None,
3824            run_id: None,
3825            flow_id: None,
3826            project_root: None,
3827            workdir: None,
3828            project_id: None,
3829        };
3830        let scope = match on {
3831            None => return Ok(actor),
3832            Some(ActivityOn::Scope(scope)) => {
3833                let selector = subc_protocol::scope::ScopeSelector::new(
3834                    principal(module),
3835                    scope.scope_ref.clone(),
3836                    Some(scope.epoch),
3837                );
3838                self.scopes
3839                    .read()
3840                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3841                    .attribute(&principal(module), module, &selector, true)
3842                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3843            }
3844            Some(ActivityOn::Route(route)) => {
3845                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3846                    .forwarding
3847                    .lookup_data_route(connection, route.channel, route.epoch)
3848                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3849                else {
3850                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3851                };
3852                if binding.module_id != module {
3853                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3854                }
3855                actor.principal = binding.principal.clone();
3856                actor.project_root = binding
3857                    .project_root
3858                    .as_ref()
3859                    .map(|root| root.as_path().to_path_buf());
3860                let Some(bound) = &binding.scope else {
3861                    return Ok(actor);
3862                };
3863                let owner = principal(&bound.owner);
3864                let selector = subc_protocol::scope::ScopeSelector::new(
3865                    owner.clone(),
3866                    bound.scope_ref.clone(),
3867                    Some(bound.tag.scope_epoch),
3868                );
3869                let admitted = self
3870                    .scopes
3871                    .read()
3872                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3873                    .attribute(&owner, module, &selector, true)
3874                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3875                if admitted.tag != bound.tag {
3876                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3877                }
3878                admitted
3879            }
3880        };
3881        actor.scope_owner = Some(scope.stamp.owner);
3882        actor.agent_id = scope.stamp.attributes.agent_id;
3883        actor.run_id = scope.stamp.attributes.run_id;
3884        actor.flow_id = scope.stamp.attributes.flow_id;
3885        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3886        actor.project_id = scope.stamp.project_id;
3887        Ok(actor)
3888    }
3889
3890    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3891        self.refresh_capability_requirements();
3892        // A bare connection count is ambiguous between many clients holding a
3893        // route each and one client accumulating hundreds, so publish the
3894        // concentration alongside it. Route state is best-effort here: a
3895        // diagnostic endpoint must still answer if the forwarding lock is
3896        // contended.
3897        let mut counters = self.counters.snapshot();
3898        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3899            obj.insert(
3900                "activity_unparseable_lines".into(),
3901                ledger.read_errors().into(),
3902            );
3903            obj.insert(
3904                "activity_lifecycle_append_failures".into(),
3905                ledger.lifecycle_failures().into(),
3906            );
3907            obj.insert(
3908                "activity_started_digest_skips".into(),
3909                ledger.lifecycle_digest_skips().into(),
3910            );
3911        }
3912        if let (Ok((connections_with_routes, max)), Some(obj)) = (
3913            self.forwarding.client_route_concentration(),
3914            counters.as_object_mut(),
3915        ) {
3916            obj.insert(
3917                "client_connections_with_routes".into(),
3918                connections_with_routes.into(),
3919            );
3920            obj.insert("max_routes_on_one_connection".into(), max.into());
3921        }
3922        // A module that is being fast-refused and a module that is fine look
3923        // identical from a client that retries and succeeds, so name the open
3924        // breakers here. This rides the existing free-form counters object
3925        // rather than a new wire field, so no sibling that deserializes
3926        // `ServerDescribe` has to be rebuilt to keep reading it.
3927        if let (Some(open_breakers), Some(obj)) = (
3928            self.route_bind_breakers.open_snapshot(),
3929            counters.as_object_mut(),
3930        ) {
3931            obj.insert("route_bind_breakers_open".into(), open_breakers);
3932        }
3933        let response = ClientControlResponse::ServerDescribe {
3934            protocol_ver: PROTOCOL_VERSION,
3935            subc_ops: subc_ops(),
3936            capabilities: self.describe_capabilities(),
3937            connected_clients: self.connected_clients.count(),
3938            counters: Some(counters),
3939            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3940            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3941            capability_requirements: self.capability_requirement_statuses(),
3942            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3943            config_path: self
3944                .rescan
3945                .as_ref()
3946                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3947        };
3948        Ok(vec![control_response_body_frame(
3949            &frame,
3950            &response,
3951            "ClientControlResponse::ServerDescribe",
3952        )?])
3953    }
3954
3955    fn handle_catalog_list(
3956        &self,
3957        frame: Frame,
3958        module_id: Option<String>,
3959    ) -> Result<Vec<Frame>, RouterError> {
3960        let (generation, modules) = self
3961            .registry
3962            .list_modules_with_nonce_verification()
3963            .map_err(|err| {
3964                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3965            })?;
3966        let entries = modules
3967            .into_iter()
3968            .filter(|(registration, _)| {
3969                module_id
3970                    .as_deref()
3971                    .map(|wanted| registration.manifest.module_id == wanted)
3972                    .unwrap_or(true)
3973            })
3974            .map(|(registration, nonce_verified)| {
3975                let not_ready = self.not_ready_reason(&registration);
3976                let roles = registration.manifest.provides;
3977                CatalogEntry::new(
3978                    registration.manifest.module_id,
3979                    roles,
3980                    registration.control_ops,
3981                )
3982                .with_ready(not_ready.is_none())
3983                .with_not_ready(not_ready)
3984                .with_module_version(Some(registration.manifest.module_version))
3985                .with_capabilities(registration.manifest.capabilities)
3986                .with_self_signals(registration.manifest.self_signals)
3987                .with_events(registration.manifest.events)
3988                .with_operation_delivery(registration.manifest.operation_delivery)
3989                .with_operation_payload(registration.manifest.operation_payload)
3990                .with_rich_ui(registration.manifest.rich_ui)
3991                .with_nonce_verified(nonce_verified)
3992            })
3993            .collect();
3994        let response = ClientControlResponse::CatalogList {
3995            generation,
3996            modules: entries,
3997            subc_ops: subc_ops(),
3998        };
3999        Ok(vec![control_response_body_frame(
4000            &frame,
4001            &response,
4002            "ClientControlResponse::CatalogList",
4003        )?])
4004    }
4005
4006    fn route_open_principal(
4007        &self,
4008        frame: &Frame,
4009        consumer_identity: Option<ConsumerIdentity>,
4010    ) -> Result<Result<Principal, Frame>, RouterError> {
4011        let Some(consumer_identity) = consumer_identity else {
4012            return Ok(Ok(Principal::Direct));
4013        };
4014
4015        if self.supervisor.spawned_consumer_authorized(
4016            &consumer_identity.module_id,
4017            &consumer_identity.launch_nonce,
4018        ) {
4019            return Ok(Ok(Principal::Reserved {
4020                module_id: consumer_identity.module_id,
4021            }));
4022        }
4023
4024        Ok(Err(control_error_frame(
4025            frame,
4026            "bad_consumer_identity",
4027            format!(
4028                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4029                consumer_identity.module_id
4030            ),
4031        )?))
4032    }
4033
4034    /// Ordinary `route.open` refusals go through here; admission and breaker
4035    /// refusals log separately with their capacity or breaker state. The daemon can
4036    /// attest which code it sent: without the event, a client's "the daemon
4037    /// refused me" and the daemon's own view could only be reconciled by
4038    /// argument. Malformed input (`invalid_project_root`) does not come here;
4039    /// rejecting a request that was never a valid open is not a refusal of one.
4040    fn route_open_refusal_frame(
4041        &self,
4042        ctx: &RouteCtx,
4043        frame: &Frame,
4044        module_id: &str,
4045        reason: &'static str,
4046        code: &'static str,
4047        message: impl Into<String>,
4048    ) -> Result<Frame, RouterError> {
4049        self.observe_route_open_refusal(ctx, module_id, reason, code);
4050        control_error_frame(frame, code, message.into())
4051    }
4052
4053    /// Refuse a `route.open` because the target module's bind-relay breaker is
4054    /// open, without attempting the relay.
4055    ///
4056    /// The wire code is `module_timeout`, which is the truth (the module has
4057    /// not been answering binds) and which both SDKs already classify as
4058    /// retryable with capped backoff. Reusing it is what keeps this change out
4059    /// of both SDKs; the daemon-side distinction lives in the counter key
4060    /// instead.
4061    ///
4062    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
4063    /// While a breaker is open this fires on every open to that module, and the
4064    /// stall written up in `docs/designs/route-open-head-of-line.md` already
4065    /// produced 261 lines about a single module inside 3000 lines of daemon
4066    /// log. The rare transitions are logged at warn/info instead and the volume
4067    /// is carried by the counter, so the evidence survives without the flood.
4068    /// The debug line keeps a per-refusal record reachable for whoever turns
4069    /// the level up.
4070    fn route_open_breaker_refusal_frame(
4071        &self,
4072        ctx: &RouteCtx,
4073        frame: &Frame,
4074        module_id: &str,
4075        consecutive_timeouts: u32,
4076        retry_in: Duration,
4077        probe_in_flight: bool,
4078    ) -> Result<Frame, RouterError> {
4079        self.counters
4080            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4081        debug!(
4082            target: "control",
4083            code = "module_timeout",
4084            module_id = ?module_id,
4085            connection_id = ctx.connection_id.get(),
4086            consecutive_timeouts,
4087            retry_in_ms = retry_in.as_millis() as u64,
4088            probe_in_flight,
4089            "route.open refused by open bind-relay breaker"
4090        );
4091        // Say what a caller can act on. An open bind-relay breaker means the
4092        // module timed out accepting several new routes in a row. The module
4093        // is still running and its established routes keep working; only new
4094        // route.open requests are refused until the cooldown ends and one
4095        // test route (the probe) gets through. A message that only counts
4096        // failed relays reads as "the module is down" to a worker that sees it.
4097        let detail = if probe_in_flight {
4098            "one test route is already being tried; retry once it settles".to_string()
4099        } else {
4100            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4101        };
4102        control_error_frame(
4103            frame,
4104            "module_timeout",
4105            format!(
4106                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4107                 timed out in a row); {detail}; its established routes are unaffected"
4108            ),
4109        )
4110    }
4111
4112    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4113    /// requester's bytes (an unknown target is whatever the client sent) and
4114    /// is Debug-formatted so control characters land in the log escaped
4115    /// rather than as terminal sequences for whoever tails it.
4116    ///
4117    /// `reason` names the check that refused, because one wire code has
4118    /// several senders: after a module registers, `target_unavailable` can
4119    /// come from a missing role, an inactive registration, a supervisor that
4120    /// has not marked the process live, a missing forwarding connection, or a
4121    /// failed relay, and a log that records only the code cannot say which of
4122    /// them fired. It is a static, daemon-chosen label per branch, so it is
4123    /// safe to print plainly and stays a closed set.
4124    fn observe_route_open_refusal(
4125        &self,
4126        ctx: &RouteCtx,
4127        module_id: &str,
4128        reason: &'static str,
4129        code: &'static str,
4130    ) {
4131        self.counters.increment_route_open_refused(code);
4132        info!(
4133            target: "control",
4134            code,
4135            reason,
4136            module_id = ?module_id,
4137            connection_id = ctx.connection_id.get(),
4138            "route.open refused"
4139        );
4140        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4141            self.route_outages.record_not_serving(module_id, reason);
4142        }
4143    }
4144
4145    /// Record an ACCEPTED route.open.
4146    ///
4147    /// Refusals have been logged and counted since the attestation work; accepts
4148    /// were invisible, so the daemon knew every principal it stamped and wrote
4149    /// none of them down. The party that attests the identity was the only party
4150    /// not recording it, which left a credential vault unable to name the sender
4151    /// of a call that reached it (claustrum #43) and left the launch-nonce
4152    /// concurrency question unanswerable from the outside.
4153    ///
4154    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4155    /// `code`/`module_id`/`connection_id` returns both directions of the same
4156    /// decision rather than two shapes a reader has to join by hand.
4157    ///
4158    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4159    /// and the difference carries information rather than being an
4160    /// inconsistency. This line is only reachable after a successful bind to a
4161    /// REGISTERED module, so the value has already passed HELLO validation
4162    /// including the path-hazard refusal and cannot contain control bytes. A
4163    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4164    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4165    ///
4166    /// Bare is also what every other daemon line already emits (`module
4167    /// registered`, `configured module supervised`). Shipping `?module_id` here
4168    /// made this instrument the only one in the file whose ids did not answer
4169    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4170    /// line whose whole purpose is being grepped beside its sibling.
4171    ///
4172    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4173    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4174    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4175    /// one are recorded identically. A test written against that harness passes
4176    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4177    /// green assertion that cannot fail. The same limit applies to the escaping
4178    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4179    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4180    /// Fencing either needs the real formatter, not the capture layer.
4181    ///
4182    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4183    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4184    /// to record. The ephemeral port would decay within minutes and answer only a
4185    /// live question. The identity question is instead answered by counting
4186    /// distinct live connections presenting one module's `consumer_identity` --
4187    /// "is anyone else holding this secret" rather than "is this the right
4188    /// process".
4189    fn observe_route_open_accept(
4190        &self,
4191        ctx: &RouteCtx,
4192        module_id: &str,
4193        principal: &str,
4194        scope: &str,
4195    ) {
4196        self.route_outages.record_accepted(module_id);
4197        self.counters.increment_route_open_accepted(principal);
4198        info!(
4199            target: "control",
4200            principal,
4201            module_id,
4202            connection_id = ctx.connection_id.get(),
4203            scope = %scope,
4204            "route.open accepted"
4205        );
4206    }
4207
4208    fn supervised_absent_route_open_refusal_frame(
4209        &self,
4210        ctx: &RouteCtx,
4211        frame: &Frame,
4212        module_id: &str,
4213        code: &'static str,
4214        status: &crate::supervise::ModuleStatus,
4215    ) -> Result<Frame, RouterError> {
4216        self.counters.increment_route_open_refused(code);
4217        info!(
4218            target: "control",
4219            code,
4220            reason = "supervised_not_registered",
4221            module_id = ?module_id,
4222            connection_id = ctx.connection_id.get(),
4223            state = %status.state,
4224            enabled = status.enabled,
4225            live = status.live,
4226            "route.open refused"
4227        );
4228        // A supervised module whose process has not registered is not
4229        // serving, whatever the reason; the supervisor knows this id, so it is
4230        // safe to track.
4231        self.route_outages
4232            .record_not_serving(module_id, "supervised_not_registered");
4233        let message = if let Some(pid) = status.waiting_for_previous_pid {
4234            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4235                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4236            })
4237        } else {
4238            format!(
4239                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4240                status.state, status.enabled, status.live
4241            )
4242        };
4243        control_error_frame(frame, code, message)
4244    }
4245
4246    async fn handle_route_open(
4247        &self,
4248        ctx: &RouteCtx,
4249        frame: Frame,
4250        request: RouteOpenRequest,
4251        timings: &mut RouteOpenTimings,
4252    ) -> Result<Vec<Frame>, RouterError> {
4253        let RouteOpenRequest {
4254            target,
4255            mut identity,
4256            consumer_identity,
4257            consumer_capabilities,
4258            role_versions,
4259            admission_facts,
4260            scope,
4261        } = request;
4262        let target_module_id = target_module_id(&target).to_string();
4263        timings.module_id.clone_from(&target_module_id);
4264        if self
4265            .registry
4266            .get_module_by_connection(ctx.connection_id)
4267            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4268            .is_some()
4269        {
4270            return Ok(vec![control_error_frame(
4271                &frame,
4272                "invalid_request",
4273                "module connections cannot open client routes",
4274            )?]);
4275        }
4276        debug!(
4277            connection_id = ctx.connection_id.get(),
4278            corr = frame.header.corr,
4279            module_id = %target_module_id,
4280            "handling route.open"
4281        );
4282
4283        // A malformed declaration is refused first, before anything about the
4284        // target is looked up: the same body would be refused against any
4285        // module, so the caller learns nothing by retrying or waiting. An empty
4286        // map declares nothing and travels as no field at all, so a provider
4287        // only ever sees a missing field or a non-empty one.
4288        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4289        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4290            self.observe_route_open_refusal(
4291                ctx,
4292                &target_module_id,
4293                "invalid_role_versions",
4294                error_codes::INVALID_REQUEST,
4295            );
4296            return Ok(vec![control_error_body_frame(
4297                &frame,
4298                ErrorBody {
4299                    code: error_codes::INVALID_REQUEST.to_string(),
4300                    message: error.to_string(),
4301                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4302                },
4303            )?]);
4304        }
4305
4306        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4307            if status.waiting_for_previous_pid.is_some() {
4308                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4309                    ctx,
4310                    &frame,
4311                    &target_module_id,
4312                    "module_warming",
4313                    &status,
4314                )?]);
4315            }
4316        }
4317
4318        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4319        // opposite. Below, a caller learns whether a module is unregistered,
4320        // supervised-but-down (with state/enabled/live), or registered without the
4321        // requested role. Elsewhere that is an enumeration leak: a probe learning
4322        // the shape of a fleet it cannot otherwise see.
4323        //
4324        // It is not one here, and the reason is the ACCESS MODEL rather than
4325        // anything about these errors. Reaching route.open requires the
4326        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4327        // connection file, so any caller who completes it already runs as this
4328        // user -- and can read subc.jsonc for the module list and `ck module
4329        // status` for live state. The reply discloses nothing the caller cannot
4330        // read more easily from disk, while the precision is load-bearing:
4331        // `unknown_module` is retryable and a missing role is not.
4332        //
4333        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4334        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4335        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4336        let Some(registration) = self
4337            .registry
4338            .get_module(&target_module_id)
4339            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4340        else {
4341            if let Some((status, warming)) =
4342                self.supervisor_status(&target_module_id, frame.header.corr)?
4343            {
4344                // BEFORE the two availability codes below, because for a module
4345                // that speaks no subc wire both of them are false comfort: they
4346                // say "not right now" and are retried, and this module will
4347                // never register no matter how long the caller waits. The
4348                // absence here is the declaration being honoured, not a module
4349                // that is late.
4350                if status.protocol == ModuleProtocol::None {
4351                    return Ok(vec![self.route_open_refusal_frame(
4352                        ctx,
4353                        &frame,
4354                        &target_module_id,
4355                        "protocol_none",
4356                        error_codes::MODULE_NO_PROTOCOL,
4357                        format!(
4358                            "module_id '{target_module_id}' is declared protocol: none; \
4359                             it speaks no subc wire and serves no routes"
4360                        ),
4361                    )?]);
4362                }
4363                let code = if warming {
4364                    "module_warming"
4365                } else {
4366                    "target_unavailable"
4367                };
4368                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4369                    ctx,
4370                    &frame,
4371                    &target_module_id,
4372                    code,
4373                    &status,
4374                )?]);
4375            }
4376            if let Some(removed_ago_ms) =
4377                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4378            {
4379                return Ok(vec![self.route_open_refusal_frame(
4380                    ctx,
4381                    &frame,
4382                    &target_module_id,
4383                    "removed",
4384                    error_codes::MODULE_REMOVED,
4385                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4386                )?]);
4387            }
4388            return Ok(vec![self.route_open_refusal_frame(
4389                ctx,
4390                &frame,
4391                &target_module_id,
4392                "not_registered",
4393                error_codes::UNKNOWN_MODULE,
4394                format!("module_id '{target_module_id}' is not registered"),
4395            )?]);
4396        };
4397
4398        // Best-effort only: registry readiness and forwarding reservation use
4399        // different locks, so a module can flip readiness between this read and
4400        // the relay. Modules must still tolerate an `on_bind` while not ready.
4401        if !registration.ready {
4402            self.counters
4403                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4404            info!(
4405                target: "control",
4406                code = error_codes::MODULE_WARMING,
4407                module_id = ?target_module_id,
4408                connection_id = ctx.connection_id.get(),
4409                reason = "declared_not_ready",
4410                "route.open refused"
4411            );
4412            // The module is registered but says it cannot take work, which is
4413            // an outage from the caller's side even though its process is up.
4414            self.route_outages
4415                .record_not_serving(&target_module_id, "declared_not_ready");
4416            return Ok(vec![control_error_body_frame(
4417                &frame,
4418                ErrorBody {
4419                    code: error_codes::MODULE_WARMING.to_string(),
4420                    message: format!(
4421                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4422                    ),
4423                    detail: Some(serde_json::json!({
4424                        "reason": "declared_not_ready"
4425                    })),
4426                },
4427            )?]);
4428        }
4429
4430        // Effective readiness, second half: a module that declares a capability
4431        // `need: required` is not routable while that capability has no
4432        // registered provider. It is enforced HERE, as a retryable routing
4433        // refusal, and deliberately not as spawn ordering or a boot block. The
4434        // module is still started and registered and can make its own calls;
4435        // spawn ordering is a promise that cannot be kept once a provider
4436        // crashes at runtime, and refusing to boot would stop the whole
4437        // machine, including the tools needed to fix its configuration.
4438        //
4439        // "Provided" is the evaluator's verdict, which counts a provider as
4440        // soon as it has REGISTERED, not once it is ready. Two modules that
4441        // require each other's capabilities are therefore both routable once
4442        // both register; counting readiness instead would deadlock them.
4443        //
4444        // Only new opens are refused. Routes already bound when a provider
4445        // goes away stay bound: nothing here tears them down, and the module
4446        // answers them as it can. Like the readiness read above this is
4447        // best-effort against a provider registering or leaving concurrently.
4448        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
4449            self.counters
4450                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4451            info!(
4452                target: "control",
4453                code = error_codes::MODULE_WARMING,
4454                module_id = ?target_module_id,
4455                connection_id = ctx.connection_id.get(),
4456                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4457                capability = %capability,
4458                "route.open refused"
4459            );
4460            return Ok(vec![control_error_body_frame(
4461                &frame,
4462                ErrorBody {
4463                    code: error_codes::MODULE_WARMING.to_string(),
4464                    message: format!(
4465                        "module_id '{target_module_id}' requires capability '{capability}', \
4466                         which no registered module provides; retry"
4467                    ),
4468                    detail: Some(serde_json::json!({
4469                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4470                        "capability": capability,
4471                    })),
4472                },
4473            )?]);
4474        }
4475
4476        if !target_has_required_role(&target, &registration.manifest.provides) {
4477            return Ok(vec![self.route_open_refusal_frame(
4478                ctx,
4479                &frame,
4480                &target_module_id,
4481                "role_not_provided",
4482                "target_unavailable",
4483                format!("module_id '{target_module_id}' does not provide the requested target"),
4484            )?]);
4485        }
4486
4487        if registration.state != ChannelState::Active {
4488            return Ok(vec![self.route_open_refusal_frame(
4489                ctx,
4490                &frame,
4491                &target_module_id,
4492                "registration_not_active",
4493                "target_unavailable",
4494                format!("module_id '{target_module_id}' is not active"),
4495            )?]);
4496        }
4497
4498        if self
4499            .forwarding
4500            .module_is_draining(&target_module_id)
4501            .map_err(RouterError::Forwarding)?
4502        {
4503            return Ok(vec![self.route_open_refusal_frame(
4504                ctx,
4505                &frame,
4506                &target_module_id,
4507                "reloading",
4508                "module_reloading",
4509                format!("module_id '{target_module_id}' is reloading"),
4510            )?]);
4511        }
4512
4513        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4514            process_liveness.process_live(&target_module_id) == Some(false)
4515        }) {
4516            // A module the supervisor is restarting or reloading can still hold
4517            // a registration: the old process before its connection closes, or
4518            // a new one that registered while the supervisor was draining. The
4519            // forwarding table does not see that as draining, but the consumer
4520            // should still be told to retry soon, exactly as for the drain
4521            // above, rather than that the target is unavailable.
4522            if process_liveness.process_replacing(&target_module_id) {
4523                return Ok(vec![self.route_open_refusal_frame(
4524                    ctx,
4525                    &frame,
4526                    &target_module_id,
4527                    "reloading",
4528                    "module_reloading",
4529                    format!("module_id '{target_module_id}' is reloading"),
4530                )?]);
4531            }
4532            return Ok(vec![self.route_open_refusal_frame(
4533                ctx,
4534                &frame,
4535                &target_module_id,
4536                "supervisor_not_live",
4537                "target_unavailable",
4538                format!("module_id '{target_module_id}' is not live"),
4539            )?]);
4540        }
4541
4542        if !self
4543            .forwarding
4544            .has_live_module_connection(&target_module_id)
4545            .map_err(RouterError::Forwarding)?
4546        {
4547            return Ok(vec![self.route_open_refusal_frame(
4548                ctx,
4549                &frame,
4550                &target_module_id,
4551                "no_forwarding_connection",
4552                "target_unavailable",
4553                format!("module_id '{target_module_id}' has no live forwarding connection"),
4554            )?]);
4555        }
4556
4557        if let Some(error) =
4558            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4559        {
4560            self.observe_route_open_refusal(
4561                ctx,
4562                &target_module_id,
4563                "op_not_allowed",
4564                "op_not_allowed",
4565            );
4566            return Ok(vec![error]);
4567        }
4568
4569        let principal = match self.route_open_principal(&frame, consumer_identity)? {
4570            Ok(principal) => principal,
4571            Err(error) => {
4572                self.observe_route_open_refusal(
4573                    ctx,
4574                    &target_module_id,
4575                    "bad_consumer_identity",
4576                    "bad_consumer_identity",
4577                );
4578                return Ok(vec![error]);
4579            }
4580        };
4581        timings.principal = principal_label(&principal);
4582
4583        // This is attested, control-plane policy for supervised module origins.
4584        // Keep it before route reservation and out of the opaque forwarding hot
4585        // path: data frames must never acquire a per-frame capability check.
4586        if let Principal::Reserved {
4587            module_id: opening_module_id,
4588        } = &principal
4589        {
4590            if let Some(opening_registration) = self
4591                .registry
4592                .get_module(opening_module_id)
4593                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4594            {
4595                if let Some(capability) =
4596                    denied_capability(&opening_registration.manifest, &registration.manifest)
4597                {
4598                    warn!(
4599                        opening_module_id,
4600                        target_module_id,
4601                        capability,
4602                        "refusing route.open because an attested capability deny edge matches"
4603                    );
4604                    return Ok(vec![self.route_open_refusal_frame(
4605                        ctx,
4606                        &frame,
4607                        &target_module_id,
4608                        "capability_deny_edge",
4609                        "capability_forbidden",
4610                        format!(
4611                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4612                        ),
4613                    )?]);
4614                }
4615            }
4616        }
4617
4618        if admission_facts.is_some() {
4619            let carrier_matches = matches!(
4620                &principal,
4621                Principal::Reserved { module_id }
4622                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4623            );
4624            if !carrier_matches {
4625                return Ok(vec![self.route_open_refusal_frame(
4626                    ctx,
4627                    &frame,
4628                    &target_module_id,
4629                    "admission_facts_carrier_not_permitted",
4630                    "admission_facts_not_permitted",
4631                    "admission facts may only be carried by the configured reserved module",
4632                )?]);
4633            }
4634
4635            let target_allowed = self
4636                .admission_facts_targets
4637                .as_ref()
4638                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4639            if !target_allowed {
4640                return Ok(vec![self.route_open_refusal_frame(
4641                    ctx,
4642                    &frame,
4643                    &target_module_id,
4644                    "admission_facts_target_not_listed",
4645                    "admission_facts_target_not_allowed",
4646                    format!(
4647                        "admission facts are not permitted for target module_id '{target_module_id}'"
4648                    ),
4649                )?]);
4650            }
4651
4652            // Keep the value opaque to subc. The downstream admission validator owns
4653            // schema and semantic checks; this daemon only enforces carrier authority
4654            // and the configured destination allowlist.
4655        }
4656
4657        // Scope admission, on the attested principal above and never on the
4658        // request body. The tag read here travels with the pending bind and is
4659        // compared with the published one at commit, so a sync between here
4660        // and the module's ack refuses the open instead of binding a stamp
4661        // that is no longer true.
4662        let (bound_scope, mut scope_stamp) = match &scope {
4663            None => (None, None),
4664            Some(selector) => {
4665                let owner_configured = match &selector.owner {
4666                    // A reserved owner counts as configured from before its
4667                    // process is spawned (see `SupervisorHandle::is_configured`).
4668                    // So an owner that has not synced its scopes yet is refused
4669                    // as retryable (`scope_not_synced`), not as one that will
4670                    // never sync.
4671                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4672                    _ => false,
4673                };
4674                let mut scopes = self
4675                    .scopes
4676                    .write()
4677                    .unwrap_or_else(|poisoned| poisoned.into_inner());
4678                let admitted = scopes
4679                    .admit(&principal, &target_module_id, selector, owner_configured)
4680                    .and_then(|admission| {
4681                        crate::scopes::check_target_flow_support(
4682                            &admission.stamp,
4683                            &target_module_id,
4684                            registration.manifest.capabilities.as_ref(),
4685                        )?;
4686                        crate::scopes::check_target_agent_run_support(
4687                            &admission.stamp,
4688                            &target_module_id,
4689                            registration.manifest.capabilities.as_ref(),
4690                        )?;
4691                        let bound = scopes.reserve_bind(
4692                            &admission,
4693                            &principal,
4694                            ctx.connection_id,
4695                            selector.hold_presence,
4696                        );
4697                        Ok((admission, bound))
4698                    });
4699                match admitted {
4700                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4701                    Err(refusal) => {
4702                        return Ok(vec![self.route_open_refusal_frame(
4703                            ctx,
4704                            &frame,
4705                            &target_module_id,
4706                            refusal.code,
4707                            refusal.code,
4708                            refusal.message,
4709                        )?]);
4710                    }
4711                }
4712            }
4713        };
4714
4715        // Bind admits a root that no longer exists on disk, because refusing here
4716        // closes the only exit from a paused run: cancel needs a bound route, and a
4717        // renamed or reclaimed directory makes that route unopenable forever. The
4718        // run itself is intact and still addressable by its recorded identity.
4719        //
4720        // This does NOT relax the rule the strict constructor protects. That rule is
4721        // that no root is ever aliased into NEW durable state -- a missing component
4722        // can reappear as a symlink elsewhere, which would move the identity and
4723        // split a session's history across two of them. The engine now refuses the
4724        // two operations that create such state (send and import) at admission,
4725        // which is a narrower way to hold the same invariant: reads and terminations
4726        // are admitted, writes are not. That refusal had to ship before this line
4727        // changed, or there is an interval where a send commits under a provisional
4728        // identity -- the exact failure the original policy existed to prevent.
4729        //
4730        // Resolution follows realpath rather than lexical cleanup: the longest
4731        // existing ancestor is canonicalized and the missing tail re-appended, so a
4732        // live root is unchanged and a vanished leaf keeps the identity it was
4733        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
4734        // same caller the moment the directory vanished, which strands the run more
4735        // quietly than refusing it.
4736        timings.enter(RouteOpenPhase::ProjectRoot);
4737        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4738            Ok(project_root) => project_root,
4739            Err(err) => {
4740                return Ok(vec![control_error_frame(
4741                    &frame,
4742                    "invalid_project_root",
4743                    err.to_string(),
4744                )?])
4745            }
4746        };
4747        timings.enter(RouteOpenPhase::Admission);
4748        identity.project_root = project_root.as_path().to_path_buf();
4749
4750        // Last gate before any relay work, and deliberately after the cheap
4751        // registry and availability checks above: those name a more precise
4752        // condition (unknown, removed, reloading) and a caller is better served
4753        // by the precise code than by this one.
4754        //
4755        // Everything below this point costs an egress permit, a reserved handle
4756        // pair and, if the module does not answer, the whole relay budget. The
4757        // reader no longer waits for that budget, so cap each target explicitly;
4758        // serial dispatch used to provide the accidental cap of one relay per
4759        // connection. Admission is a mutex-protected count and never waits.
4760        let _concurrency_guard = match self
4761            .route_bind_concurrency
4762            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4763        {
4764            Ok(guard) => guard,
4765            Err(in_flight) => {
4766                return Ok(vec![self.route_open_target_capacity_refusal(
4767                    ctx,
4768                    &frame,
4769                    &target_module_id,
4770                    in_flight,
4771                )?]);
4772            }
4773        };
4774
4775        // A module that has already burned the whole budget `threshold` times
4776        // in a row does not get to charge it again until a probe says it recovered.
4777        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4778            RouteBindAdmission::Admitted { guard, probe } => {
4779                if probe {
4780                    info!(
4781                        module_id = %target_module_id,
4782                        connection_id = ctx.connection_id.get(),
4783                        "route.bind breaker half-open: admitting one probe"
4784                    );
4785                }
4786                guard
4787            }
4788            RouteBindAdmission::Refused {
4789                consecutive_timeouts,
4790                retry_in,
4791                probe_in_flight,
4792            } => {
4793                return Ok(vec![self.route_open_breaker_refusal_frame(
4794                    ctx,
4795                    &frame,
4796                    &target_module_id,
4797                    consecutive_timeouts,
4798                    retry_in,
4799                    probe_in_flight,
4800                )?]);
4801            }
4802        };
4803
4804        // Resolve the per-module budget here so the wait matches the operator's
4805        // intent for this specific target. A per-module override in
4806        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
4807        // daemons) wins over the daemon-wide default.
4808        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4809        let relay_deadline = Instant::now() + route_bind_relay_timeout;
4810        timings.enter(RouteOpenPhase::EgressReservation);
4811        let pending = match self
4812            .forwarding
4813            .begin_route_bind_relay_for(
4814                ctx.connection_id,
4815                ctx.egress.clone(),
4816                response_version(&frame),
4817                frame.header.corr,
4818                &target_module_id,
4819                principal.clone(),
4820                bound_scope,
4821                Some(project_root),
4822                relay_deadline,
4823            )
4824            .await
4825        {
4826            Ok(pending) => pending,
4827            Err(err) => {
4828                return Ok(vec![self.route_open_refusal_frame(
4829                    ctx,
4830                    &frame,
4831                    &target_module_id,
4832                    "relay_reservation_failed",
4833                    forwarding_error_code(&err),
4834                    err.to_string(),
4835                )?])
4836            }
4837        };
4838        timings.enter(RouteOpenPhase::Admission);
4839        let crate::forwarding::PendingRouteBindRelay {
4840            endpoint,
4841            module_sink,
4842            negotiated_ver,
4843            client_channel,
4844            client_epoch,
4845            module_channel,
4846            module_epoch,
4847            corr: relay_corr,
4848            receiver,
4849        } = pending;
4850        let mut reservation =
4851            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4852
4853        // Reserving egress can wait while a module reconnects or a swap cuts
4854        // over. Check the relay's captured connection, not the earlier by-id
4855        // lookup: the original module's flow/run capabilities cannot authorize
4856        // its replacement. The captured sink stays bound to that connection.
4857        let relay_registration = self
4858            .registry
4859            .get_module_by_connection(endpoint.connection_id)
4860            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4861        let relay_capabilities = relay_registration
4862            .as_ref()
4863            .and_then(|registration| registration.manifest.capabilities.as_ref());
4864        if let Some(selector) = &scope {
4865            // Egress reservation may wait across a scope update. Recheck the
4866            // proof against the current record before sending the captured bind.
4867            if let Err(refusal) = self
4868                .scopes
4869                .read()
4870                .unwrap_or_else(|poisoned| poisoned.into_inner())
4871                .check_bind_proof(&principal, selector)
4872            {
4873                reservation.release_and_disarm();
4874                return Ok(vec![self.route_open_refusal_frame(
4875                    ctx,
4876                    &frame,
4877                    &target_module_id,
4878                    refusal.code,
4879                    refusal.code,
4880                    refusal.message,
4881                )?]);
4882            }
4883        }
4884        if let Some(stamp) = scope_stamp
4885            .as_ref()
4886            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4887        {
4888            if let Err(refusal) = crate::scopes::check_target_flow_support(
4889                stamp,
4890                &target_module_id,
4891                relay_capabilities,
4892            )
4893            .and_then(|()| {
4894                crate::scopes::check_target_agent_run_support(
4895                    stamp,
4896                    &target_module_id,
4897                    relay_capabilities,
4898                )
4899            }) {
4900                reservation.release_and_disarm();
4901                return Ok(vec![self.route_open_refusal_frame(
4902                    ctx,
4903                    &frame,
4904                    &target_module_id,
4905                    refusal.code,
4906                    refusal.code,
4907                    refusal.message,
4908                )?]);
4909            }
4910        }
4911
4912        // Older readers may reject unknown fields. Select optional fields from
4913        // the captured endpoint's declaration, including on unscoped binds.
4914        if !relay_capabilities.is_some_and(|caps| {
4915            caps.provides
4916                .iter()
4917                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
4918        }) {
4919            if let Some(stamp) = scope_stamp.as_mut() {
4920                stamp.workdir = None;
4921                stamp.project_id = None;
4922            }
4923        }
4924        if !relay_capabilities.is_some_and(|caps| {
4925            caps.provides
4926                .iter()
4927                .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4928        }) {
4929            if let Some(stamp) = scope_stamp.as_mut() {
4930                stamp.attributes.on_behalf_of_agent = None;
4931            }
4932        }
4933        let call_mode = if relay_capabilities
4934            .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4935        {
4936            Some(derive_call_mode(scope_stamp.as_ref()))
4937        } else {
4938            if let Some(stamp) = scope_stamp.as_mut() {
4939                stamp.expires_at_ms = None;
4940            }
4941            None
4942        };
4943
4944        debug!(
4945            connection_id = ctx.connection_id.get(),
4946            client_channel,
4947            client_epoch,
4948            module_channel,
4949            module_epoch,
4950            "reserved route handle pair"
4951        );
4952        // Rendered BEFORE the move into the relay, because the accept arm below
4953        // is where it is logged and the principal is gone by then.
4954        let route_principal_label = principal_label(&principal);
4955        let scope_label = scope_stamp
4956            .as_ref()
4957            .map_or_else(|| "none".to_string(), route_open_scope_label);
4958        let relay = ModuleControlRequest::RouteBind {
4959            route_channel: module_channel,
4960            epoch: module_epoch,
4961            target,
4962            identity,
4963            principal: Some(principal),
4964            consumer_capabilities,
4965            role_versions,
4966            admission_facts,
4967            scope: scope_stamp,
4968            call_mode,
4969        };
4970        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4971            RouterError::backend(
4972                0,
4973                frame.header.corr,
4974                format!("failed to encode route.bind request: {err}"),
4975            )
4976        })?;
4977        let relay_frame = Frame::build_with_version(
4978            negotiated_ver,
4979            FrameType::Request,
4980            control_flags(),
4981            0,
4982            0,
4983            relay_corr,
4984            relay_body,
4985        )
4986        .map_err(RouterError::FrameBuild)?;
4987
4988        timings.enter(RouteOpenPhase::BindSend);
4989        let sent = module_sink.send(relay_frame).await;
4990        timings.enter(RouteOpenPhase::Admission);
4991        if let Err(err) = sent {
4992            reservation.release_and_disarm();
4993            return Ok(vec![self.route_open_refusal_frame(
4994                ctx,
4995                &frame,
4996                &target_module_id,
4997                "relay_send_failed",
4998                "target_unavailable",
4999                err.to_string(),
5000            )?]);
5001        }
5002
5003        #[cfg(test)]
5004        if let Some(hold) = &self.route_bind_relay_sent_hold {
5005            // First wait: tell the test the bind is queued. Second wait: stay
5006            // here until the test has driven whatever it wants to happen
5007            // between queueing the bind and marking it enqueued.
5008            hold.wait().await;
5009            hold.wait().await;
5010        }
5011
5012        // A missing relay is not by itself an abandoned bind: the module may
5013        // already have acked it and the route may be committed. Only a relay
5014        // dropped without any module reply needs this GOODBYE; one a reply
5015        // settled has already had its GOODBYE decided by the reply path.
5016        if self
5017            .forwarding
5018            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5019            .map_err(RouterError::Forwarding)?
5020            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5021        {
5022            self.send_abandoned_route_bind_goodbye(
5023                &module_sink,
5024                negotiated_ver,
5025                module_channel,
5026                module_epoch,
5027            );
5028        }
5029
5030        timings.enter(RouteOpenPhase::BindAckWait);
5031        let outcome = timeout_at(relay_deadline, receiver).await;
5032        timings.enter(RouteOpenPhase::Admission);
5033        match outcome {
5034            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5035                reservation.disarm();
5036                if breaker.record_accepted() {
5037                    info!(
5038                        module_id = %target_module_id,
5039                        "route.bind breaker closed: the probe was accepted"
5040                    );
5041                }
5042                self.observe_route_open_accept(
5043                    ctx,
5044                    &target_module_id,
5045                    &route_principal_label,
5046                    &scope_label,
5047                );
5048                Ok(Vec::new())
5049            }
5050            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5051                reservation.release_and_disarm();
5052                // A module that says no in microseconds is healthy. Rejection
5053                // is a different condition with its own refusal and must not
5054                // move the breaker.
5055                breaker.record_inconclusive();
5056                // The daemon's own commit re-check refused the bind because the
5057                // scope ended or changed after admission. The module accepted;
5058                // counting it as a module rejection would blame the module.
5059                let scope_code = match body.code.as_str() {
5060                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
5061                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
5062                    _ => None,
5063                };
5064                if let Some(code) = scope_code {
5065                    self.observe_route_open_refusal(
5066                        ctx,
5067                        &target_module_id,
5068                        "scope_changed_before_commit",
5069                        code,
5070                    );
5071                    return Ok(vec![control_error_body_frame(&frame, body)?]);
5072                }
5073                self.counters
5074                    .increment_route_open_refused("module_rejected");
5075                info!(
5076                    target: "control",
5077                    code = "module_rejected",
5078                    module_code = ?body.code,
5079                    module_id = ?target_module_id,
5080                    connection_id = ctx.connection_id.get(),
5081                    "route.open refused"
5082                );
5083                Ok(vec![control_error_body_frame(&frame, body)?])
5084            }
5085            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5086                reservation.release_and_disarm();
5087                breaker.record_inconclusive();
5088                // Fires when the module's connection closes while a relayed
5089                // bind is pending -- typically a caller racing a module restart
5090                // whose bind was relayed BEFORE the drain mark went up. Logged
5091                // because the caller sees only its own error and the fleet has
5092                // already spent one diagnosis round unable to tell this arm
5093                // from a relay timeout without daemon-side evidence.
5094                tracing::warn!(
5095                    module_id = %target_module_id,
5096                    "route.bind relay abandoned: {message}"
5097                );
5098                Ok(vec![self.route_open_refusal_frame(
5099                    ctx,
5100                    &frame,
5101                    &target_module_id,
5102                    "relay_abandoned",
5103                    "target_unavailable",
5104                    message,
5105                )?])
5106            }
5107            Ok(Err(_)) => {
5108                reservation.release_and_disarm();
5109                breaker.record_inconclusive();
5110                Ok(vec![self.route_open_refusal_frame(
5111                    ctx,
5112                    &frame,
5113                    &target_module_id,
5114                    "relay_waiter_canceled",
5115                    "target_unavailable",
5116                    "route.bind relay waiter was canceled before the module responded",
5117                )?])
5118            }
5119            Err(_) => {
5120                reservation.release_and_disarm();
5121                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5122                // answer at all is the one condition a fast refusal can
5123                // usefully stand in for; every other arm already answered.
5124                if let Some(opened) = breaker.record_timeout(
5125                    self.route_bind_breaker_threshold,
5126                    self.route_bind_breaker_cooldown,
5127                ) {
5128                    warn!(
5129                        module_id = %target_module_id,
5130                        consecutive_timeouts = opened.consecutive_timeouts,
5131                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5132                        reopened_after_probe = opened.reopened_after_probe,
5133                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5134                    );
5135                }
5136                // The generous budget just burned to no answer: the module is
5137                // registered and its connection is up, but its bind handler sat
5138                // on the ack for the full budget (warm-on-bind, cold configure,
5139                // or a wedged handler). Every earlier unavailability shape
5140                // fast-refuses BEFORE the relay, so this arm firing means the
5141                // slowness is module-side -- log it so the per-module timeline
5142                // is reconstructable without client audit rows.
5143                tracing::warn!(
5144                    module_id = %target_module_id,
5145                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5146                    "route.bind relay timed out: module did not ack within budget"
5147                );
5148                Ok(vec![self.route_open_refusal_frame(
5149                    ctx,
5150                    &frame,
5151                    &target_module_id,
5152                    "relay_timed_out",
5153                    "module_timeout",
5154                    format!(
5155                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5156                        route_bind_relay_timeout
5157                    ),
5158                )?])
5159            }
5160        }
5161    }
5162
5163    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5164        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5165            snapshot: self.supervisor.spawn_snapshot(),
5166        };
5167        Ok(vec![control_response_body_frame(
5168            &frame,
5169            &response,
5170            "ClientControlResponse::SupervisorSpawnSnapshot",
5171        )?])
5172    }
5173
5174    fn handle_supervisor_spawn_subscribe(
5175        &self,
5176        ctx: &RouteCtx,
5177        frame: Frame,
5178        since: Option<SpawnCursor>,
5179    ) -> Result<Vec<Frame>, RouterError> {
5180        match self.supervisor.subscribe_spawns(
5181            ctx.connection_id,
5182            frame.header.corr,
5183            response_version(&frame),
5184            since,
5185            ctx.egress.clone(),
5186        ) {
5187            Ok(()) => Ok(Vec::new()),
5188            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5189                Ok(vec![control_error_body_frame(
5190                    &frame,
5191                    ErrorBody {
5192                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5193                        message: "spawn cursor belongs to a different daemon incarnation"
5194                            .to_string(),
5195                        detail: Some(serde_json::json!({
5196                            "current_daemon_incarnation": current
5197                        })),
5198                    },
5199                )?])
5200            }
5201            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5202                &frame,
5203                ErrorBody {
5204                    code: "spawn_cursor_too_old".to_string(),
5205                    message: "spawn cursor predates the retained event ring".to_string(),
5206                    detail: Some(serde_json::json!({
5207                        "oldest_retained_cursor": oldest
5208                    })),
5209                },
5210            )?]),
5211            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5212                0,
5213                frame.header.corr,
5214                format!("failed to open supervisor spawn subscription: {error}"),
5215            )),
5216        }
5217    }
5218
5219    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5220        let generation = self
5221            .registry
5222            .generation()
5223            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5224        let mut modules = Vec::new();
5225        for module in self.supervisor.list() {
5226            let status = module.status_for_control("list").map_err(|err| {
5227                RouterError::backend(
5228                    0,
5229                    frame.header.corr,
5230                    format!("failed to read supervisor status: {err}"),
5231                )
5232            })?;
5233            let (configured, _) = module.configuration().map_err(|err| {
5234                RouterError::backend(
5235                    0,
5236                    frame.header.corr,
5237                    format!("failed to read module configuration: {err}"),
5238                )
5239            })?;
5240            // Status and configuration snapshots release their locks before the image probe awaits.
5241            let image = module.running_image_agreement().await;
5242            // Read per request so the figure is current when the operator asks;
5243            // the daemon samples nothing in between.
5244            let resources = Some(module.child_resource_usage());
5245            let mut verdict =
5246                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5247            verdict.spec = Some(launch_spec_agreement(
5248                &configured,
5249                status.process_alive,
5250                status.launch_spec_digest.as_ref(),
5251            ));
5252            let pending_reload = Some(verdict);
5253            let configured_enabled = self
5254                .capability_evaluator
5255                .configured_enabled(&status.module_id);
5256            // Keep the retired policy field on the wire for one release so
5257            // existing status consumers still receive the platform policy.
5258            modules.push(
5259                SupervisorEntry::new(
5260                    status.module_id,
5261                    status.state.to_string(),
5262                    status.enabled,
5263                    status.live,
5264                    status.health.status,
5265                )
5266                .with_configured_enabled(configured_enabled)
5267                .with_launch_nonce_env(Some(!cfg!(unix)))
5268                .with_launch_nonce_source(status.launch_nonce_source)
5269                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5270                .with_protocol(status.protocol)
5271                .with_pending_reload(pending_reload)
5272                .with_last_probe_ms(status.health.last_probe_ms)
5273                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5274                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5275                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5276                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5277                .with_restart_count(Some(status.restart_count))
5278                .with_max_restarts(Some(status.max_restarts))
5279                .with_lifetime_restarts(Some(status.lifetime_restarts))
5280                .with_spawn_generation(Some(status.spawn_generation))
5281                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5282                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5283                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5284                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5285                .with_resources(resources),
5286            );
5287        }
5288        let response = ClientControlResponse::SupervisorList {
5289            generation,
5290            modules,
5291        };
5292        Ok(vec![control_response_body_frame(
5293            &frame,
5294            &response,
5295            "ClientControlResponse::SupervisorList",
5296        )?])
5297    }
5298
5299    fn handle_supervisor_stderr_tail(
5300        &self,
5301        frame: Frame,
5302        module_id: String,
5303        max_lines: Option<u32>,
5304        max_bytes: Option<u32>,
5305    ) -> Result<Vec<Frame>, RouterError> {
5306        let Some(module) = self.supervisor.get(&module_id) else {
5307            return Ok(vec![control_error_frame(
5308                &frame,
5309                "unknown_module",
5310                format!("module_id '{module_id}' is not supervised"),
5311            )?]);
5312        };
5313
5314        let snapshot = module.stderr_tail(
5315            max_lines.map(|value| value as usize),
5316            max_bytes.map(|value| value as usize),
5317        );
5318
5319        let response = ClientControlResponse::SupervisorStderrTail {
5320            module_id,
5321            tail: StderrTail {
5322                capture: match snapshot.capture {
5323                    CaptureState::Captured => StderrCaptureState::Captured,
5324                    CaptureState::Incomplete { reason } => {
5325                        StderrCaptureState::Incomplete { reason }
5326                    }
5327                    CaptureState::NotCaptured { reason } => {
5328                        StderrCaptureState::NotCaptured { reason }
5329                    }
5330                },
5331                entries: snapshot
5332                    .entries
5333                    .into_iter()
5334                    .map(|entry| match entry {
5335                        TailEntry::Line {
5336                            text,
5337                            truncated,
5338                            at_ms,
5339                        } => StderrTailEntry::Line {
5340                            text,
5341                            truncated,
5342                            at_ms,
5343                        },
5344                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5345                    })
5346                    .collect(),
5347                dropped_lines: snapshot.dropped_lines,
5348            },
5349        };
5350        Ok(vec![control_response_body_frame(
5351            &frame,
5352            &response,
5353            "ClientControlResponse::SupervisorStderrTail",
5354        )?])
5355    }
5356
5357    async fn handle_supervisor_terminals(
5358        &self,
5359        frame: Frame,
5360        module_id: String,
5361    ) -> Result<Vec<Frame>, RouterError> {
5362        let Some(module) = self.supervisor.get(&module_id) else {
5363            return Ok(vec![control_error_frame(
5364                &frame,
5365                "unknown_module",
5366                format!("module_id '{module_id}' is not supervised"),
5367            )?]);
5368        };
5369
5370        // The journal read runs on a blocking thread: it can be megabytes of
5371        // file I/O and must not occupy a runtime worker.
5372        let terminals = module
5373            .read_durable_terminal_history()
5374            .await
5375            .map_err(|error| {
5376                RouterError::backend(
5377                    0,
5378                    frame.header.corr,
5379                    format!("failed to read terminal history: {error}"),
5380                )
5381            })?;
5382        let response = ClientControlResponse::SupervisorTerminals {
5383            module_id,
5384            terminals,
5385        };
5386        Ok(vec![control_response_body_frame(
5387            &frame,
5388            &response,
5389            "ClientControlResponse::SupervisorTerminals",
5390        )?])
5391    }
5392
5393    fn handle_supervisor_routes(
5394        &self,
5395        frame: Frame,
5396        module_id: Option<String>,
5397    ) -> Result<Vec<Frame>, RouterError> {
5398        let modules = self
5399            .forwarding
5400            .route_census(module_id.as_deref())
5401            .map_err(RouterError::Forwarding)?
5402            .into_iter()
5403            .map(|(module_id, routes)| SupervisorRouteModule {
5404                module_id,
5405                routes: routes
5406                    .into_iter()
5407                    .map(|route| SupervisorRoute {
5408                        consumer: match route.principal {
5409                            Principal::Reserved { module_id } => {
5410                                SupervisorRouteConsumer::Reserved { module_id }
5411                            }
5412                            Principal::Direct | Principal::Unverified => {
5413                                SupervisorRouteConsumer::Direct {
5414                                    connection_id: route.goodbye_target.connection_id.get(),
5415                                }
5416                            }
5417                        },
5418                        age_ms: Instant::now()
5419                            .saturating_duration_since(route.bound_at)
5420                            .as_millis()
5421                            .try_into()
5422                            .unwrap_or(u64::MAX),
5423                        draining: route.draining,
5424                        drain_reason: route.drain_reason,
5425                    })
5426                    .collect(),
5427            })
5428            .collect();
5429        let response = ClientControlResponse::SupervisorRoutes { modules };
5430        Ok(vec![control_response_body_frame(
5431            &frame,
5432            &response,
5433            "ClientControlResponse::SupervisorRoutes",
5434        )?])
5435    }
5436
5437    async fn handle_supervisor_provenance(
5438        &self,
5439        frame: Frame,
5440        module_id: Option<String>,
5441    ) -> Result<Vec<Frame>, RouterError> {
5442        let mut selected = if let Some(module_id) = module_id {
5443            let Some(module) = self.supervisor.get(&module_id) else {
5444                return Ok(vec![control_error_frame(
5445                    &frame,
5446                    "unknown_module",
5447                    format!("module_id '{module_id}' is not supervised"),
5448                )?]);
5449            };
5450            vec![module]
5451        } else {
5452            self.supervisor.list()
5453        };
5454
5455        let mut modules = Vec::with_capacity(selected.len());
5456        for module in selected.drain(..) {
5457            let (status, observed_image) = module
5458                .status_and_running_image_agreement()
5459                .await
5460                .map_err(|err| {
5461                    RouterError::backend(
5462                        0,
5463                        frame.header.corr,
5464                        format!("failed to read supervisor status: {err}"),
5465                    )
5466                })?;
5467            let module_declared = self
5468                .registry
5469                .get_module(&status.module_id)
5470                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5471                .and_then(|registration| registration.manifest.provenance)
5472                .map(|build| ModuleDeclaredProvenance::Reported { build })
5473                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5474            #[cfg(test)]
5475            let running_image = match &self.provenance_probe_override {
5476                Some(result) => result.clone(),
5477                None => observed_image,
5478            };
5479            #[cfg(not(test))]
5480            let running_image = observed_image;
5481            modules.push(SupervisorModuleProvenance {
5482                module_id: status.module_id,
5483                module_declared,
5484                daemon_observed: SupervisorObservedProcess {
5485                    launch_nonce_source: status.launch_nonce_source,
5486                    pid: status.pid,
5487                    spawned_at_ms: status.spawned_at_ms,
5488                    spawned_from: status.spawned_from,
5489                    running_image,
5490                },
5491            });
5492        }
5493        let daemon = SupervisorDaemonProvenance {
5494            daemon_build: self.daemon_provenance.build.clone(),
5495            daemon_observed: DaemonObservedProcess {
5496                pid: self.daemon_provenance.pid,
5497                started_at_ms: self
5498                    .daemon_provenance
5499                    .start_clock
5500                    .map(|clock| clock.started_at_ms())
5501                    .or(self.daemon_provenance.started_at_ms),
5502                running_image: self
5503                    .daemon_provenance
5504                    .probe
5505                    .observe(
5506                        self.daemon_provenance.pid,
5507                        self.daemon_provenance.executable_path.as_deref(),
5508                        self.daemon_provenance.executable_identity.clone(),
5509                        self.daemon_provenance.process_start_time,
5510                    )
5511                    .await,
5512            },
5513        };
5514        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5515        Ok(vec![control_response_body_frame(
5516            &frame,
5517            &response,
5518            "ClientControlResponse::SupervisorProvenance",
5519        )?])
5520    }
5521
5522    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5523        self.refresh_capability_requirements();
5524        let generation = self
5525            .registry
5526            .generation()
5527            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5528        let modules = self
5529            .supervisor
5530            .list()
5531            .into_iter()
5532            .map(|module| {
5533                let status = module.status_for_control("health").map_err(|err| {
5534                    RouterError::backend(
5535                        0,
5536                        frame.header.corr,
5537                        format!("failed to read supervisor health: {err}"),
5538                    )
5539                })?;
5540                let module_id = status.module_id;
5541                let capability_detail = self
5542                    .capability_evaluator
5543                    .required_problem_detail(&module_id);
5544                Ok(SupervisorHealthEntry {
5545                    module_id,
5546                    status: status.health.status,
5547                    detail: append_capability_problem_detail(
5548                        status.health.detail,
5549                        capability_detail,
5550                    ),
5551                    metrics: status.health.metrics,
5552                    consecutive_failures: status.health.consecutive_failures,
5553                    late_answer_count: status.health.late_answer_count,
5554                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5555                    last_action: status.health.last_action,
5556                    last_action_ms: status.health.last_action_ms,
5557                    last_probe_ms: status.health.last_probe_ms,
5558                })
5559            })
5560            .collect::<Result<Vec<_>, RouterError>>()?;
5561        let response = ClientControlResponse::SupervisorHealth {
5562            generation,
5563            modules,
5564        };
5565        Ok(vec![control_response_body_frame(
5566            &frame,
5567            &response,
5568            "ClientControlResponse::SupervisorHealth",
5569        )?])
5570    }
5571
5572    async fn handle_supervisor_restart(
5573        &self,
5574        frame: Frame,
5575        module_id: String,
5576        drain_timeout_ms: Option<u64>,
5577        requester: Principal,
5578    ) -> Result<Vec<Frame>, RouterError> {
5579        let operation_lock = self.supervisor.operation_lock();
5580        let _operation_guard = operation_lock.lock().await;
5581        let Some(module) = self.supervisor.get(&module_id) else {
5582            return Ok(vec![control_error_frame(
5583                &frame,
5584                "unknown_module",
5585                format!("module_id '{module_id}' is not supervised"),
5586            )?]);
5587        };
5588
5589        self.route_outages.mark_operator_action(&module_id);
5590        if let Err(err) = module
5591            .restart_requested(drain_timeout_ms, Some(requester))
5592            .await
5593        {
5594            self.route_outages
5595                .operator_action_ended_unrefused(&module_id);
5596            let (code, message) = match err {
5597                crate::supervise::SuperviseError::Disabled { .. } => {
5598                    ("module_disabled", err.to_string())
5599                }
5600                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5601                    ("swap_in_progress", err.to_string())
5602                }
5603                _ => (
5604                    "target_unavailable",
5605                    format!("failed to restart module_id '{module_id}': {err}"),
5606                ),
5607            };
5608            return Ok(vec![control_error_frame(&frame, code, message)?]);
5609        }
5610
5611        let response = ClientControlResponse::SupervisorAck {
5612            module_id,
5613            applied: true,
5614        };
5615        Ok(vec![control_response_body_frame(
5616            &frame,
5617            &response,
5618            "ClientControlResponse::SupervisorAck",
5619        )?])
5620    }
5621
5622    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
5623    /// when the old process has finished draining: a caller whose own lane
5624    /// rides the old process must get its reply before that drain waits on it.
5625    async fn handle_supervisor_swap(
5626        &self,
5627        frame: Frame,
5628        module_id: String,
5629        ready_timeout_ms: Option<u64>,
5630        requester: Principal,
5631    ) -> Result<Vec<Frame>, RouterError> {
5632        // The daemon-wide operation lock is held only to resolve the handle,
5633        // not across the swap. The swap can take its whole readiness budget,
5634        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
5635        // holding it here would park an operator's stop behind the swap it is
5636        // meant to abort. A rescan or stop that reaches the module during the
5637        // swap is served by the swap itself (see `supervise_swap`).
5638        let module = {
5639            let operation_lock = self.supervisor.operation_lock();
5640            let _operation_guard = operation_lock.lock().await;
5641            self.supervisor.get(&module_id)
5642        };
5643        let Some(module) = module else {
5644            return Ok(vec![control_error_frame(
5645                &frame,
5646                "unknown_module",
5647                format!("module_id '{module_id}' is not supervised"),
5648            )?]);
5649        };
5650
5651        self.route_outages.mark_operator_action(&module_id);
5652        if let Err(err) = module
5653            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5654            .await
5655        {
5656            self.route_outages
5657                .operator_action_ended_unrefused(&module_id);
5658            use crate::supervise::SuperviseError;
5659            let message = err.to_string();
5660            let error = match err {
5661                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5662                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5663                    code: "swap_refused".to_string(),
5664                    message,
5665                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5666                },
5667                SuperviseError::SwapFailed {
5668                    arm,
5669                    candidate_exit,
5670                    ..
5671                } => ErrorBody {
5672                    code: "swap_failed".to_string(),
5673                    message,
5674                    detail: Some(serde_json::json!({
5675                        "arm": arm.as_str(),
5676                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5677                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5678                    })),
5679                },
5680                _ => ErrorBody::new(
5681                    "target_unavailable",
5682                    format!("failed to swap module_id '{module_id}': {message}"),
5683                ),
5684            };
5685            return Ok(vec![control_error_body_frame(&frame, error)?]);
5686        }
5687        // A completed swap kept the incumbent serving until cutover, so it
5688        // usually opened no outage; a mark left behind would make the next,
5689        // unrelated outage read as requested.
5690        self.route_outages
5691            .operator_action_ended_unrefused(&module_id);
5692
5693        let response = ClientControlResponse::SupervisorAck {
5694            module_id,
5695            applied: true,
5696        };
5697        Ok(vec![control_response_body_frame(
5698            &frame,
5699            &response,
5700            "ClientControlResponse::SupervisorAck",
5701        )?])
5702    }
5703
5704    async fn handle_supervisor_reload(
5705        &self,
5706        frame: Frame,
5707        module_id: String,
5708        requester: Principal,
5709    ) -> Result<Vec<Frame>, RouterError> {
5710        let operation_lock = self.supervisor.operation_lock();
5711        let _operation_guard = operation_lock.lock().await;
5712        let Some(module) = self.supervisor.get(&module_id) else {
5713            return Ok(vec![control_error_frame(
5714                &frame,
5715                "unknown_module",
5716                format!("module_id '{module_id}' is not supervised"),
5717            )?]);
5718        };
5719
5720        self.route_outages.mark_operator_action(&module_id);
5721        if let Err(err) = module.reload_requested(Some(requester)).await {
5722            self.route_outages
5723                .operator_action_ended_unrefused(&module_id);
5724            let (code, message) = match err {
5725                crate::supervise::SuperviseError::Disabled { .. } => {
5726                    ("module_disabled", err.to_string())
5727                }
5728                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5729                    ("swap_in_progress", err.to_string())
5730                }
5731                _ => (
5732                    "reload_failed",
5733                    format!("failed to reload module_id '{module_id}': {err}"),
5734                ),
5735            };
5736            return Ok(vec![control_error_frame(&frame, code, message)?]);
5737        }
5738
5739        let response = ClientControlResponse::SupervisorAck {
5740            module_id,
5741            applied: true,
5742        };
5743        Ok(vec![control_response_body_frame(
5744            &frame,
5745            &response,
5746            "ClientControlResponse::SupervisorAck",
5747        )?])
5748    }
5749
5750    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
5751    /// the configuration comes from and whether to apply it; every mode then
5752    /// runs the same comparison, so a preview describes the apply it previews.
5753    async fn handle_supervisor_rescan(
5754        &self,
5755        frame: Frame,
5756        mode: RescanMode,
5757        requester: Principal,
5758    ) -> Result<Vec<Frame>, RouterError> {
5759        let preview = mode.previews();
5760        let Some(context) = self.rescan.clone() else {
5761            return Ok(vec![control_error_frame(
5762                &frame,
5763                "rescan_unavailable",
5764                "the daemon was not started with a reloadable config path".to_string(),
5765            )?]);
5766        };
5767
5768        let operation_lock = self.supervisor.operation_lock();
5769        let _operation_guard = operation_lock.lock().await;
5770        // Loaded under the operation lock, so no other rescan can change the
5771        // staged configuration between a checked apply's comparison and the
5772        // apply itself.
5773        let loaded = match self.load_rescan_config(&mode, &context) {
5774            Ok(config) => config,
5775            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5776        };
5777        // `load` reports a missing file as Ok(None), which is correct at boot
5778        // (no config, nothing to supervise) and catastrophic here: rescan treats
5779        // "not in the config" as "remove it", so an absent file would read as an
5780        // empty module list and retire the entire running fleet. An editor
5781        // writing via write-new-then-rename, or a half-finished edit, is enough
5782        // to open that window. Refuse instead: a config that cannot be read
5783        // carries no instruction to remove anything.
5784        let Some((config, activity_config)) = loaded else {
5785            return Ok(vec![control_error_frame(
5786                &frame,
5787                "invalid_daemon_config",
5788                format!(
5789                    "daemon config not found at {}; refusing to rescan (an absent config would \
5790                     retire every supervised module)",
5791                    context.config_path.display()
5792                ),
5793            )?]);
5794        };
5795        let (
5796            configured_port,
5797            storage_config,
5798            admission_facts_carrier_module_id,
5799            admission_facts_targets,
5800            scope_authority_owners,
5801            privileged_operator_prompt,
5802            modules,
5803            reserved_capabilities,
5804        ) = (
5805            config.port,
5806            config.storage,
5807            config.admission_facts_carrier_module_id,
5808            config.admission_facts_targets,
5809            config.scope_authority_owners,
5810            config.privileged_operator_prompt,
5811            config.modules,
5812            config.reserved_capabilities,
5813        );
5814
5815        // Collect the sections rescan cannot apply, so the REPLY carries them.
5816        //
5817        // The warning below has always been correct and has always gone only to
5818        // the journal -- addressed to whoever reads logs, while the person who
5819        // just edited the config is looking at the CLI. Naming each section
5820        // individually rather than setting a flag: "something outside modules
5821        // changed" sends the operator back to diffing their own file, which is
5822        // the work this is meant to save.
5823        let mut restart_required = Vec::new();
5824        for section in RestartRequiredSection::ALL {
5825            let changed = match section {
5826                RestartRequiredSection::Port => configured_port != context.configured_port,
5827                RestartRequiredSection::Storage => storage_config != context.storage_config,
5828                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5829                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5830                }
5831                RestartRequiredSection::AdmissionFactsTargets => {
5832                    admission_facts_targets != context.admission_facts_targets
5833                }
5834                RestartRequiredSection::ScopeAuthorityOwners => {
5835                    scope_authority_owners != context.scope_authority_owners
5836                }
5837                RestartRequiredSection::PrivilegedOperatorPrompt => {
5838                    privileged_operator_prompt != context.privileged_operator_prompt
5839                }
5840            };
5841            if changed {
5842                restart_required.push(section.label().to_string());
5843            }
5844        }
5845        if !restart_required.is_empty() {
5846            warn!(
5847                config_path = %context.config_path.display(),
5848                sections = %restart_required.join(", "),
5849                "daemon config changed outside the modules section; restart the daemon to apply those changes"
5850            );
5851        }
5852
5853        for configured in &modules {
5854            if let Err(err) = validate_spec(&configured.module_spec()) {
5855                return Ok(vec![control_error_frame(
5856                    &frame,
5857                    "invalid_daemon_config",
5858                    format!("supervisor rescan rejected daemon config: {err}"),
5859                )?]);
5860            }
5861        }
5862
5863        let configured_capabilities = modules
5864            .iter()
5865            .map(|module| (module.module_id.clone(), module.enabled))
5866            .collect::<Vec<_>>();
5867        let preview_capability_warnings = if preview {
5868            let (_, registrations) = self.runtime_capability_snapshot()?;
5869            let current_modules = self
5870                .supervisor
5871                .list()
5872                .into_iter()
5873                .map(|module| module.module_id().to_string())
5874                .collect::<BTreeSet<_>>();
5875            let resulting_modules = configured_capabilities.clone();
5876            let removed = current_modules
5877                .into_iter()
5878                .filter(|module_id| {
5879                    !resulting_modules
5880                        .iter()
5881                        .any(|(configured_id, _)| configured_id == module_id)
5882                })
5883                .collect::<Vec<_>>();
5884            // A candidate preview also names providers the candidate disables.
5885            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
5886            let disabled = match &mode {
5887                RescanMode::Candidate(_) => resulting_modules
5888                    .iter()
5889                    .filter(|(_, enabled)| !enabled)
5890                    .map(|(module_id, _)| module_id.clone())
5891                    .collect::<Vec<_>>(),
5892                _ => Vec::new(),
5893            };
5894            self.capability_evaluator.preview_warnings(
5895                resulting_modules,
5896                &removed,
5897                &disabled,
5898                &registrations,
5899            )
5900        } else {
5901            Vec::new()
5902        };
5903        let reconciled = match &mode {
5904            RescanMode::Candidate(candidate) => self
5905                .preview_candidate(&context, modules, candidate)
5906                .map(|result| (result, Vec::new())),
5907            _ => {
5908                self.reconcile_supervised_modules(
5909                    &context.supervisor,
5910                    modules,
5911                    preview,
5912                    requester.clone(),
5913                )
5914                .await
5915            }
5916        };
5917        let (result, changed) = match reconciled {
5918            Ok(result) => result,
5919            Err(message) => {
5920                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5921            }
5922        };
5923        if !preview {
5924            if let Some(ledger) = &self.activity {
5925                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
5926                ledger.config_changed(&result.added, &result.removed, &changed, requester);
5927            }
5928            if let Err(code) = self.configure_activity(activity_config).await {
5929                warn!(
5930                    code,
5931                    "activity retention could not be applied during rescan"
5932                );
5933            }
5934            self.capability_evaluator
5935                .configure(configured_capabilities, reserved_capabilities);
5936            self.capability_evaluator.wake_deadline_loop();
5937            self.refresh_capability_requirements();
5938        }
5939        let mut result = result;
5940        result.restart_required = restart_required;
5941        result.capability_warnings = preview_capability_warnings;
5942        let response = ClientControlResponse::SupervisorRescan { result };
5943        Ok(vec![control_response_body_frame(
5944            &frame,
5945            &response,
5946            "ClientControlResponse::SupervisorRescan",
5947        )?])
5948    }
5949
5950    async fn handle_supervisor_release_reserved(
5951        &self,
5952        frame: Frame,
5953        module_id: String,
5954    ) -> Result<Vec<Frame>, RouterError> {
5955        let Some(context) = self.rescan.clone() else {
5956            return Ok(vec![control_error_frame(
5957                &frame,
5958                "release_unavailable",
5959                "reserved-id release requires a daemon started with a reloadable config path",
5960            )?]);
5961        };
5962        let operation_lock = self.supervisor.operation_lock();
5963        let _operation_guard = operation_lock.lock().await;
5964        let loaded = match crate::daemon_config::load(&context.config_path) {
5965            Ok(Some(config)) => config,
5966            Ok(None) => {
5967                return Ok(vec![control_error_frame(
5968                    &frame,
5969                    "invalid_daemon_config",
5970                    format!(
5971                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5972                        context.config_path.display()
5973                    ),
5974                )?])
5975            }
5976            Err(err) => {
5977                return Ok(vec![control_error_frame(
5978                    &frame,
5979                    "invalid_daemon_config",
5980                    format!("unable to verify reserved-id release against daemon config: {err}"),
5981                )?])
5982            }
5983        };
5984        if loaded
5985            .modules
5986            .iter()
5987            .any(|configured| configured.module_id == module_id)
5988        {
5989            return Ok(vec![control_error_frame(
5990                &frame,
5991                "reserved_module_configured",
5992                format!(
5993                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5994                ),
5995            )?]);
5996        }
5997        if !self.supervisor.release_retained_reserved_gate(&module_id) {
5998            return Ok(vec![control_error_frame(
5999                &frame,
6000                "reserved_gate_not_retained",
6001                format!(
6002                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6003                ),
6004            )?]);
6005        }
6006
6007        let response = ClientControlResponse::SupervisorAck {
6008            module_id,
6009            applied: true,
6010        };
6011        Ok(vec![control_response_body_frame(
6012            &frame,
6013            &response,
6014            "ClientControlResponse::SupervisorAck",
6015        )?])
6016    }
6017
6018    /// Compare the configured module set with the supervised one, changing
6019    /// nothing. The staged spec each supervised module holds is what the
6020    /// configured entry is compared with.
6021    fn plan_rescan(
6022        &self,
6023        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6024    ) -> Result<RescanPlan, String> {
6025        let mut current = BTreeMap::new();
6026        for module in self.supervisor.list() {
6027            let (spec, health) = module.configuration().map_err(|err| {
6028                format!(
6029                    "failed to read configuration for module_id '{}': {err}",
6030                    module.module_id()
6031                )
6032            })?;
6033            let enabled = module
6034                .status()
6035                .map_err(|err| {
6036                    format!(
6037                        "failed to read status for module_id '{}': {err}",
6038                        module.module_id()
6039                    )
6040                })?
6041                .enabled;
6042            current.insert(
6043                module.module_id().to_string(),
6044                (module, spec, health, enabled),
6045            );
6046        }
6047        let configured = configured_modules
6048            .into_iter()
6049            .map(|module| (module.module_id.clone(), module))
6050            .collect::<BTreeMap<_, _>>();
6051
6052        let added = configured
6053            .keys()
6054            .filter(|module_id| !current.contains_key(*module_id))
6055            .cloned()
6056            .collect::<Vec<_>>();
6057        let removed = current
6058            .keys()
6059            .filter(|module_id| !configured.contains_key(*module_id))
6060            .cloned()
6061            .collect::<Vec<_>>();
6062        let mut changed_pending_reload = Vec::new();
6063        let mut configuration_changes = BTreeSet::new();
6064        let mut enabled_changes = BTreeSet::new();
6065        let mut unchanged = 0_u32;
6066
6067        for (module_id, configured_module) in &configured {
6068            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6069            else {
6070                continue;
6071            };
6072            // Compare the whole launch spec so a future launch field cannot
6073            // accidentally become a live-only policy change. Health is stored
6074            // separately and applies live without replacing the process.
6075            let launch_changed = *current_spec != configured_module.module_spec();
6076            let configuration_changed =
6077                launch_changed || *current_health != configured_module.health;
6078            let enabled_changed = *current_enabled != configured_module.enabled;
6079            if configuration_changed {
6080                configuration_changes.insert(module_id.clone());
6081            }
6082            if launch_changed {
6083                changed_pending_reload.push(module_id.clone());
6084            }
6085            if enabled_changed {
6086                enabled_changes.insert(module_id.clone());
6087            }
6088            if !configuration_changed && !enabled_changed {
6089                unchanged = unchanged.saturating_add(1);
6090            }
6091        }
6092        Ok(RescanPlan {
6093            current,
6094            configured,
6095            added,
6096            removed,
6097            changed_pending_reload,
6098            configuration_changes,
6099            enabled_changes,
6100            unchanged,
6101        })
6102    }
6103
6104    /// Reconcile the running module set against the configured one.
6105    ///
6106    /// With `preview` set, the diff is computed and returned WITHOUT applying any
6107    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
6108    /// deliberately shares this function with the executing path rather than
6109    /// computing the same diff somewhere else -- two implementations of one
6110    /// decision agree until they do not, and the whole value of a preview is that
6111    /// it describes the operation that will actually run.
6112    async fn reconcile_supervised_modules(
6113        &self,
6114        supervisor: &Supervisor,
6115        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6116        preview: bool,
6117        requester: Principal,
6118    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6119        let plan = self.plan_rescan(configured_modules)?;
6120
6121        // Everything above this point is pure computation over two snapshots.
6122        // Everything below MUTATES. The preview returns here so the boundary is a
6123        // single early return rather than a condition repeated at each mutation
6124        // site, where one missed guard would apply part of a change the caller was
6125        // told would not happen.
6126        if preview {
6127            return Ok((plan.preview_result(), Vec::new()));
6128        }
6129        let changed = plan
6130            .configuration_changes
6131            .union(&plan.enabled_changes)
6132            .cloned()
6133            .collect::<Vec<_>>();
6134        let RescanPlan {
6135            current,
6136            configured,
6137            added,
6138            removed,
6139            changed_pending_reload,
6140            configuration_changes,
6141            enabled_changes,
6142            unchanged,
6143        } = plan;
6144        // The staged generation moves before the first change rather than
6145        // after the last, so a rescan that fails partway still invalidates
6146        // every earlier `supervisor.rescan_checked` preview: part of its diff
6147        // may have run.
6148        if !(added.is_empty()
6149            && removed.is_empty()
6150            && configuration_changes.is_empty()
6151            && enabled_changes.is_empty())
6152        {
6153            if let Some(context) = &self.rescan {
6154                context
6155                    .generation
6156                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6157            }
6158        }
6159
6160        for module_id in &removed {
6161            let module = &current
6162                .get(module_id)
6163                .expect("removed module came from current supervisor state")
6164                .0;
6165            module
6166                .retire_requested(Some(requester.clone()))
6167                .await
6168                .map_err(|err| {
6169                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
6170                })?;
6171            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
6172            //
6173            // `handle_route_open` resolves an absent module in three steps:
6174            // registry, then supervisor status, then tombstone. Retiring first
6175            // opens a window where ALL THREE ARE ABSENT -- the registry entry
6176            // went with the teardown above, the supervisor entry went with
6177            // `retire`, and the tombstone does not exist yet -- so a route.open
6178            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
6179            // it") for a module that was deliberately removed and whose caller
6180            // should get `module_removed` (TERMINAL, carrying a removal age).
6181            //
6182            // Writing the tombstone first closes it: during the window the
6183            // supervisor entry still answers, so the caller gets
6184            // `target_unavailable` -- retryable, and TRUE, because the module
6185            // is mid-teardown. After both statements it is `module_removed`.
6186            // No instant remains where a removed module reads as one that
6187            // never existed.
6188            //
6189            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6190            // the absence of a test beside a fix invites deletion: these are
6191            // two sync statements with no await between them, so reaching the
6192            // window needs a second worker thread to land exactly between them
6193            // and there is no hook to force it. MEASURED: the 25 daemon_config
6194            // tests pass identically with the old order and the new one, so
6195            // the existing suite cannot see this and a green run is not
6196            // evidence either way. What the suite does hold is the
6197            // post-condition -- a removed module answers `module_removed` --
6198            // which this preserves.
6199            //
6200            // Found by an Athena panel reading the shipped tree against a
6201            // design note (2026-09-19), as the one concrete instance of that
6202            // note's class that survived contact with source. Direction is
6203            // benign: retryable where terminal was intended, never the reverse.
6204            self.supervisor.record_rescan_removal(module_id);
6205            self.supervisor.retire(module_id);
6206            self.route_outages.forget(module_id);
6207        }
6208
6209        for module_id in configured.keys() {
6210            let Some((module, _, _, _)) = current.get(module_id) else {
6211                continue;
6212            };
6213            let configured_module = configured
6214                .get(module_id)
6215                .expect("configured module id came from configured map");
6216            if configuration_changes.contains(module_id) {
6217                module
6218                    .update_configuration(
6219                        configured_module.module_spec(),
6220                        configured_module.health.clone(),
6221                        configured_module.drain_timeout_ms,
6222                    )
6223                    .await
6224                    .map_err(|err| {
6225                        format!(
6226                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6227                        )
6228                    })?;
6229            }
6230            if enabled_changes.contains(module_id) {
6231                // A rescan that starts or stops a module applies an operator's
6232                // edit to the config, so the resulting outage was asked for.
6233                self.route_outages.mark_operator_action(module_id);
6234                module
6235                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6236                    .await
6237                    .map_err(|err| {
6238                        self.route_outages.operator_action_ended_unrefused(module_id);
6239                        format!(
6240                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6241                            configured_module.enabled
6242                        )
6243                    })?;
6244            }
6245        }
6246
6247        for module_id in &added {
6248            let configured_module = configured
6249                .get(module_id)
6250                .expect("added module id came from configured map");
6251            supervisor
6252                .supervise_configured_requested(
6253                    configured_module.module_spec(),
6254                    configured_module.enabled,
6255                    configured_module.health.clone(),
6256                    configured_module.drain_timeout_ms,
6257                    configured_module.restart,
6258                    Some(requester.clone()),
6259                )
6260                .map_err(|err| {
6261                    format!("failed to add module_id '{module_id}' during rescan: {err}")
6262                })?;
6263        }
6264
6265        Ok((
6266            SupervisorRescanResult {
6267                added,
6268                removed,
6269                changed_pending_reload,
6270                enabled_changes: enabled_changes.iter().cloned().collect(),
6271                unchanged,
6272                preview: false,
6273                // Filled by the caller, which is the only layer that can see the
6274                // previous config to diff against.
6275                restart_required: Vec::new(),
6276                capability_warnings: Vec::new(),
6277                module_actions: Vec::new(),
6278                earlier_pending: Vec::new(),
6279                expect: None,
6280            },
6281            changed,
6282        ))
6283    }
6284
6285    async fn handle_supervisor_set_enabled(
6286        &self,
6287        frame: Frame,
6288        module_id: String,
6289        enabled: bool,
6290        requester: Principal,
6291    ) -> Result<Vec<Frame>, RouterError> {
6292        let operation_lock = self.supervisor.operation_lock();
6293        let _operation_guard = operation_lock.lock().await;
6294        let Some(module) = self.supervisor.get(&module_id) else {
6295            return Ok(vec![control_error_frame(
6296                &frame,
6297                "unknown_module",
6298                format!("module_id '{module_id}' is not supervised"),
6299            )?]);
6300        };
6301
6302        // Enabling counts as well as disabling: a module an operator starts
6303        // is refused until it registers, and that wait was asked for.
6304        self.route_outages.mark_operator_action(&module_id);
6305        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6306            Ok(applied) => applied,
6307            Err(err) => {
6308                self.route_outages
6309                    .operator_action_ended_unrefused(&module_id);
6310                return Ok(vec![control_error_frame(
6311                    &frame,
6312                    "target_unavailable",
6313                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6314                )?]);
6315            }
6316        };
6317        if !applied {
6318            // Already in the requested state: nothing was made unavailable,
6319            // so the mark must not outlive this request.
6320            self.route_outages
6321                .operator_action_ended_unrefused(&module_id);
6322        }
6323
6324        self.capability_evaluator.wake_deadline_loop();
6325        self.refresh_capability_requirements();
6326        let response = ClientControlResponse::SupervisorAck { module_id, applied };
6327        Ok(vec![control_response_body_frame(
6328            &frame,
6329            &response,
6330            "ClientControlResponse::SupervisorAck",
6331        )?])
6332    }
6333
6334    async fn handle_supervisor_health_probe(
6335        &self,
6336        frame: Frame,
6337        module_id: String,
6338    ) -> Result<Vec<Frame>, RouterError> {
6339        self.refresh_capability_requirements();
6340        let Some(registration) = self
6341            .registry
6342            .get_module(&module_id)
6343            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6344        else {
6345            return Ok(vec![control_error_frame(
6346                &frame,
6347                "unknown_module",
6348                format!("module_id '{module_id}' is not registered"),
6349            )?]);
6350        };
6351
6352        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
6353        // named for it. Making `module_registration_grants_op` return false
6354        // unconditionally reddens five tests, and every one is named for something
6355        // else -- capability relay, probe/bind demultiplexing, supervision-only
6356        // probing. They exercise a successful advertisement check on the way to their
6357        // own subject.
6358        //
6359        // Real protection, fragile in a specific way: narrowing any of those tests to
6360        // focus on its stated subject would silently remove coverage nobody knows
6361        // they are carrying. Recorded here rather than as a sixth test, because the
6362        // useful fact is WHICH tests hold the guard up -- a new test would add
6363        // coverage without telling the next person what the existing ones quietly do.
6364        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6365        {
6366            return Ok(vec![control_error_frame(
6367                &frame,
6368                "health_not_advertised",
6369                format!("module_id '{module_id}' did not advertise health.check"),
6370            )?]);
6371        }
6372
6373        let deadline = Instant::now() + self.health_probe_timeout;
6374        let pending = match self.forwarding.begin_module_control_rpc_for(
6375            &module_id,
6376            MODULE_CONTROL_OP_HEALTH_CHECK,
6377            deadline,
6378        ) {
6379            Ok(pending) => pending,
6380            Err(err) => {
6381                return Ok(vec![control_error_frame(
6382                    &frame,
6383                    forwarding_error_code(&err),
6384                    err.to_string(),
6385                )?])
6386            }
6387        };
6388
6389        let PendingModuleControlRpc {
6390            endpoint,
6391            module_sink,
6392            negotiated_ver,
6393            corr: probe_corr,
6394            receiver,
6395        } = pending;
6396        let mut guard =
6397            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6398        let probe_body =
6399            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6400                RouterError::backend(
6401                    0,
6402                    frame.header.corr,
6403                    format!("failed to encode health.check request: {err}"),
6404                )
6405            })?;
6406        let probe_frame = Frame::build_with_version(
6407            negotiated_ver,
6408            FrameType::Request,
6409            control_flags(),
6410            0,
6411            0,
6412            probe_corr,
6413            probe_body,
6414        )
6415        .map_err(RouterError::FrameBuild)?;
6416
6417        // Queue admission and the reply share one deadline, so a module that
6418        // stops reading cannot park the requesting connection's reader forever.
6419        match timeout_at(deadline, module_sink.send(probe_frame)).await {
6420            Ok(Ok(())) => {}
6421            Ok(Err(err)) => {
6422                return Ok(vec![control_error_frame(
6423                    &frame,
6424                    "target_unavailable",
6425                    err.to_string(),
6426                )?]);
6427            }
6428            Err(_) => {
6429                return Ok(vec![control_error_frame(
6430                    &frame,
6431                    "module_timeout",
6432                    format!(
6433                        "module_id '{module_id}' did not answer health.check within {:?}",
6434                        self.health_probe_timeout
6435                    ),
6436                )?]);
6437            }
6438        }
6439
6440        match timeout_at(deadline, receiver).await {
6441            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6442                guard.disarm();
6443                let Some(report) = response.health_report() else {
6444                    return Ok(vec![control_error_frame(
6445                        &frame,
6446                        "invalid_control_body",
6447                        "health.check RPC returned a non-health response",
6448                    )?]);
6449                };
6450                // Metrics go out whole here. The supervisor's cached snapshot
6451                // caps this blob (see truncate_health_metrics), and this path
6452                // exists precisely to answer without that cap -- so applying it
6453                // here would leave no way to see what the cached view drops.
6454                let HealthReport {
6455                    status,
6456                    detail,
6457                    metrics,
6458                } = report;
6459                let capability_detail = self
6460                    .capability_evaluator
6461                    .required_problem_detail(&module_id);
6462                let response = ClientControlResponse::SupervisorHealthProbe {
6463                    module_id,
6464                    status,
6465                    detail: append_capability_problem_detail(detail, capability_detail),
6466                    metrics,
6467                };
6468                Ok(vec![control_response_body_frame(
6469                    &frame,
6470                    &response,
6471                    "ClientControlResponse::SupervisorHealthProbe",
6472                )?])
6473            }
6474            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6475                guard.disarm();
6476                Ok(vec![control_error_body_frame(&frame, body)?])
6477            }
6478            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6479                guard.disarm();
6480                Ok(vec![control_error_frame(
6481                    &frame,
6482                    "target_unavailable",
6483                    message,
6484                )?])
6485            }
6486            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6487                guard.disarm();
6488                Ok(vec![control_error_frame(
6489                    &frame,
6490                    "invalid_control_body",
6491                    message,
6492                )?])
6493            }
6494            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6495                guard.disarm();
6496                Ok(vec![control_error_frame(
6497                    &frame,
6498                    "invalid_control_body",
6499                    format!("expected module-control op '{expected}', got '{actual}'"),
6500                )?])
6501            }
6502            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6503                guard.disarm();
6504                Ok(vec![control_error_frame(
6505                    &frame,
6506                    "module_timeout",
6507                    format!(
6508                        "module_id '{module_id}' answered health.check after {:?}",
6509                        self.health_probe_timeout
6510                    ),
6511                )?])
6512            }
6513            Ok(Err(_)) => Ok(vec![control_error_frame(
6514                &frame,
6515                "target_unavailable",
6516                "health.check waiter was canceled before the module responded",
6517            )?]),
6518            Err(_) => Ok(vec![control_error_frame(
6519                &frame,
6520                "module_timeout",
6521                format!(
6522                    "module_id '{module_id}' did not answer health.check within {:?}",
6523                    self.health_probe_timeout
6524                ),
6525            )?]),
6526        }
6527    }
6528
6529    fn supervisor_status(
6530        &self,
6531        module_id: &str,
6532        corr: u64,
6533    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6534        self.supervisor
6535            .get(module_id)
6536            .map(|module| {
6537                let warming = module.is_warming_for_control("status").map_err(|err| {
6538                    RouterError::backend(
6539                        0,
6540                        corr,
6541                        format!(
6542                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
6543                        ),
6544                    )
6545                })?;
6546                module.status_for_control("status").map_err(|err| {
6547                    RouterError::backend(
6548                        0,
6549                        corr,
6550                        format!(
6551                            "failed to read supervisor status for module_id '{module_id}': {err}"
6552                        ),
6553                    )
6554                }).map(|status| (status, warming))
6555            })
6556            .transpose()
6557    }
6558
6559    fn guard_module_control_op(
6560        &self,
6561        frame: &Frame,
6562        module_id: &str,
6563        op: &str,
6564    ) -> Result<Option<Frame>, RouterError> {
6565        if self.module_grants_op(module_id, op, frame.header.corr)? {
6566            return Ok(None);
6567        }
6568
6569        Ok(Some(control_error_frame(
6570            frame,
6571            "op_not_allowed",
6572            format!("module_id '{module_id}' did not grant control op '{op}'"),
6573        )?))
6574    }
6575
6576    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6577        let Some(registration) = self
6578            .registry
6579            .get_module(module_id)
6580            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6581        else {
6582            return Ok(false);
6583        };
6584        Ok(module_registration_grants_op(&registration.control_ops, op))
6585    }
6586
6587    fn handle_status_update(
6588        &self,
6589        endpoint: ModuleEndpointId,
6590        frame: Frame,
6591    ) -> Result<Vec<Frame>, RouterError> {
6592        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6593            Ok(update) => update,
6594            Err(err) => {
6595                // Forward-compat: a newer module may push a channel-0 op this subc
6596                // version doesn't know. The control contract says unknown push ops
6597                // are IGNORED, never answered with an error. Only a malformed body
6598                // for an op we DO know is a real error worth surfacing.
6599                if is_known_module_push_op(&frame.body) {
6600                    return Ok(vec![control_error_frame(
6601                        &frame,
6602                        "invalid_control_body",
6603                        format!("malformed module control push body: {err}"),
6604                    )?]);
6605                }
6606                return Ok(Vec::new());
6607            }
6608        };
6609
6610        match update {
6611            ModuleControlPush::RouteStatus {
6612                route_channel,
6613                route_epoch,
6614                status,
6615            } => {
6616                self.forwarding
6617                    .cache_status(endpoint, route_channel, route_epoch, status)
6618                    .map_err(RouterError::Forwarding)?;
6619            }
6620        }
6621        Ok(Vec::new())
6622    }
6623
6624    fn handle_route_poll(
6625        &self,
6626        ctx: &RouteCtx,
6627        frame: Frame,
6628        route_channel: u16,
6629        route_epoch: u32,
6630        kind: PollKind,
6631    ) -> Result<Vec<Frame>, RouterError> {
6632        let snapshot = self
6633            .forwarding
6634            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6635            .map_err(RouterError::Forwarding)?;
6636        let response = match (kind, snapshot) {
6637            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6638                ClientControlResponse::RoutePoll {
6639                    route_channel,
6640                    route_epoch,
6641                    status,
6642                    live: None,
6643                }
6644            }
6645            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6646                route_channel,
6647                route_epoch,
6648                status: None,
6649                live: None,
6650            },
6651            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6652                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
6653                // is what makes reporting `true` correct rather than a
6654                // confident guess. `process_live` returns None only when the
6655                // module id has no supervisor snapshot at all -- an
6656                // externally-started module the daemon did not spawn -- and
6657                // for those the supervisor has no opinion to offer, ever. It
6658                // is never None for a supervised module in an unknown state:
6659                // a supervised module always has a snapshot, and the answer
6660                // comes from `state == Running && process_alive`.
6661                //
6662                // The route is Bound, so the module completed a HELLO on a
6663                // live connection; "the process this route points at is
6664                // running" is therefore attested by the binding rather than
6665                // assumed. Reporting `false` for an unsupervised module would
6666                // be the actual lie -- it would tell a client its healthy
6667                // route is dead because the daemon does not manage the
6668                // process.
6669                //
6670                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
6671                // module whose liveness is genuinely unknown, e.g. a snapshot
6672                // that has not been populated yet -- THIS DEFAULT BECOMES
6673                // WRONG and must split: unsupervised stays true, unknown
6674                // becomes null so the client can tell the two apart. The
6675                // response field is already `Option<bool>`, so the wire can
6676                // carry that distinction today.
6677                let live = self
6678                    .process_liveness
6679                    .as_ref()
6680                    .and_then(|source| source.process_live(&module_id))
6681                    .unwrap_or(true);
6682                ClientControlResponse::RoutePoll {
6683                    route_channel,
6684                    route_epoch,
6685                    status: None,
6686                    live: Some(live),
6687                }
6688            }
6689            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6690                route_channel,
6691                route_epoch,
6692                status: None,
6693                live: Some(false),
6694            },
6695        };
6696
6697        Ok(vec![control_response_body_frame(
6698            &frame,
6699            &response,
6700            "ClientControlResponse::RoutePoll",
6701        )?])
6702    }
6703
6704    pub(crate) fn observe_module_control_completion(
6705        &self,
6706        completion: ModuleControlRpcCompletion,
6707    ) -> bool {
6708        match completion {
6709            ModuleControlRpcCompletion::Unknown => false,
6710            ModuleControlRpcCompletion::Settled => true,
6711            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6712                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6713                info!(
6714                    module_id = %module_id,
6715                    latency_ms,
6716                    "late health.check answer proves the module is alive"
6717                );
6718                match self
6719                    .supervisor
6720                    .record_late_health_answer(&module_id, latency_ms)
6721                {
6722                    Ok(true) => {}
6723                    Ok(false) => debug!(
6724                        module_id = %module_id,
6725                        latency_ms,
6726                        "late health.check answer has no active supervisor snapshot"
6727                    ),
6728                    Err(err) => warn!(
6729                        module_id = %module_id,
6730                        latency_ms,
6731                        error = %err,
6732                        "failed to record late health.check answer"
6733                    ),
6734                }
6735                true
6736            }
6737        }
6738    }
6739
6740    /// Decide whether a failure while settling a relayed `route.bind` belongs to
6741    /// the module connection whose frame is being handled, or to the client that
6742    /// relay was opened for.
6743    ///
6744    /// This runs on the MODULE connection's frame handler, where returning `Err`
6745    /// ends that connection -- and a module connection carries every client's
6746    /// routes to that module, so ending it costs the whole fleet its tools.
6747    /// `ConnectionClosing` carries the id of the connection that is closing, and
6748    /// when that id is a CLIENT's, the condition is entirely about that one
6749    /// client's route.open. A client-scoped condition has no authority over a
6750    /// shared module connection, so it is logged and the single relay is dropped:
6751    /// the client is going away, and `complete_pending_relay` already removed the
6752    /// relay before failing, so there is nothing left to settle. Anything that
6753    /// relay still reserved is released by that client's own connection teardown,
6754    /// which is already under way -- that is what "closing" means.
6755    ///
6756    /// Every other failure is a statement about THIS connection and stays fatal:
6757    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
6758    /// id in `ConnectionClosing` all mean this connection cannot keep serving
6759    /// frames correctly.
6760    fn refuse_to_end_module_connection_for_a_client(
6761        &self,
6762        module_connection_id: ConnectionId,
6763        corr: u64,
6764        err: ForwardingError,
6765    ) -> Result<(), RouterError> {
6766        if let ForwardingError::ConnectionClosing { connection_id } = err {
6767            if connection_id != module_connection_id {
6768                warn!(
6769                    module_connection_id = module_connection_id.get(),
6770                    client_connection_id = connection_id.get(),
6771                    corr,
6772                    "dropping a route.bind response for a closing client; the module connection keeps serving"
6773                );
6774                return Ok(());
6775            }
6776        }
6777        Err(RouterError::Forwarding(err))
6778    }
6779
6780    fn handle_module_relay_response(
6781        &self,
6782        connection_id: ConnectionId,
6783        frame: Frame,
6784    ) -> Result<Vec<Frame>, RouterError> {
6785        let mut secondary_error = None;
6786        let outcome = match frame.header.ty {
6787            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6788                Ok(probe) if probe.op == "route.bind" => {
6789                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6790                        Ok(ModuleControlResponse::RouteBindAck {}) => {
6791                            RouteBindRelayOutcome::Accepted
6792                        }
6793                        Ok(other) => {
6794                            let message =
6795                                format!("route.bind response carried unexpected body: {other:?}");
6796                            secondary_error = Some(control_error_frame(
6797                                &frame,
6798                                "invalid_control_body",
6799                                message.clone(),
6800                            )?);
6801                            RouteBindRelayOutcome::ModuleGone(message)
6802                        }
6803                        Err(err) => {
6804                            let message = format!("malformed route.bind response body: {err}");
6805                            secondary_error = Some(control_error_frame(
6806                                &frame,
6807                                "invalid_control_body",
6808                                message.clone(),
6809                            )?);
6810                            RouteBindRelayOutcome::ModuleGone(message)
6811                        }
6812                    }
6813                }
6814                Ok(probe) => {
6815                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6816                    {
6817                        Ok(response) => ModuleControlRpcOutcome::Response(response),
6818                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6819                            "malformed {} response body: {err}",
6820                            probe.op
6821                        )),
6822                    };
6823                    let completion = self
6824                        .forwarding
6825                        .complete_module_control_rpc(
6826                            connection_id,
6827                            frame.header.corr,
6828                            Some(&probe.op),
6829                            outcome,
6830                        )
6831                        .map_err(RouterError::Forwarding)?;
6832                    if !self.observe_module_control_completion(completion) {
6833                        debug!(
6834                            connection_id = connection_id.get(),
6835                            corr = frame.header.corr,
6836                            op = %probe.op,
6837                            "dropping late or unknown module-control RPC response"
6838                        );
6839                    }
6840                    return Ok(Vec::new());
6841                }
6842                Err(err) => {
6843                    if let Some(expected_op) = self
6844                        .forwarding
6845                        .pending_module_control_op(connection_id, frame.header.corr)
6846                        .map_err(RouterError::Forwarding)?
6847                    {
6848                        let completion = self
6849                            .forwarding
6850                            .complete_module_control_rpc(
6851                                connection_id,
6852                                frame.header.corr,
6853                                None,
6854                                ModuleControlRpcOutcome::MalformedResponse(format!(
6855                                    "malformed {expected_op} response body: {err}"
6856                                )),
6857                            )
6858                            .map_err(RouterError::Forwarding)?;
6859                        if !self.observe_module_control_completion(completion) {
6860                            debug!(
6861                                connection_id = connection_id.get(),
6862                                corr = frame.header.corr,
6863                                "dropping late malformed module-control RPC response"
6864                            );
6865                        }
6866                        return Ok(Vec::new());
6867                    }
6868                    let message = format!("malformed route.bind response body: {err}");
6869                    secondary_error = Some(control_error_frame(
6870                        &frame,
6871                        "invalid_control_body",
6872                        message.clone(),
6873                    )?);
6874                    RouteBindRelayOutcome::ModuleGone(message)
6875                }
6876            },
6877            FrameType::Error => {
6878                if self
6879                    .forwarding
6880                    .pending_module_control_op(connection_id, frame.header.corr)
6881                    .map_err(RouterError::Forwarding)?
6882                    .is_some()
6883                {
6884                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6885                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6886                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6887                            "malformed module-control ERROR body: {err}"
6888                        )),
6889                    };
6890                    let completion = self
6891                        .forwarding
6892                        .complete_module_control_rpc(
6893                            connection_id,
6894                            frame.header.corr,
6895                            None,
6896                            outcome,
6897                        )
6898                        .map_err(RouterError::Forwarding)?;
6899                    if !self.observe_module_control_completion(completion) {
6900                        debug!(
6901                            connection_id = connection_id.get(),
6902                            corr = frame.header.corr,
6903                            "dropping late or unknown module-control RPC error"
6904                        );
6905                    }
6906                    return Ok(Vec::new());
6907                }
6908                match serde_json::from_slice::<ErrorBody>(&frame.body) {
6909                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
6910                    Err(err) => {
6911                        let message = format!("malformed route.bind ERROR body: {err}");
6912                        secondary_error = Some(control_error_frame(
6913                            &frame,
6914                            "invalid_control_body",
6915                            message.clone(),
6916                        )?);
6917                        RouteBindRelayOutcome::ModuleGone(message)
6918                    }
6919                }
6920            }
6921            ty => {
6922                return Ok(vec![control_error_frame(
6923                    &frame,
6924                    "unsupported_control_frame",
6925                    format!("unsupported module channel-0 frame {ty:?}"),
6926                )?])
6927            }
6928        };
6929
6930        let settled =
6931            self.forwarding
6932                .complete_pending_relay(connection_id, frame.header.corr, outcome);
6933        let completion = match settled {
6934            Ok(completion) => completion,
6935            Err(err) => {
6936                self.refuse_to_end_module_connection_for_a_client(
6937                    connection_id,
6938                    frame.header.corr,
6939                    err,
6940                )?;
6941                return Ok(secondary_error.into_iter().collect());
6942            }
6943        };
6944        if let Some(target) = completion.abandoned.as_ref() {
6945            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6946        }
6947        if !completion.settled {
6948            debug!(
6949                connection_id = connection_id.get(),
6950                corr = frame.header.corr,
6951                frame_type = ?frame.header.ty,
6952                "dropping late or unknown route.bind relay response"
6953            );
6954        }
6955        Ok(secondary_error.into_iter().collect())
6956    }
6957
6958    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6959        debug!(connection_id = connection_id.get(), "handling GOODBYE");
6960        // GOODBYE ends the connection's logical session even when its socket
6961        // stays open. Use disconnect teardown so verdicts, client notices and
6962        // scope authority are released at the same lifecycle boundary.
6963        self.cleanup_connection_with_end_reason(
6964            connection_id,
6965            RegistrationEndReason::ExplicitGoodbye,
6966        )
6967        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6968        Ok(Vec::new())
6969    }
6970}
6971
6972impl Default for ControlHandler {
6973    fn default() -> Self {
6974        Self::new(Arc::new(Registry::default()))
6975    }
6976}
6977
6978impl crate::supervise::SwapPromotionObserver for ControlHandler {
6979    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6980        self.apply_registration_capabilities(registration);
6981    }
6982}
6983
6984fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6985    CapabilityRequirementStatus {
6986        consumer: status.consumer,
6987        capability: status.capability,
6988        need: match status.need {
6989            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6990            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6991        },
6992        verdict: status.verdict.as_str().to_string(),
6993        episode_seq: status.episode_seq,
6994        config_satisfiable: status.config_satisfiable,
6995        runtime_available: status.runtime_available,
6996        detail: status.detail,
6997    }
6998}
6999
7000fn append_capability_problem_detail(
7001    detail: Option<String>,
7002    capability_detail: Option<String>,
7003) -> Option<String> {
7004    match (detail, capability_detail) {
7005        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7006        (Some(detail), None) => Some(detail),
7007        (None, Some(capability_detail)) => Some(capability_detail),
7008        (None, None) => None,
7009    }
7010}
7011
7012fn subc_ops() -> Vec<String> {
7013    SUBC_CONTROL_OPS
7014        .iter()
7015        .map(|op| (*op).to_string())
7016        .collect()
7017}
7018
7019fn module_subc_ops() -> Vec<String> {
7020    SUBC_CONTROL_OPS
7021        .iter()
7022        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7023        .map(|op| (*op).to_string())
7024        .collect()
7025}
7026
7027#[cfg(test)]
7028fn module_baseline_control_ops() -> Vec<String> {
7029    MODULE_BASELINE_CONTROL_OPS
7030        .iter()
7031        .map(|op| (*op).to_string())
7032        .collect()
7033}
7034
7035fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7036    let mut seen = HashSet::new();
7037    let mut effective = Vec::new();
7038    for op in MODULE_BASELINE_CONTROL_OPS {
7039        if seen.insert((*op).to_string()) {
7040            effective.push((*op).to_string());
7041        }
7042    }
7043    for op in declared.unwrap_or_default() {
7044        if seen.insert(op.clone()) {
7045            effective.push(op);
7046        }
7047    }
7048    effective
7049}
7050
7051fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7052    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7053}
7054
7055fn target_module_id(target: &RouteTarget) -> &str {
7056    match target {
7057        RouteTarget::ToolProvider { module_id }
7058        | RouteTarget::ManagementSurface { module_id }
7059        | RouteTarget::InternalService { module_id, .. } => module_id,
7060    }
7061}
7062
7063fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7064    roles.iter().any(|role| match (target, role) {
7065        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7066        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7067        (
7068            RouteTarget::InternalService { service_id, .. },
7069            ProviderRole::InternalService {
7070                service_id: provided,
7071                ..
7072            },
7073        ) => service_id == provided,
7074        _ => false,
7075    })
7076}
7077
7078fn is_routable_role(role: &ProviderRole) -> bool {
7079    matches!(
7080        role,
7081        ProviderRole::ToolProvider { .. }
7082            | ProviderRole::ManagementSurface { .. }
7083            | ProviderRole::InternalService { .. }
7084    )
7085}
7086
7087#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7088enum ControlRequestBodyError {
7089    UnknownOp,
7090    InvalidBody,
7091}
7092
7093#[derive(Debug, Deserialize)]
7094struct ControlOpProbe {
7095    op: String,
7096}
7097
7098/// Channel-0 push ops this subc version understands. A push whose `op` is not in
7099/// this set is treated as a forward-compat unknown and ignored rather than errored.
7100const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7101
7102fn is_known_module_push_op(body: &[u8]) -> bool {
7103    serde_json::from_slice::<ControlOpProbe>(body)
7104        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7105        .unwrap_or(false)
7106}
7107
7108fn is_known_module_request_op(body: &[u8]) -> bool {
7109    serde_json::from_slice::<ControlOpProbe>(body)
7110        .map(|probe| is_module_to_subc_op(&probe.op))
7111        .unwrap_or(false)
7112}
7113
7114fn is_module_to_subc_op(op: &str) -> bool {
7115    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7116}
7117
7118fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7119    debug!(
7120        op = %op,
7121        connection_id = connection_id.get(),
7122        corr,
7123        "control dispatch"
7124    );
7125}
7126
7127fn log_slow_control_dispatch(
7128    dispatch_started_at: Option<StdInstant>,
7129    op: &'static str,
7130    connection_id: ConnectionId,
7131    corr: u64,
7132    route_open_timings: Option<&RouteOpenTimings>,
7133) {
7134    let Some(dispatch_started_at) = dispatch_started_at else {
7135        return;
7136    };
7137    // Use the same monotonic clock as the phase timers and relay deadline.
7138    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7139    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7140        if let Some(timings) = route_open_timings {
7141            warn!(
7142                op = %op,
7143                connection_id = connection_id.get(),
7144                corr,
7145                elapsed_ms = elapsed.as_millis() as u64,
7146                module_id = ?timings.module_id,
7147                principal = %timings.principal,
7148                admission_ms = timings.admission.as_millis() as u64,
7149                project_root_ms = timings.project_root.as_millis() as u64,
7150                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7151                bind_send_ms = timings.bind_send.as_millis() as u64,
7152                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7153                "slow control dispatch"
7154            );
7155            return;
7156        }
7157        warn!(
7158            op = %op,
7159            connection_id = connection_id.get(),
7160            corr,
7161            elapsed_ms = elapsed.as_millis() as u64,
7162            "slow control dispatch"
7163        );
7164    }
7165}
7166
7167/// A test stand-in for the OS caller lookup, given the recorded endpoints.
7168#[cfg(test)]
7169type TestCallerLookup =
7170    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7171
7172/// Every new client operation needs an explicit privilege decision here.
7173fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7174    match request {
7175        ClientControlRequest::SupervisorRestart { module_id, .. } => {
7176            Some(PrivilegedOperation::Restart { module_id })
7177        }
7178        ClientControlRequest::SupervisorSwap { module_id, .. } => {
7179            Some(PrivilegedOperation::Swap { module_id })
7180        }
7181        ClientControlRequest::SupervisorReload { module_id } => {
7182            Some(PrivilegedOperation::Reload { module_id })
7183        }
7184        ClientControlRequest::SupervisorRescan { preview } => {
7185            if *preview {
7186                None
7187            } else {
7188                Some(PrivilegedOperation::Rescan)
7189            }
7190        }
7191        // Only the well-formed apply asks for approval. A preview changes
7192        // nothing, and any other combination is refused before it touches
7193        // anything, so prompting the operator for it would ask about a request
7194        // that cannot run.
7195        ClientControlRequest::SupervisorRescanChecked {
7196            preview,
7197            candidate,
7198            expect,
7199        } => (!*preview && candidate.is_none() && expect.is_some())
7200            .then_some(PrivilegedOperation::Rescan),
7201        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7202            Some(PrivilegedOperation::SetEnabled {
7203                module_id,
7204                enabled: *enabled,
7205            })
7206        }
7207        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7208            Some(PrivilegedOperation::ReleaseReserved { module_id })
7209        }
7210        ClientControlRequest::ServerDescribe {}
7211        | ClientControlRequest::CatalogList { .. }
7212        | ClientControlRequest::RouteOpen { .. }
7213        | ClientControlRequest::RoutePoll { .. }
7214        | ClientControlRequest::SupervisorList {}
7215        | ClientControlRequest::SupervisorHealth {}
7216        | ClientControlRequest::SupervisorHealthProbe { .. }
7217        | ClientControlRequest::SupervisorRoutes { .. }
7218        | ClientControlRequest::SupervisorProvenance { .. }
7219        | ClientControlRequest::SupervisorStderrTail { .. }
7220        | ClientControlRequest::SupervisorTerminals { .. }
7221        | ClientControlRequest::SupervisorSpawnSnapshot {}
7222        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7223    }
7224}
7225
7226fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7227    match request {
7228        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7229        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7230        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7231        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7232        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7233        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7234        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7235        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7236        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7237        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7238        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7239        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7240        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7241        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7242        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7243        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7244        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7245        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7246        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7247        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7248    }
7249}
7250
7251/// The `scope=` field of the `route.open accepted` line, from the stamp that
7252/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
7253/// so a ref holding control characters cannot forge a log line, and cut to 64
7254/// characters with a trailing `…` to bound the line. The bind proof and the
7255/// scope's attributes never appear.
7256fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7257    use subc_protocol::scope::ScopeKind;
7258    const MAX_REF_CHARS: usize = 64;
7259    let kind = match stamp.kind {
7260        ScopeKind::Head => "head",
7261        ScopeKind::Worker => "worker",
7262        ScopeKind::Ephemeral => "ephemeral",
7263    };
7264    let mut scope_ref: String = stamp.scope_ref.chars().take(MAX_REF_CHARS).collect();
7265    if stamp.scope_ref.chars().nth(MAX_REF_CHARS).is_some() {
7266        scope_ref.push('…');
7267    }
7268    format!(
7269        "{kind}/{}/{scope_ref:?}@{}",
7270        principal_label(&stamp.owner),
7271        stamp.scope_epoch
7272    )
7273}
7274
7275fn principal_label(principal: &Principal) -> String {
7276    match principal {
7277        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7278        Principal::Direct => "direct".to_string(),
7279        other => format!("{other:?}"),
7280    }
7281}
7282
7283fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7284    match request {
7285        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7286        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7287        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7288        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7289        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7290    }
7291}
7292
7293fn parse_client_control_request(
7294    body: &[u8],
7295) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7296    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7297        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7298            Ok(probe)
7299                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7300                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7301            {
7302                ControlRequestBodyError::InvalidBody
7303            }
7304            Ok(_) => ControlRequestBodyError::UnknownOp,
7305            Err(_) => ControlRequestBodyError::InvalidBody,
7306        };
7307        (err, classification)
7308    })
7309}
7310
7311fn parse_module_control_request_from_module(
7312    body: &[u8],
7313) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7314    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7315        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7316            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7317            Ok(_) => ControlRequestBodyError::UnknownOp,
7318            Err(_) => ControlRequestBodyError::InvalidBody,
7319        };
7320        (err, classification)
7321    })
7322}
7323
7324#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7325enum ProviderRoleKind {
7326    ToolProvider,
7327    PipelineStage,
7328    ManagementSurface,
7329    InternalService,
7330}
7331
7332fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7333    match role {
7334        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7335        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7336        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7337        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7338    }
7339}
7340
7341fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7342    roles.iter().map(provider_role_kind).collect()
7343}
7344
7345/// Most refused scope records named individually in the log per sync; the
7346/// `refused` count on the accepted line is always complete.
7347const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7348
7349/// Per-outcome counts of one accepted `scope.sync`, for its log line.
7350#[derive(Debug, Default, PartialEq, Eq)]
7351struct ScopeOutcomeCounts {
7352    created: usize,
7353    replaced: usize,
7354    updated: usize,
7355    unchanged: usize,
7356    refused: usize,
7357}
7358
7359impl ScopeOutcomeCounts {
7360    fn of(results: &[ScopeRecordResult]) -> Self {
7361        let mut counts = Self::default();
7362        for result in results {
7363            let slot = match result.outcome {
7364                ScopeRecordOutcome::Created => &mut counts.created,
7365                ScopeRecordOutcome::Replaced => &mut counts.replaced,
7366                ScopeRecordOutcome::Updated => &mut counts.updated,
7367                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7368                ScopeRecordOutcome::Refused => &mut counts.refused,
7369            };
7370            *slot += 1;
7371        }
7372        counts
7373    }
7374}
7375
7376#[cfg(test)]
7377mod scope_outcome_count_tests {
7378    use super::*;
7379
7380    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7381        ScopeRecordResult {
7382            scope_ref: "r".to_string(),
7383            scope_epoch: 1,
7384            outcome,
7385            code: None,
7386            message: None,
7387            version: None,
7388            parent_state: None,
7389        }
7390    }
7391
7392    /// Each outcome lands in its own count, so a refused record can never be
7393    /// hidden inside the total the log already printed.
7394    #[test]
7395    fn every_outcome_is_counted_in_its_own_field() {
7396        let results = [
7397            result(ScopeRecordOutcome::Created),
7398            result(ScopeRecordOutcome::Created),
7399            result(ScopeRecordOutcome::Replaced),
7400            result(ScopeRecordOutcome::Updated),
7401            result(ScopeRecordOutcome::Unchanged),
7402            result(ScopeRecordOutcome::Refused),
7403            result(ScopeRecordOutcome::Refused),
7404            result(ScopeRecordOutcome::Refused),
7405        ];
7406        assert_eq!(
7407            ScopeOutcomeCounts::of(&results),
7408            ScopeOutcomeCounts {
7409                created: 2,
7410                replaced: 1,
7411                updated: 1,
7412                unchanged: 1,
7413                refused: 3,
7414            }
7415        );
7416    }
7417}
7418
7419/// Return whether a catalog change can create a newly violating live route.
7420/// Removing an attested claim is intentionally excluded: it makes fewer routes
7421/// forbidden and therefore must leave the existing route census untouched.
7422fn capability_census_trigger(
7423    old: Option<&CapabilityDeclarations>,
7424    new: Option<&CapabilityDeclarations>,
7425) -> bool {
7426    let old_provides = old
7427        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7428        .unwrap_or_default();
7429    let old_denies = old
7430        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7431        .unwrap_or_default();
7432    let new = new.cloned().unwrap_or(CapabilityDeclarations {
7433        provides: Vec::new(),
7434        requires: Vec::new(),
7435        must_never_reach: Vec::new(),
7436    });
7437
7438    new.provides
7439        .iter()
7440        .any(|capability| !old_provides.contains(capability))
7441        || new
7442            .must_never_reach
7443            .iter()
7444            .any(|capability| !old_denies.contains(capability))
7445}
7446
7447/// Find the first capability an attested opener denies that an attested target
7448/// claims. Both manifests are live registry records, never cached or client data.
7449fn denied_capability<'a>(
7450    opening_manifest: &'a ModuleManifest,
7451    target_manifest: &ModuleManifest,
7452) -> Option<&'a str> {
7453    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7454    let target_capabilities = target_manifest.capabilities.as_ref()?;
7455    opening_capabilities
7456        .must_never_reach
7457        .iter()
7458        .find(|denied| {
7459            target_capabilities
7460                .provides
7461                .iter()
7462                .any(|provided| provided == *denied)
7463        })
7464        .map(String::as_str)
7465}
7466
7467fn catalog_update_frozen_field_message(
7468    registered: &ModuleManifest,
7469    provides: &[ProviderRole],
7470) -> Option<String> {
7471    let old_has_provides = !registered.provides.is_empty();
7472    let new_has_provides = !provides.is_empty();
7473    if old_has_provides != new_has_provides {
7474        return Some(format!(
7475            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7476            registered.module_id
7477        ));
7478    }
7479
7480    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
7481        return Some(format!(
7482            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7483            registered.module_id
7484        ));
7485    }
7486
7487    let registered_concurrency = manifest_concurrency(registered);
7488    let mut candidate = registered.clone();
7489    candidate.provides = provides.to_vec();
7490    let candidate_concurrency = manifest_concurrency(&candidate);
7491    if candidate_concurrency != registered_concurrency {
7492        return Some(format!(
7493            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7494            registered.module_id, registered_concurrency, candidate_concurrency
7495        ));
7496    }
7497
7498    // control_ops live beside the manifest in the HELLO body, not inside
7499    // ModuleManifest, so a provides-only catalog.update cannot change them.
7500    None
7501}
7502
7503fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7504    manifest.provides.iter().any(is_routable_role)
7505}
7506
7507/// Returns the routable-provider concurrency subc should enforce for this manifest.
7508///
7509/// ToolProvider and ManagementSurface store their delivery concurrency directly.
7510/// InternalService has no role-specific concurrency field, so it retains the
7511/// existing ModuleManaged default for backward compatibility.
7512fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7513    manifest
7514        .provides
7515        .iter()
7516        .find_map(|provider| match provider {
7517            ProviderRole::ToolProvider { concurrency, .. }
7518            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7519            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7520        })
7521        .unwrap_or(Concurrency::ModuleManaged)
7522}
7523
7524/// True when the manifest carries a ManagementSurface role whose concurrency
7525/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
7526/// bytes because the typed manifest deliberately erases that distinction: the
7527/// default exists for wire compatibility, and this probe exists so the default
7528/// stays observable. Any parse irregularity returns false -- the caller only
7529/// logs, and a malformed body already failed registration upstream.
7530fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7531    let has_management_surface = manifest
7532        .provides
7533        .iter()
7534        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7535    if !has_management_surface {
7536        return false;
7537    }
7538    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7539        return false;
7540    };
7541    let Some(provides) = raw
7542        .get("manifest")
7543        .and_then(|manifest| manifest.get("provides"))
7544        .and_then(serde_json::Value::as_array)
7545    else {
7546        return false;
7547    };
7548    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
7549    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
7550    // against the management_surface_manifest_without_concurrency golden, not
7551    // recalled (the externally-tagged guess was this function's first bug).
7552    provides.iter().any(|role| {
7553        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7554            && role.get("concurrency").is_none()
7555    })
7556}
7557
7558fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7559    if peer_version != PROTOCOL_VERSION {
7560        return Err(format!(
7561            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7562        ));
7563    }
7564    Ok(PROTOCOL_VERSION)
7565}
7566
7567fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7568    Frame::build_with_version(
7569        response_version(frame),
7570        FrameType::Pong,
7571        frame.header.flags,
7572        0,
7573        0,
7574        frame.header.corr,
7575        Vec::new(),
7576    )
7577    .map_err(RouterError::FrameBuild)
7578}
7579
7580pub(crate) fn control_error_frame(
7581    frame: &Frame,
7582    code: &'static str,
7583    message: impl Into<String>,
7584) -> Result<Frame, RouterError> {
7585    control_error_body_frame(
7586        frame,
7587        ErrorBody {
7588            code: code.to_string(),
7589            message: message.into(),
7590            detail: None,
7591        },
7592    )
7593}
7594
7595fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7596    let body = serde_json::to_vec(&error).map_err(|err| {
7597        RouterError::backend(
7598            0,
7599            frame.header.corr,
7600            format!("failed to encode control ERROR: {err}"),
7601        )
7602    })?;
7603
7604    Frame::build_with_version(
7605        response_version(frame),
7606        FrameType::Error,
7607        control_flags(),
7608        0,
7609        0,
7610        frame.header.corr,
7611        body,
7612    )
7613    .map_err(RouterError::FrameBuild)
7614}
7615
7616fn control_response_body_frame<T: Serialize>(
7617    frame: &Frame,
7618    reply: &T,
7619    label: &'static str,
7620) -> Result<Frame, RouterError> {
7621    let body = serde_json::to_vec(reply).map_err(|err| {
7622        RouterError::backend(
7623            0,
7624            frame.header.corr,
7625            format!("failed to encode {label}: {err}"),
7626        )
7627    })?;
7628
7629    Frame::build_with_version(
7630        response_version(frame),
7631        FrameType::Response,
7632        control_flags(),
7633        0,
7634        0,
7635        frame.header.corr,
7636        body,
7637    )
7638    .map_err(RouterError::FrameBuild)
7639}
7640
7641/// Map a forwarding failure to the wire code a client sees.
7642///
7643/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
7644/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
7645/// chosen here decides whether a caller retries or gives up.
7646///
7647/// That makes attribution the load-bearing property, not merely having a code. A
7648/// permanent fault published as a retryable one produces a fleet-wide retry storm
7649/// against something that can never recover; a transient fault published as
7650/// permanent gives up on work that would have succeeded. Both look correct in a
7651/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
7652/// the mapping per variant rather than merely asserting that some code exists.
7653///
7654/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
7655/// two codes on the same side of the boundary passes it. Measured rather than
7656/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
7657/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
7658/// a test named for something else that happens to assert the string.
7659///
7660/// That accidental coverage is deliberately left alone rather than promoted to a
7661/// named test, because it guards a property this function does not promise.
7662/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
7663/// specific code within a class, so identity is free to change and only the
7664/// partition is a contract. Splitting it out would assert a guarantee nothing
7665/// depends on — and a suite that promises more than the code does is the harder
7666/// thing to correct later, because the next reader cannot tell which assertions
7667/// are load-bearing.
7668///
7669/// Pin identity here the moment a consumer branches on a specific code.
7670fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7671    match err {
7672        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7673        ForwardingError::NoModuleConnection => "target_unavailable",
7674        ForwardingError::ModuleReloading { .. } => "module_reloading",
7675        ForwardingError::ClientRouteChannelExhausted { .. }
7676        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7677        ForwardingError::StaleModuleEndpoint
7678        | ForwardingError::UnknownReservation { .. }
7679        | ForwardingError::ConnectionClosing { .. }
7680        | ForwardingError::ClientEgressClosed { .. }
7681        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7682        // Only a swap candidate's registration can produce this, and it means
7683        // exactly what a second active HELLO for a live id means.
7684        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7685        ForwardingError::RelayCorrelationExhausted
7686        | ForwardingError::RouteOpenBuild(_)
7687        | ForwardingError::Poisoned => "forwarding_error",
7688    }
7689}
7690
7691fn response_version(frame: &Frame) -> u8 {
7692    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7693        frame.header.ver
7694    } else {
7695        PROTOCOL_VERSION
7696    }
7697}
7698
7699fn control_flags() -> Flags {
7700    Flags::new(false, Priority::Passive, false)
7701}
7702
7703/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
7704/// channel. The target is the module (a client never saw the route), so this
7705/// takes the module path: delivered late rather than dropped when the module's
7706/// queue is momentarily full, and never closing its connection.
7707fn send_goodbye_target_best_effort(
7708    counters: &DaemonCounters,
7709    target: &GoodbyeTarget,
7710    context: &'static str,
7711) {
7712    let Ok(frame) = Frame::build_with_version(
7713        target.negotiated_ver,
7714        FrameType::Goodbye,
7715        control_flags(),
7716        target.channel,
7717        target.epoch,
7718        0,
7719        Vec::new(),
7720    ) else {
7721        return;
7722    };
7723    crate::forwarding::send_module_route_goodbye(
7724        counters,
7725        &target.sink,
7726        frame,
7727        target.module_id.as_deref(),
7728        context,
7729    );
7730}
7731
7732pub(crate) fn send_route_control_pushes(
7733    forwarding: &ForwardingTable,
7734    routes: Vec<EndpointRoute>,
7735    push: ClientControlPush,
7736) {
7737    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7738    for route in routes {
7739        let target = route.goodbye_target;
7740        if let Some((existing, channels)) = targets
7741            .iter_mut()
7742            .find(|(existing, _)| existing.connection_id == target.connection_id)
7743        {
7744            debug_assert_eq!(
7745                existing.negotiated_ver, target.negotiated_ver,
7746                "one connection cannot negotiate multiple frame versions"
7747            );
7748            if !channels.contains(&target.channel) {
7749                channels.push(target.channel);
7750            }
7751            continue;
7752        }
7753        let channel = target.channel;
7754        targets.push((target, vec![channel]));
7755    }
7756    for (target, mut channels) in targets {
7757        channels.sort_unstable();
7758        let mut push = push.clone();
7759        match &mut push {
7760            ClientControlPush::RouteClosing {
7761                channels: covered, ..
7762            }
7763            | ClientControlPush::RouteClosed {
7764                channels: covered, ..
7765            } => *covered = channels,
7766        }
7767        let body = match serde_json::to_vec(&push) {
7768            Ok(body) => body,
7769            Err(err) => {
7770                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7771                continue;
7772            }
7773        };
7774        let frame = match Frame::build_with_version(
7775            target.negotiated_ver,
7776            FrameType::Push,
7777            control_flags(),
7778            0,
7779            0,
7780            0,
7781            body.clone(),
7782        ) {
7783            Ok(frame) => frame,
7784            Err(err) => {
7785                warn!(
7786                    route_channel = target.channel,
7787                    error = %err,
7788                    "failed to build route lifecycle control PUSH frame"
7789                );
7790                continue;
7791            }
7792        };
7793        if let Err(err) = target.sink.try_send(frame) {
7794            if target.close_on_delivery_failure() {
7795                warn!(
7796                    target_connection_id = target.connection_id.get(),
7797                    route_channel = target.channel,
7798                    error = %err,
7799                    "route lifecycle control PUSH was not delivered to client; closing target connection"
7800                );
7801                let _ = forwarding.escalate_client_delivery_failure(
7802                    target.connection_id,
7803                    target.channel,
7804                    target.epoch,
7805                    CloseReason::new(
7806                        "route_lifecycle_push_delivery_failed",
7807                        format!(
7808                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7809                            target.channel
7810                        ),
7811                    ),
7812                    crate::forwarding::UndeliveredFrame {
7813                        module_id: target.module_id.as_deref(),
7814                        sink: &target.sink,
7815                    },
7816                );
7817            }
7818        }
7819    }
7820}
7821
7822#[cfg(test)]
7823mod tests {
7824    mod activity_read {
7825        include!("activity_read_tests.rs");
7826    }
7827
7828    mod lifecycle_activity {
7829        include!("control_lifecycle_tests.rs");
7830    }
7831    #[cfg(unix)]
7832    #[tokio::test]
7833    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7834        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7835        let path = dir.join("subc.jsonc");
7836        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7837            "program":"/bin/sleep","args":["60"],"protocol":"none",
7838            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7839        }}}).to_string()).unwrap();
7840        let mut configured = crate::daemon_config::load(&path)
7841            .unwrap()
7842            .unwrap()
7843            .modules
7844            .pop()
7845            .unwrap();
7846        let registry = std::sync::Arc::new(crate::Registry::default());
7847        let handle = crate::SupervisorHandle::new();
7848        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7849            .with_handle(handle.clone());
7850        let module = supervisor
7851            .supervise_configured_with_health(
7852                configured.module_spec(),
7853                true,
7854                configured.health.clone(),
7855                None,
7856                configured.restart,
7857            )
7858            .unwrap();
7859        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7860        let before = module.status().unwrap().pid;
7861        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7862        configured.health.cadence = std::time::Duration::from_secs(3600);
7863        let health_only = handler
7864            .reconcile_supervised_modules(
7865                &supervisor,
7866                vec![configured.clone()],
7867                false,
7868                Principal::Direct,
7869            )
7870            .await
7871            .unwrap();
7872        assert!(
7873            health_only.0.changed_pending_reload.is_empty(),
7874            "health policy is already applied live"
7875        );
7876        assert_eq!(module.status().unwrap().pid, before);
7877        assert_eq!(
7878            module.configuration().unwrap().1.http,
7879            configured.health.http
7880        );
7881        configured.args = vec!["61".into()];
7882        let launch = handler
7883            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7884            .await
7885            .unwrap();
7886        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
7887        assert_eq!(
7888            module.status().unwrap().pid,
7889            before,
7890            "a launch edit is stored until reload"
7891        );
7892        module.drain().await.unwrap();
7893    }
7894    use cortexkit_test_support::ScratchDir;
7895    use std::{
7896        collections::BTreeMap,
7897        fmt,
7898        path::PathBuf,
7899        sync::{Arc, Mutex},
7900        time::Duration,
7901    };
7902
7903    use serde_json::{json, Value};
7904    use subc_protocol::{
7905        manifest::{
7906            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7907            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7908        },
7909        session::HealthStatus,
7910        FrameType,
7911    };
7912
7913    use super::*;
7914    use crate::{
7915        forwarding::{DataRoute, DataRouteState},
7916        registry::ChannelState,
7917        router::FrameSink,
7918        stderr_tail::DEFAULT_MAX_LINE_BYTES,
7919        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7920        RouteCtx, Router,
7921    };
7922    use tokio::{
7923        sync::mpsc,
7924        time::{sleep, Instant},
7925    };
7926    use tracing::{
7927        field::{Field, Visit},
7928        Event, Subscriber,
7929    };
7930    use tracing_subscriber::{layer::Context, prelude::*, Layer};
7931
7932    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
7933    ///
7934    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
7935    /// is only populated for `tests/*.rs` integration test binaries -- this file
7936    /// compiles as part of the library target, which gets neither. This test's
7937    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
7938    /// and the sibling binary lives two directories up at
7939    /// `<target-dir>/<profile>/fake-aft-stub`.
7940    ///
7941    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
7942    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
7943    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
7944    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
7945    /// `NotFound`, which reads as a broken test rather than an unbuilt
7946    /// dependency -- so state the cause and the remedy instead. Deliberately a
7947    /// panic and not a silent skip: a test that quietly passes when it could not
7948    /// run is worse than one that fails, because it reports health it never
7949    /// verified.
7950    fn fake_aft_stub_path() -> PathBuf {
7951        let mut path = std::env::current_exe().expect("current_exe available in tests");
7952        path.pop(); // .../deps/
7953        path.pop(); // .../<profile>/
7954        path.push(if cfg!(windows) {
7955            "fake-aft-stub.exe"
7956        } else {
7957            "fake-aft-stub"
7958        });
7959        assert!(
7960            path.exists(),
7961            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7962             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7963            path.display()
7964        );
7965        path
7966    }
7967
7968    /// Whether clients retry `code` in place: the predicate itself, never a copy
7969    /// of its set. A copied list breaks silently when a code is added to or
7970    /// removed from the real one, and a stale copy here would let exactly the
7971    /// failure this test exists to catch pass.
7972    fn client_retries(code: &str) -> bool {
7973        subc_protocol::error_codes::is_retryable_route_open(code)
7974    }
7975
7976    /// A code is not a label — clients branch on it, so publishing the wrong KIND
7977    /// of failure is worse than publishing none. A permanent fault dressed as
7978    /// retryable makes every client in the fleet retry forever against something
7979    /// that cannot recover; a transient fault dressed as permanent abandons work
7980    /// that would have succeeded.
7981    ///
7982    /// Asserting "a code exists" cannot catch either, because the string is free
7983    /// to say anything. This enumerates every variant and pins which side of the
7984    /// retry boundary it lands on, so a new variant must be classified here
7985    /// deliberately rather than inheriting whichever arm it was appended to.
7986    #[test]
7987    fn retryability_of_forwarding_codes_matches_the_failure() {
7988        // Transient by nature: the target is booting, reloading, or its endpoint
7989        // was swapped mid-flight. Retrying is how these resolve.
7990        let transient = [
7991            ForwardingError::NoModuleConnection,
7992            ForwardingError::ModuleReloading {
7993                module_id: "m".into(),
7994            },
7995            ForwardingError::StaleModuleEndpoint,
7996            ForwardingError::UnknownReservation {
7997                client_channel: 1,
7998                module_channel: 1,
7999            },
8000            ForwardingError::ConnectionClosing {
8001                connection_id: ConnectionId::new(1),
8002            },
8003            ForwardingError::ClientEgressClosed {
8004                connection_id: ConnectionId::new(1),
8005            },
8006            ForwardingError::ModuleEgressUnavailable {
8007                connection_id: ConnectionId::new(1),
8008            },
8009        ];
8010        for err in transient {
8011            let code = forwarding_error_code(&err);
8012            assert!(
8013                client_retries(code),
8014                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8015            );
8016        }
8017
8018        // Not fixed by retrying. Channel and correlation exhaustion need the
8019        // caller to close routes, and a poisoned lock is a daemon that cannot
8020        // recover at all — the worst thing to advertise as retryable, since every
8021        // client would storm a daemon that will never answer.
8022        let permanent = [
8023            ForwardingError::ConnectionRoleConflict {
8024                connection_id: ConnectionId::new(1),
8025            },
8026            ForwardingError::ClientRouteChannelExhausted {
8027                connection_id: ConnectionId::new(1),
8028            },
8029            ForwardingError::ModuleRouteChannelExhausted {
8030                endpoint: ModuleEndpointId {
8031                    connection_id: ConnectionId::new(1),
8032                    generation: 1,
8033                },
8034            },
8035            ForwardingError::RelayCorrelationExhausted,
8036            ForwardingError::RouteOpenBuild("x".into()),
8037            ForwardingError::Poisoned,
8038        ];
8039        for err in permanent {
8040            let code = forwarding_error_code(&err);
8041            assert!(
8042                !client_retries(code),
8043                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8044            );
8045        }
8046    }
8047
8048    /// The principal is the daemon's answer to "who is calling", and modules
8049    /// branch on it: aft gates bash on it, cerebellum gates browser control,
8050    /// plexus gates connector invocation. So a stamp is an authorization input in
8051    /// another process, not a label — and both possible answers SUCCEED, which is
8052    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
8053    /// first-party capability to something that never proved it; a supervised one
8054    /// stamped `Direct` silently strips a module of capability it is entitled to.
8055    ///
8056    /// Neither shows up in a test that only checks the bind succeeded. Before this
8057    /// test the only coverage was accidental —
8058    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
8059    /// stamped principal on its way past, so narrowing that wire-shape test to its
8060    /// stated subject would have deleted the last assertion on this value. It
8061    /// still asserts the stamp, which is now redundancy rather than the only
8062    /// guard: both fail under the same mutation, and this one names the reason.
8063    /// SCOPE: this handler's supervisor has spawned nothing, so
8064    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
8065    /// is unreachable here. Both assertions below are refusals, and a mutant that
8066    /// refuses everything would satisfy them.
8067    ///
8068    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
8069    /// spawns a supervised consumer, reads its live nonce, and asserts the module
8070    /// observed `principal.kind == "reserved"` carrying that module_id — verified
8071    /// at source rather than assumed, since a citation is a claim about another
8072    /// file and ages like one. Recorded because a harness that structurally
8073    /// cannot reach an arm reports "none" for that arm identically to one that
8074    /// covers it and found nothing.
8075    #[tokio::test]
8076    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8077        let handler = ControlHandler::default();
8078        let frame =
8079            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8080
8081        // Absent consumer_identity is the ordinary case: a human at a terminal, or
8082        // any process holding the connection file. Nothing was proved, so nothing
8083        // may be granted beyond the unattested floor.
8084        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8085        assert_eq!(
8086            stamped,
8087            Principal::Direct,
8088            "a caller that proved nothing must not be stamped as a supervised module"
8089        );
8090
8091        // A claimed module_id with a nonce no supervised child was given is a
8092        // forgery attempt, not a weaker caller: it must be REFUSED rather than
8093        // quietly demoted to Direct, or an impersonation attempt looks identical
8094        // to an ordinary unattested connection.
8095        let forged = handler
8096            .route_open_principal(
8097                &frame,
8098                Some(ConsumerIdentity {
8099                    module_id: "aft".to_string(),
8100                    launch_nonce: "not-a-real-nonce".to_string(),
8101                }),
8102            )
8103            .unwrap();
8104        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8105        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8106    }
8107
8108    /// The test above hands `route_open_principal` an identity it built itself,
8109    /// which proves the stamping rule and nothing about where the identity comes
8110    /// from. The real producer is a wire body, and the two are joined by a serde
8111    /// field name that nothing else asserts.
8112    ///
8113    /// That join fails quietly in one specific way: an unrecognised key is simply
8114    /// absent after parsing, so a renamed or misspelled `consumer_identity`
8115    /// yields `None` and every supervised module silently drops to `Direct`.
8116    /// Capability-wise that is the safe direction, but it surfaces far from its
8117    /// cause — as a module mysteriously refused bash — and it would pass every
8118    /// test that builds its own input.
8119    ///
8120    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
8121    /// would break every client the moment the daemon gains a field, trading a
8122    /// quiet demotion for a hard refusal on additive change. Asserting the join
8123    /// instead means a rename breaks a test here rather than the fleet.
8124    #[test]
8125    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8126        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8127        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8128        let ClientControlRequest::RouteOpen {
8129            consumer_identity, ..
8130        } = parsed
8131        else {
8132            panic!("route.open body must parse as RouteOpen");
8133        };
8134        assert_eq!(
8135            consumer_identity,
8136            Some(ConsumerIdentity {
8137                module_id: "aft".to_string(),
8138                launch_nonce: "n".to_string(),
8139            }),
8140            "the wire field name must reach the value route_open_principal reads"
8141        );
8142    }
8143
8144    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8145        ModuleManifest::builder(module_id, "0.1.0")
8146            .protocol_ver(protocol_ver)
8147            .provides(vec![ProviderRole::ToolProvider {
8148                tools: vec![Tool {
8149                    name: "read".to_string(),
8150                    description: None,
8151                    execution_mode: ExecutionMode::Pure,
8152                    schema: json!({"type": "object"}),
8153                }],
8154                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8155                concurrency: Concurrency::ModuleManaged,
8156                emits_push: true,
8157                sub_supervises: true,
8158            }])
8159            .build()
8160    }
8161
8162    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8163        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8164    }
8165
8166    fn hello_frame_with_control_ops(
8167        module_id: &str,
8168        protocol_ver: u8,
8169        corr: u64,
8170        control_ops: Option<Vec<String>>,
8171    ) -> Frame {
8172        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8173    }
8174
8175    fn hello_frame_with_nonce(
8176        module_id: &str,
8177        protocol_ver: u8,
8178        corr: u64,
8179        launch_nonce: Option<&str>,
8180    ) -> Frame {
8181        hello_frame_full(
8182            module_id,
8183            protocol_ver,
8184            corr,
8185            None,
8186            launch_nonce.map(ToOwned::to_owned),
8187        )
8188    }
8189
8190    fn hello_frame_full(
8191        module_id: &str,
8192        protocol_ver: u8,
8193        corr: u64,
8194        control_ops: Option<Vec<String>>,
8195        launch_nonce: Option<String>,
8196    ) -> Frame {
8197        let body = serde_json::to_vec(&ModuleHelloBody {
8198            manifest: manifest(module_id, protocol_ver),
8199            protocol_ver,
8200            control_ops,
8201            launch_nonce,
8202        })
8203        .unwrap();
8204        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8205    }
8206
8207    fn hello_frame_with_rich_ui(
8208        module_id: &str,
8209        protocol_ver: u8,
8210        corr: u64,
8211        launch_nonce: Option<&str>,
8212        rich_ui: Value,
8213    ) -> Frame {
8214        let mut body = serde_json::to_value(ModuleHelloBody {
8215            manifest: manifest(module_id, protocol_ver),
8216            protocol_ver,
8217            control_ops: None,
8218            launch_nonce: launch_nonce.map(str::to_string),
8219        })
8220        .unwrap();
8221        body["manifest"]["rich_ui"] = rich_ui;
8222        Frame::build(
8223            FrameType::Hello,
8224            control_flags(),
8225            0,
8226            0,
8227            corr,
8228            serde_json::to_vec(&body).unwrap(),
8229        )
8230        .unwrap()
8231    }
8232
8233    fn non_routable_hello_frame_with_control_ops(
8234        module_id: &str,
8235        corr: u64,
8236        control_ops: Option<Vec<String>>,
8237    ) -> Frame {
8238        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8239        manifest.provides.clear();
8240        let body = serde_json::to_vec(&ModuleHelloBody {
8241            manifest,
8242            protocol_ver: PROTOCOL_VERSION,
8243            control_ops,
8244            launch_nonce: None,
8245        })
8246        .unwrap();
8247        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8248    }
8249
8250    fn capability_grammar_hello_frame(
8251        capabilities: Value,
8252        runtime_computed: Option<Value>,
8253        corr: u64,
8254    ) -> Frame {
8255        let mut body = serde_json::to_value(ModuleHelloBody {
8256            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8257            protocol_ver: PROTOCOL_VERSION,
8258            control_ops: None,
8259            launch_nonce: None,
8260        })
8261        .expect("HELLO body serializes");
8262        body["manifest"]["capabilities"] = capabilities;
8263        if let Some(runtime_computed) = runtime_computed {
8264            body["runtime_computed"] = runtime_computed;
8265        }
8266        Frame::build(
8267            FrameType::Hello,
8268            control_flags(),
8269            0,
8270            0,
8271            corr,
8272            serde_json::to_vec(&body).expect("HELLO body reserializes"),
8273        )
8274        .expect("HELLO frame builds")
8275    }
8276
8277    fn channel_request(channel: u16, corr: u64) -> Frame {
8278        Frame::build(
8279            FrameType::Request,
8280            Flags::new(true, Priority::Interactive, false),
8281            channel,
8282            0,
8283            corr,
8284            b"opaque".to_vec(),
8285        )
8286        .unwrap()
8287    }
8288
8289    fn route_ctx(
8290        connection_id: ConnectionId,
8291    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8292        let (tx, rx) = mpsc::channel(8);
8293        (
8294            RouteCtx {
8295                connection_id,
8296                egress: FrameSink::new(tx),
8297            },
8298            rx,
8299        )
8300    }
8301
8302    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8303        serde_json::from_slice(&frame.body).unwrap()
8304    }
8305
8306    /// Register a module over a connection that has a sink and return the
8307    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
8308    /// module's own sink rather than returning it as a reply, so the ack is
8309    /// taken off `rx` here and whatever the test reads next is what followed it.
8310    async fn hello_via_sink(
8311        handler: &ControlHandler,
8312        ctx: &RouteCtx,
8313        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8314        hello: Frame,
8315    ) -> Frame {
8316        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8317        assert!(
8318            replies.is_empty(),
8319            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8320        );
8321        let ack = rx
8322            .try_recv()
8323            .expect("HELLO_ACK is queued on the module sink")
8324            .frame;
8325        assert_eq!(ack.header.ty, FrameType::HelloAck);
8326        ack
8327    }
8328
8329    fn parse_error(frame: &Frame) -> Value {
8330        serde_json::from_slice(&frame.body).unwrap()
8331    }
8332
8333    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8334        serde_json::from_slice(&frame.body).unwrap()
8335    }
8336
8337    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8338        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8339            route_channel,
8340            route_epoch: 0,
8341            kind,
8342        })
8343        .unwrap();
8344        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8345    }
8346
8347    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8348        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8349            module_id: module_id.to_string(),
8350        })
8351        .unwrap();
8352        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8353    }
8354
8355    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8356        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8357    }
8358
8359    fn route_open_frame_with_consumer_capabilities(
8360        corr: u64,
8361        module_id: &str,
8362        project_root: ScratchDir,
8363        consumer_capabilities: Option<Vec<String>>,
8364    ) -> Frame {
8365        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8366            target: RouteTarget::ToolProvider {
8367                module_id: module_id.to_string(),
8368            },
8369            identity: BindIdentity::new(
8370                project_root.path().to_path_buf(),
8371                "unit".to_string(),
8372                "session".to_string(),
8373            ),
8374            consumer_identity: None,
8375            consumer_capabilities,
8376            role_versions: None,
8377            admission_facts: None,
8378            scope: None,
8379        })
8380        .unwrap();
8381        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8382    }
8383
8384    fn route_open_frame_with_role_versions(
8385        corr: u64,
8386        module_id: &str,
8387        project_root: ScratchDir,
8388        role_versions: Option<BTreeMap<String, String>>,
8389    ) -> Frame {
8390        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8391            target: RouteTarget::ToolProvider {
8392                module_id: module_id.to_string(),
8393            },
8394            identity: BindIdentity::new(
8395                project_root.path().to_path_buf(),
8396                "unit".to_string(),
8397                format!("session-{corr}"),
8398            ),
8399            consumer_identity: None,
8400            consumer_capabilities: None,
8401            role_versions,
8402            admission_facts: None,
8403            scope: None,
8404        })
8405        .unwrap();
8406        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8407    }
8408
8409    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8410        entries
8411            .iter()
8412            .map(|(role, version)| (role.to_string(), version.to_string()))
8413            .collect()
8414    }
8415
8416    fn route_open_frame_with_admission_facts(
8417        corr: u64,
8418        module_id: &str,
8419        project_root: ScratchDir,
8420        consumer_identity: Option<subc_control::ConsumerIdentity>,
8421        facts: Option<Value>,
8422    ) -> Frame {
8423        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8424            target: RouteTarget::ToolProvider {
8425                module_id: module_id.to_string(),
8426            },
8427            identity: BindIdentity::new(
8428                project_root.path().to_path_buf(),
8429                "unit".to_string(),
8430                format!("session-{corr}"),
8431            ),
8432            consumer_identity,
8433            consumer_capabilities: None,
8434            role_versions: None,
8435            admission_facts: facts,
8436            scope: None,
8437        })
8438        .unwrap();
8439        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8440    }
8441
8442    #[derive(Clone, Default)]
8443    struct EventCapture {
8444        events: Arc<Mutex<Vec<CapturedEvent>>>,
8445    }
8446
8447    #[derive(Clone, Debug)]
8448    struct CapturedEvent {
8449        target: String,
8450        level: tracing::Level,
8451        fields: BTreeMap<String, String>,
8452    }
8453
8454    impl EventCapture {
8455        fn events(&self) -> Vec<CapturedEvent> {
8456            self.events.lock().unwrap().clone()
8457        }
8458    }
8459
8460    impl<S> Layer<S> for EventCapture
8461    where
8462        S: Subscriber,
8463    {
8464        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8465            let mut visitor = EventFieldVisitor::default();
8466            event.record(&mut visitor);
8467            self.events.lock().unwrap().push(CapturedEvent {
8468                target: event.metadata().target().to_string(),
8469                level: *event.metadata().level(),
8470                fields: visitor.fields,
8471            });
8472        }
8473    }
8474
8475    #[derive(Default)]
8476    struct EventFieldVisitor {
8477        fields: BTreeMap<String, String>,
8478    }
8479
8480    impl Visit for EventFieldVisitor {
8481        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8482            self.fields
8483                .insert(field.name().to_string(), format!("{value:?}"));
8484        }
8485    }
8486
8487    fn health_response(corr: u64, status: HealthStatus) -> Frame {
8488        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8489            status,
8490            detail: Some("warming".to_string()),
8491            metrics: Some(json!({"queue_depth": 3})),
8492        })
8493        .unwrap();
8494        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8495    }
8496
8497    fn route_bind_ack(corr: u64) -> Frame {
8498        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8499        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8500    }
8501
8502    fn unique_project_root(label: &str) -> ScratchDir {
8503        ScratchDir::new(label)
8504    }
8505
8506    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8507        match parse_route_poll(frame) {
8508            ClientControlResponse::RoutePoll {
8509                status: None,
8510                live: Some(live),
8511                ..
8512            } => assert_eq!(live, expected_live),
8513            other => panic!("unexpected route.poll response: {other:?}"),
8514        }
8515    }
8516
8517    fn bind_liveness_route(
8518        registry: &Registry,
8519        forwarding: &ForwardingTable,
8520        module_id: &str,
8521    ) -> (RouteCtx, u16, u32) {
8522        let module_connection = ConnectionId::new(101);
8523        let client_connection = ConnectionId::new(202);
8524        let registration = registry
8525            .register_with_control_ops(
8526                manifest(module_id, PROTOCOL_VERSION),
8527                PROTOCOL_VERSION,
8528                module_connection,
8529                module_baseline_control_ops(),
8530            )
8531            .unwrap();
8532        let (module_tx, _module_rx) = mpsc::channel(8);
8533        let endpoint = forwarding
8534            .register_module_connection(
8535                module_connection,
8536                module_id.to_string(),
8537                PROTOCOL_VERSION,
8538                manifest_concurrency(&registration.manifest),
8539                FrameSink::new(module_tx),
8540            )
8541            .unwrap();
8542        let (client_ctx, _client_rx) = route_ctx(client_connection);
8543        let pending = forwarding
8544            .begin_route_bind_relay_for_test(
8545                client_connection,
8546                client_ctx.egress.clone(),
8547                1,
8548                module_id,
8549            )
8550            .unwrap();
8551        assert_eq!(pending.endpoint, endpoint);
8552        let route_channel = pending.client_channel;
8553        let route_epoch = pending.client_epoch;
8554        forwarding
8555            .complete_pending_relay(
8556                module_connection,
8557                pending.corr,
8558                RouteBindRelayOutcome::Accepted,
8559            )
8560            .unwrap();
8561        (client_ctx, route_channel, route_epoch)
8562    }
8563
8564    struct FakeProcessLiveness {
8565        live: Option<bool>,
8566    }
8567
8568    impl ModuleProcessLiveness for FakeProcessLiveness {
8569        fn process_live(&self, _module_id: &str) -> Option<bool> {
8570            self.live
8571        }
8572    }
8573
8574    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8575    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8576    {
8577        let registry = Arc::new(Registry::default());
8578        let supervisor_handle = SupervisorHandle::new();
8579        let supervisor = Supervisor::new_for_test(
8580            Arc::clone(&registry),
8581            RestartPolicy::new(1, Duration::from_millis(10)),
8582        )
8583        .with_handle(supervisor_handle.clone());
8584        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8585        let module = supervisor
8586            .spawn(ModuleSpec {
8587                module_id: "stderr-tail-wire".to_string(),
8588                program: fake_aft_stub_path(),
8589                args: Vec::new(),
8590                env: vec![
8591                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8592                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8593                ],
8594                reserved: false,
8595                reserved_prefixes: Vec::new(),
8596                protocol: ModuleProtocol::Subc,
8597                overlap: Default::default(),
8598            })
8599            .unwrap();
8600
8601        let deadline = Instant::now() + Duration::from_secs(5);
8602        loop {
8603            let tail = module.stderr_tail(None, None);
8604            if tail
8605                .entries
8606                .iter()
8607                .any(|entry| matches!(entry, TailEntry::ProcessStart))
8608                && tail.entries.iter().any(|entry| {
8609                    matches!(
8610                        entry,
8611                        TailEntry::Line {
8612                            truncated: true,
8613                            ..
8614                        }
8615                    )
8616                })
8617            {
8618                break;
8619            }
8620            assert!(
8621                Instant::now() < deadline,
8622                "module did not produce a truncated line and restart boundary: {tail:?}"
8623            );
8624            sleep(Duration::from_millis(10)).await;
8625        }
8626
8627        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8628        let request = ClientControlRequest::SupervisorStderrTail {
8629            module_id: "stderr-tail-wire".to_string(),
8630            max_lines: None,
8631            max_bytes: None,
8632        };
8633        let frame = Frame::build(
8634            FrameType::Request,
8635            control_flags(),
8636            0,
8637            0,
8638            1,
8639            serde_json::to_vec(&request).unwrap(),
8640        )
8641        .unwrap();
8642        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8643        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8644        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8645            serde_json::from_slice(&responses[0].body).unwrap()
8646        else {
8647            panic!("expected supervisor.stderr_tail response");
8648        };
8649
8650        assert!(
8651            tail.entries
8652                .iter()
8653                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8654            "the control response lost the restart boundary"
8655        );
8656        let Some(StderrTailEntry::Line {
8657            text,
8658            truncated,
8659            at_ms,
8660        }) = tail.entries.iter().find(|entry| {
8661            matches!(
8662                entry,
8663                StderrTailEntry::Line {
8664                    truncated: true,
8665                    ..
8666                }
8667            )
8668        })
8669        else {
8670            panic!("the control response lost the truncated line");
8671        };
8672        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8673        assert!(*truncated);
8674        assert!(
8675            at_ms.is_some(),
8676            "the control response lost the line's capture time"
8677        );
8678    }
8679
8680    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
8681    /// read done on the worker thread would stall every other task until it
8682    /// finished; the read must run off the worker so this test's own task keeps
8683    /// running while the read is paused.
8684    #[tokio::test(flavor = "current_thread")]
8685    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8686        let dir = ScratchDir::new("terminals-off-worker");
8687        let journal_path = dir.join("terminals.jsonl");
8688        let registry = Arc::new(Registry::default());
8689        let supervisor_handle = SupervisorHandle::new();
8690        let supervisor =
8691            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8692                .with_handle(supervisor_handle.clone())
8693                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8694        let module = supervisor
8695            .spawn(ModuleSpec {
8696                module_id: "terminal-off-worker".to_string(),
8697                program: fake_aft_stub_path(),
8698                args: Vec::new(),
8699                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8700                reserved: false,
8701                reserved_prefixes: Vec::new(),
8702                protocol: ModuleProtocol::Subc,
8703                overlap: Default::default(),
8704            })
8705            .unwrap();
8706        let deadline = Instant::now() + Duration::from_secs(5);
8707        while module.terminal_history().entries.len() != 2 {
8708            assert!(Instant::now() < deadline, "module did not record two exits");
8709            sleep(Duration::from_millis(10)).await;
8710        }
8711
8712        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8713        let handler =
8714            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
8715        let frame = Frame::build(
8716            FrameType::Request,
8717            control_flags(),
8718            0,
8719            0,
8720            1,
8721            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8722                module_id: "terminal-off-worker".to_string(),
8723            })
8724            .unwrap(),
8725        )
8726        .unwrap();
8727        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8728        let spawned_at = std::time::Instant::now();
8729        let read = tokio::spawn({
8730            let handler = Arc::clone(&handler);
8731            async move { handler.handle_control_frame(&ctx, frame).await }
8732        });
8733        // Waiting for the pause from a blocking thread keeps this task pending,
8734        // so the runtime's single worker is free to run the read task.
8735        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8736            .await
8737            .unwrap()
8738            .expect("the history read reached its pause");
8739        let elapsed = spawned_at.elapsed();
8740        assert!(
8741            elapsed < Duration::from_secs(2) && !read.is_finished(),
8742            "this task could not run while the history read was paused \
8743             (resumed after {elapsed:?}, read finished: {})",
8744            read.is_finished()
8745        );
8746
8747        drop(release);
8748        let responses = read.await.unwrap().unwrap();
8749        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8750        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8751            panic!("expected supervisor.terminals response");
8752        };
8753        assert_eq!(terminals.entries.len(), 2);
8754        assert_eq!(terminals.journal_skipped_lines, 0);
8755        assert_eq!(terminals.journal_read_errors, 0);
8756    }
8757
8758    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8759    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8760        let registry = Arc::new(Registry::default());
8761        let supervisor_handle = SupervisorHandle::new();
8762        let supervisor =
8763            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8764                .with_handle(supervisor_handle.clone());
8765        let module = supervisor
8766            .spawn(ModuleSpec {
8767                module_id: "terminal-golden".to_string(),
8768                program: fake_aft_stub_path(),
8769                args: Vec::new(),
8770                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8771                reserved: false,
8772                reserved_prefixes: Vec::new(),
8773                protocol: ModuleProtocol::Subc,
8774                overlap: Default::default(),
8775            })
8776            .unwrap();
8777
8778        let deadline = Instant::now() + Duration::from_secs(5);
8779        while module.terminal_history().entries.len() != 2 {
8780            assert!(
8781                Instant::now() < deadline,
8782                "module did not retain two terminal exits: {:?}",
8783                module.terminal_history()
8784            );
8785            sleep(Duration::from_millis(10)).await;
8786        }
8787
8788        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8789        let request = ClientControlRequest::SupervisorTerminals {
8790            module_id: "terminal-golden".to_string(),
8791        };
8792        let frame = Frame::build(
8793            FrameType::Request,
8794            control_flags(),
8795            0,
8796            0,
8797            1,
8798            serde_json::to_vec(&request).unwrap(),
8799        )
8800        .unwrap();
8801        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8802        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8803        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8804        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8805            panic!("expected supervisor.terminals response");
8806        };
8807        assert_eq!(terminals.entries.len(), 2);
8808        assert_eq!(terminals.dropped, 0);
8809
8810        let mut rendered = serde_json::to_value(response).unwrap();
8811        // Wall-clock fields are the observation contract, but not stable fixture
8812        // bytes; normalize only them after the real handler has shaped the response.
8813        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8814        for (index, entry) in rendered["entries"]
8815            .as_array_mut()
8816            .expect("terminal response entries array")
8817            .iter_mut()
8818            .enumerate()
8819        {
8820            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8821        }
8822
8823        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8824            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8825        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8826        if std::env::var_os("UPDATE_GOLDEN").is_some() {
8827            std::fs::write(&golden_path, &serialized).unwrap();
8828        }
8829        let expected: Value =
8830            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8831        assert_eq!(rendered, expected);
8832    }
8833
8834    #[test]
8835    fn hello_registers_manifest_and_returns_ack() {
8836        let registry = Arc::new(Registry::default());
8837        let handler = ControlHandler::new(Arc::clone(&registry));
8838        let conn = ConnectionId::new(1);
8839
8840        let responses = handler
8841            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8842            .unwrap();
8843
8844        assert_eq!(responses.len(), 1);
8845        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8846        assert_eq!(responses[0].header.channel, 0);
8847        assert_eq!(responses[0].header.corr, 7);
8848        let ack = parse_ack(&responses[0]);
8849        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8850        assert!(ack
8851            .subc_capabilities
8852            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8853        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8854        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8855        assert!(ack
8856            .subc_ops
8857            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8858        assert!(ack
8859            .subc_ops
8860            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8861
8862        let registration = registry.get_module("aft").unwrap().unwrap();
8863        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8864        assert_eq!(registration.state, ChannelState::Active);
8865        assert_eq!(registration.connection_id, conn);
8866        assert_eq!(registration.control_ops, module_baseline_control_ops());
8867    }
8868
8869    #[test]
8870    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8871        let invalid_identifiers = [
8872            ("case_change", "credentials-Provider/v1"),
8873            ("leading_zero", "credentials-provider/v01"),
8874            ("trailing_hyphen", "credentials-provider-/v1"),
8875            ("consecutive_hyphens", "credentials--provider/v1"),
8876            ("uppercase", "Credentials-provider/v1"),
8877            ("missing_v", "credentials-provider/1"),
8878            ("whitespace", "credentials provider/v1"),
8879            ("zero_version", "credentials-provider/v0"),
8880            ("out_of_range_version", "credentials-provider/v4294967296"),
8881            (
8882                "overlength_name",
8883                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8884            ),
8885        ];
8886        let mut cases = invalid_identifiers
8887            .into_iter()
8888            .map(|(name, identifier)| {
8889                (
8890                    format!("identifier_{name}"),
8891                    "capabilities.provides[0]".to_string(),
8892                    identifier.to_string(),
8893                    json!({ "provides": [identifier] }),
8894                    None,
8895                )
8896            })
8897            .collect::<Vec<_>>();
8898        cases.extend([
8899            (
8900                "unknown_need".to_string(),
8901                "capabilities.requires[0].need".to_string(),
8902                "deferred".to_string(),
8903                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8904                None,
8905            ),
8906            (
8907                "duplicate_provides".to_string(),
8908                "capabilities.provides[1]".to_string(),
8909                "credentials-provider/v1".to_string(),
8910                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8911                None,
8912            ),
8913            (
8914                "duplicate_must_never_reach".to_string(),
8915                "capabilities.must_never_reach[1]".to_string(),
8916                "credentials-provider/v1".to_string(),
8917                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8918                None,
8919            ),
8920            (
8921                "duplicate_requires_same_need".to_string(),
8922                "capabilities.requires[1]".to_string(),
8923                "credentials-provider/v1".to_string(),
8924                json!({ "requires": [
8925                    { "capability": "credentials-provider/v1", "need": "required" },
8926                    { "capability": "credentials-provider/v1", "need": "required" }
8927                ] }),
8928                None,
8929            ),
8930            (
8931                "duplicate_requires_conflicting_need".to_string(),
8932                "capabilities.requires[1]".to_string(),
8933                "credentials-provider/v1".to_string(),
8934                json!({ "requires": [
8935                    { "capability": "credentials-provider/v1", "need": "required" },
8936                    { "capability": "credentials-provider/v1", "need": "optional" }
8937                ] }),
8938                None,
8939            ),
8940            (
8941                "capabilities_root_pointer".to_string(),
8942                "runtime_computed[0]".to_string(),
8943                "/capabilities".to_string(),
8944                json!({}),
8945                Some(json!(["/capabilities"])),
8946            ),
8947            (
8948                "capabilities_descendant_pointer".to_string(),
8949                "runtime_computed[0]".to_string(),
8950                "/capabilities/provides".to_string(),
8951                json!({}),
8952                Some(json!(["/capabilities/provides"])),
8953            ),
8954            (
8955                "malformed_pointer_without_leading_slash".to_string(),
8956                "runtime_computed[0]".to_string(),
8957                "capabilities".to_string(),
8958                json!({}),
8959                Some(json!(["capabilities"])),
8960            ),
8961            (
8962                "malformed_pointer_escape".to_string(),
8963                "runtime_computed[0]".to_string(),
8964                "/roles/~2/tools".to_string(),
8965                json!({}),
8966                Some(json!(["/roles/~2/tools"])),
8967            ),
8968            (
8969                "unknown_capabilities_field".to_string(),
8970                "capabilities.future".to_string(),
8971                "<array>".to_string(),
8972                json!({ "future": [] }),
8973                None,
8974            ),
8975        ]);
8976
8977        for (index, (name, field, value, capabilities, runtime_computed)) in
8978            cases.into_iter().enumerate()
8979        {
8980            let registry = Arc::new(Registry::default());
8981            let handler = ControlHandler::new(Arc::clone(&registry));
8982            let response = handler
8983                .handle_control(
8984                    ConnectionId::new((index + 1) as u64),
8985                    capability_grammar_hello_frame(
8986                        capabilities,
8987                        runtime_computed,
8988                        index as u64 + 1,
8989                    ),
8990                )
8991                .expect("invalid HELLO returns a refusal");
8992
8993            assert_eq!(response.len(), 1, "{name} must emit one refusal");
8994            let error = parse_error(&response[0]);
8995            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8996            let message = error["message"]
8997                .as_str()
8998                .expect("error message is a string");
8999            assert!(
9000                message.contains(&field),
9001                "{name}: field missing from {message}"
9002            );
9003            assert!(
9004                message.contains(&value),
9005                "{name}: value missing from {message}"
9006            );
9007            assert_eq!(
9008                registry
9009                    .active_registration_count()
9010                    .expect("registry reads"),
9011                0,
9012                "{name}: refused HELLO must not create a catalog entry"
9013            );
9014        }
9015    }
9016
9017    #[test]
9018    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9019        let registry = Arc::new(Registry::default());
9020        let handler = ControlHandler::new(Arc::clone(&registry));
9021        let capabilities = json!({
9022            "provides": ["credentials-provider/v1"],
9023            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9024            "must_never_reach": ["federation-transport/v1"]
9025        });
9026        let response = handler
9027            .handle_control(
9028                ConnectionId::new(99),
9029                capability_grammar_hello_frame(
9030                    capabilities.clone(),
9031                    Some(json!(["/roles/0/tools"])),
9032                    99,
9033                ),
9034            )
9035            .expect("valid HELLO registers");
9036        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9037
9038        let request = Frame::build(
9039            FrameType::Request,
9040            control_flags(),
9041            0,
9042            0,
9043            100,
9044            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9045                .expect("catalog request serializes"),
9046        )
9047        .expect("catalog request frame builds");
9048        let response = handler
9049            .handle_catalog_list(request, None)
9050            .expect("catalog list succeeds");
9051        let ClientControlResponse::CatalogList { modules, .. } =
9052            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9053        else {
9054            panic!("catalog request must return catalog.list");
9055        };
9056        assert_eq!(modules.len(), 1);
9057        assert_eq!(
9058            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9059            capabilities
9060        );
9061    }
9062
9063    #[test]
9064    fn catalog_list_mirrors_management_operation_description() {
9065        let registry = Arc::new(Registry::default());
9066        let handler = ControlHandler::new(Arc::clone(&registry));
9067        let description = "List managed records and return their identifiers and metadata.";
9068        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9069        manifest.provides = vec![ProviderRole::ManagementSurface {
9070            operations: vec![ManagementOperation {
9071                name: "records.list".to_string(),
9072                kind: ManagementOperationKind::Query,
9073                description: Some(description.to_string()),
9074            }],
9075            config_schema: json!({"type": "object"}),
9076            observability: vec![ObservabilitySurface {
9077                name: "records.stats".to_string(),
9078                kind: ObservabilityKind::Snapshot,
9079            }],
9080            identity_scope: vec![IdentityScope::Project],
9081            concurrency: Concurrency::ModuleManaged,
9082        }];
9083        registry
9084            .register_with_control_ops(
9085                manifest,
9086                PROTOCOL_VERSION,
9087                ConnectionId::new(99),
9088                Vec::new(),
9089            )
9090            .expect("described management manifest registers");
9091
9092        let request = Frame::build(
9093            FrameType::Request,
9094            control_flags(),
9095            0,
9096            0,
9097            100,
9098            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9099                .expect("catalog request serializes"),
9100        )
9101        .expect("catalog request frame builds");
9102        let response = handler
9103            .handle_catalog_list(request, None)
9104            .expect("catalog list succeeds");
9105        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9106        assert_eq!(
9107            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9108            "catalog.list must preserve the declared operation description verbatim"
9109        );
9110    }
9111
9112    #[test]
9113    fn catalog_list_mirrors_registered_operation_delivery() {
9114        let registry = Arc::new(Registry::default());
9115        let handler = ControlHandler::new(Arc::clone(&registry));
9116        let delivery =
9117            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9118        let mut body = serde_json::to_value(ModuleHelloBody {
9119            manifest: manifest("watch-management", PROTOCOL_VERSION),
9120            protocol_ver: PROTOCOL_VERSION,
9121            control_ops: None,
9122            launch_nonce: None,
9123        })
9124        .unwrap();
9125        body["manifest"]["provides"] = json!([{
9126            "role": "management_surface",
9127            "operations": [
9128                {"name": "status", "kind": "query"},
9129                {"name": "future", "kind": "query"},
9130                {"name": "reset", "kind": "mutate"}
9131            ],
9132            "config_schema": {}, "observability": [], "identity_scope": []
9133        }]);
9134        body["manifest"]["operation_delivery"] = delivery.clone();
9135        let hello = Frame::build(
9136            FrameType::Hello,
9137            control_flags(),
9138            0,
9139            0,
9140            99,
9141            serde_json::to_vec(&body).unwrap(),
9142        )
9143        .unwrap();
9144        let response = handler
9145            .handle_control(ConnectionId::new(99), hello)
9146            .unwrap();
9147        assert_eq!(response[0].header.ty, FrameType::HelloAck);
9148
9149        let (_, registrations) = registry.list_modules().unwrap();
9150        let registered = &registrations[0].manifest;
9151        assert!(registered.is_watch_operation("status"));
9152        assert!(!registered.is_watch_operation("future"));
9153        assert!(!registered.is_watch_operation("reset"));
9154        assert!(!registered.is_watch_operation("missing"));
9155
9156        let request = Frame::build(
9157            FrameType::Request,
9158            control_flags(),
9159            0,
9160            0,
9161            100,
9162            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9163        )
9164        .unwrap();
9165        let response = handler.handle_catalog_list(request, None).unwrap();
9166        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9167        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9168        let ClientControlResponse::CatalogList { modules, .. } =
9169            serde_json::from_slice(&response[0].body).unwrap()
9170        else {
9171            panic!("expected catalog.list");
9172        };
9173        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9174    }
9175
9176    #[test]
9177    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9178        let registry = Arc::new(Registry::default());
9179        let supervisor = SupervisorHandle::new();
9180        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9181        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9182        let descriptor = json!({
9183            "schema": "ckios.richui.manifest/1",
9184            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9185            "bytes": 4096,
9186            "chunks": 4
9187        });
9188        let connection = ConnectionId::new(201);
9189        let hello = handler
9190            .handle_control(
9191                connection,
9192                hello_frame_with_rich_ui(
9193                    "vault",
9194                    PROTOCOL_VERSION,
9195                    201,
9196                    Some("the-real-nonce"),
9197                    descriptor.clone(),
9198                ),
9199            )
9200            .unwrap();
9201        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9202
9203        let request =
9204            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9205        let provides = registry
9206            .get_module_by_connection(connection)
9207            .unwrap()
9208            .unwrap()
9209            .manifest
9210            .provides;
9211        handler
9212            .handle_catalog_update(connection, request, provides, None, Some(false))
9213            .unwrap();
9214
9215        let request = Frame::build(
9216            FrameType::Request,
9217            control_flags(),
9218            0,
9219            0,
9220            203,
9221            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9222        )
9223        .unwrap();
9224        let response = handler.handle_catalog_list(request, None).unwrap();
9225        let ClientControlResponse::CatalogList { modules, .. } =
9226            serde_json::from_slice(&response[0].body).unwrap()
9227        else {
9228            panic!("expected catalog.list");
9229        };
9230        assert_eq!(
9231            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9232            descriptor
9233        );
9234        assert!(modules[0].nonce_verified);
9235    }
9236
9237    #[test]
9238    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9239        let registry = Arc::new(Registry::default());
9240        let handler = ControlHandler::new(registry.clone());
9241        let descriptor = json!({
9242            "schema": "ckios.richui.manifest/1",
9243            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9244            "bytes": 128,
9245            "chunks": 2
9246        });
9247        let malformed = [
9248            ("schema", json!("")),
9249            ("schema", json!("a".repeat(65))),
9250            ("schema", json!("Bad schema")),
9251            (
9252                "manifest_digest",
9253                json!(format!("sha256:{}", "A".repeat(64))),
9254            ),
9255            ("manifest_digest", json!("sha256:abc")),
9256            ("bytes", json!(0)),
9257            ("bytes", json!(16_777_217)),
9258            ("bytes", json!("128")),
9259            ("chunks", json!(0)),
9260            ("chunks", json!(1025)),
9261            ("chunks", json!("2")),
9262        ];
9263        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9264            let mut invalid = descriptor.clone();
9265            invalid[field] = invalid_value;
9266            let response = handler
9267                .handle_control(
9268                    ConnectionId::new(210 + index as u64),
9269                    hello_frame_with_rich_ui(
9270                        &format!("bad-rich-ui-{index}"),
9271                        PROTOCOL_VERSION,
9272                        210 + index as u64,
9273                        None,
9274                        invalid,
9275                    ),
9276                )
9277                .unwrap();
9278            assert_eq!(response[0].header.ty, FrameType::Error);
9279            let error = parse_error(&response[0]);
9280            assert_eq!(error["code"], "invalid_manifest");
9281            assert!(error["message"]
9282                .as_str()
9283                .unwrap()
9284                .contains(&format!("rich_ui.{field}")));
9285            assert!(registry
9286                .get_module(&format!("bad-rich-ui-{index}"))
9287                .unwrap()
9288                .is_none());
9289        }
9290    }
9291
9292    #[test]
9293    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9294        let registry = Arc::new(Registry::default());
9295        let handler = ControlHandler::new(Arc::clone(&registry));
9296        let connection = ConnectionId::new(219);
9297        registry
9298            .register_with_control_ops(
9299                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9300                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9301                        "bad schema",
9302                        format!("sha256:{}", "a".repeat(64)),
9303                        128,
9304                        2,
9305                    )))
9306                    .build(),
9307                PROTOCOL_VERSION,
9308                connection,
9309                Vec::new(),
9310            )
9311            .unwrap();
9312        let request =
9313            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9314        let response = handler
9315            .handle_catalog_update(connection, request, Vec::new(), None, None)
9316            .unwrap();
9317        assert_eq!(response[0].header.ty, FrameType::Error);
9318        let error = parse_error(&response[0]);
9319        assert_eq!(error["code"], "invalid_manifest");
9320        assert!(error["message"]
9321            .as_str()
9322            .unwrap()
9323            .contains("rich_ui.schema"));
9324    }
9325
9326    #[test]
9327    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9328        let registry = Arc::new(Registry::default());
9329        let supervisor = SupervisorHandle::new();
9330        supervisor.open_swap("provider", "candidate-nonce".to_string());
9331        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9332        let connection = ConnectionId::new(218);
9333        let hello = handler
9334            .handle_control(
9335                connection,
9336                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9337            )
9338            .unwrap();
9339        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9340        assert!(registry.list_modules().unwrap().1.is_empty());
9341
9342        registry.promote_candidate("provider").unwrap().unwrap();
9343        let request = Frame::build(
9344            FrameType::Request,
9345            control_flags(),
9346            0,
9347            0,
9348            219,
9349            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9350        )
9351        .unwrap();
9352        let response = handler.handle_catalog_list(request, None).unwrap();
9353        let ClientControlResponse::CatalogList { modules, .. } =
9354            serde_json::from_slice(&response[0].body).unwrap()
9355        else {
9356            panic!("expected catalog.list");
9357        };
9358        assert_eq!(modules.len(), 1);
9359        assert!(modules[0].nonce_verified);
9360    }
9361
9362    #[test]
9363    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
9364        let registry = Arc::new(Registry::default());
9365        let handler = ControlHandler::new(Arc::clone(&registry));
9366        let hello = Frame::build(
9367            FrameType::Hello,
9368            control_flags(),
9369            0,
9370            0,
9371            220,
9372            serde_json::to_vec(&ModuleHelloBody {
9373                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
9374                protocol_ver: PROTOCOL_VERSION,
9375                control_ops: None,
9376                launch_nonce: Some("unverified".to_string()),
9377            })
9378            .unwrap(),
9379        )
9380        .unwrap();
9381        assert_eq!(
9382            handler
9383                .handle_control(ConnectionId::new(220), hello)
9384                .unwrap()[0]
9385                .header
9386                .ty,
9387            FrameType::HelloAck
9388        );
9389        let request = Frame::build(
9390            FrameType::Request,
9391            control_flags(),
9392            0,
9393            0,
9394            221,
9395            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9396        )
9397        .unwrap();
9398        let response = handler.handle_catalog_list(request, None).unwrap();
9399        let ClientControlResponse::CatalogList { modules, .. } =
9400            serde_json::from_slice(&response[0].body).unwrap()
9401        else {
9402            panic!("expected catalog.list");
9403        };
9404        assert_eq!(
9405            serde_json::to_vec(&modules[0]).unwrap(),
9406            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
9407        );
9408        assert!(!modules[0].nonce_verified);
9409    }
9410
9411    #[test]
9412    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9413        let registry = Arc::new(Registry::default());
9414        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9415            [("vault".to_string(), true), ("squatter".to_string(), true)],
9416            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9417        );
9418        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9419        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9420            provides: vec!["credentials-provider/v1".to_string()],
9421            requires: Vec::new(),
9422            must_never_reach: Vec::new(),
9423        });
9424        let frame = Frame::build(
9425            FrameType::Hello,
9426            control_flags(),
9427            0,
9428            0,
9429            77,
9430            serde_json::to_vec(&ModuleHelloBody {
9431                manifest: squatter,
9432                protocol_ver: PROTOCOL_VERSION,
9433                control_ops: None,
9434                launch_nonce: None,
9435            })
9436            .expect("HELLO serializes"),
9437        )
9438        .expect("HELLO frame builds");
9439        let response = handler
9440            .handle_control(ConnectionId::new(77), frame)
9441            .expect("reserved claim receives a typed refusal");
9442        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9443        assert_eq!(
9444            registry
9445                .active_registration_count()
9446                .expect("registry reads"),
9447            0,
9448            "a reserved capability refusal must not leave a catalog entry"
9449        );
9450    }
9451
9452    #[test]
9453    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9454        for settlement in ["timeout", "inconclusive", "drop"] {
9455            let breakers = RouteBindBreakers::default();
9456            let RouteBindAdmission::Admitted {
9457                guard: mut old,
9458                probe: false,
9459            } = breakers.admit("prov")
9460            else {
9461                panic!("ordinary relay admitted")
9462            };
9463            let RouteBindAdmission::Admitted {
9464                guard: mut opener, ..
9465            } = breakers.admit("prov")
9466            else {
9467                panic!("second relay admitted")
9468            };
9469            assert!(
9470                !opener
9471                    .record_timeout(1, Duration::ZERO)
9472                    .unwrap()
9473                    .reopened_after_probe
9474            );
9475            let RouteBindAdmission::Admitted {
9476                guard: mut probe,
9477                probe: true,
9478            } = breakers.admit("prov")
9479            else {
9480                panic!("one cooldown probe admitted")
9481            };
9482            match settlement {
9483                "timeout" => assert!(
9484                    !old.record_timeout(1, Duration::ZERO)
9485                        .unwrap()
9486                        .reopened_after_probe
9487                ),
9488                "inconclusive" => old.record_inconclusive(),
9489                "drop" => drop(old),
9490                _ => unreachable!(),
9491            }
9492            assert!(
9493                matches!(
9494                    breakers.admit("prov"),
9495                    RouteBindAdmission::Refused {
9496                        probe_in_flight: true,
9497                        ..
9498                    }
9499                ),
9500                "{settlement} of a pre-open relay cannot release the real probe"
9501            );
9502            assert!(
9503                probe
9504                    .record_timeout(1, Duration::ZERO)
9505                    .unwrap()
9506                    .reopened_after_probe
9507            );
9508            assert!(matches!(
9509                breakers.admit("prov"),
9510                RouteBindAdmission::Admitted { probe: true, .. }
9511            ));
9512        }
9513        let breakers = RouteBindBreakers::default();
9514        let admit = || match breakers.admit("prov") {
9515            RouteBindAdmission::Admitted { guard, .. } => guard,
9516            _ => panic!("relay admitted"),
9517        };
9518        admit().record_timeout(1, Duration::ZERO);
9519        let mut old_probe = admit();
9520        breakers.reset_for_new_module_connection("prov");
9521        admit().record_timeout(1, Duration::ZERO);
9522        let _new_probe = admit();
9523        old_probe.record_inconclusive();
9524        assert!(matches!(
9525            breakers.admit("prov"),
9526            RouteBindAdmission::Refused {
9527                probe_in_flight: true,
9528                ..
9529            }
9530        ));
9531    }
9532
9533    #[tokio::test]
9534    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9535        for candidate in [false, true] {
9536            let registry = Arc::new(Registry::default());
9537            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9538                [("vault".to_string(), true), ("squatter".to_string(), true)],
9539                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9540            );
9541            let conn = ConnectionId::new(77);
9542            let (ctx, mut rx) = route_ctx(conn);
9543            let initial = capability_manifest("squatter", &[], &[]);
9544            if candidate {
9545                registry
9546                    .register_candidate_with_control_ops(
9547                        initial.clone(),
9548                        PROTOCOL_VERSION,
9549                        conn,
9550                        module_baseline_control_ops(),
9551                    )
9552                    .unwrap();
9553                handler
9554                    .forwarding
9555                    .register_candidate_module_connection(
9556                        conn,
9557                        "squatter".to_string(),
9558                        PROTOCOL_VERSION,
9559                        manifest_concurrency(&initial),
9560                        ctx.egress.clone(),
9561                    )
9562                    .unwrap();
9563            } else {
9564                hello_via_sink(
9565                    &handler,
9566                    &ctx,
9567                    &mut rx,
9568                    hello_frame_with_manifest(initial.clone(), 1),
9569                )
9570                .await;
9571            }
9572            let response = handler
9573                .handle_control_frame(
9574                    &ctx,
9575                    catalog_update_with_capabilities_frame(
9576                        2,
9577                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
9578                            .capabilities
9579                            .unwrap(),
9580                    ),
9581                )
9582                .await
9583                .unwrap();
9584            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9585            assert_eq!(
9586                registry
9587                    .get_module_by_connection(conn)
9588                    .unwrap()
9589                    .unwrap()
9590                    .manifest,
9591                initial
9592            );
9593        }
9594    }
9595
9596    #[test]
9597    fn server_describe_surfaces_required_capability_verdict_fields() {
9598        let registry = Arc::new(Registry::default());
9599        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9600            [
9601                ("consumer".to_string(), true),
9602                ("provider".to_string(), false),
9603            ],
9604            BTreeMap::new(),
9605        );
9606        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9607        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9608            provides: Vec::new(),
9609            requires: vec![subc_protocol::manifest::CapabilityRequirement {
9610                capability: "credentials-provider/v1".to_string(),
9611                need: subc_protocol::manifest::CapabilityNeed::Required,
9612            }],
9613            must_never_reach: Vec::new(),
9614        });
9615        let hello = Frame::build(
9616            FrameType::Hello,
9617            control_flags(),
9618            0,
9619            0,
9620            78,
9621            serde_json::to_vec(&ModuleHelloBody {
9622                manifest: consumer,
9623                protocol_ver: PROTOCOL_VERSION,
9624                control_ops: None,
9625                launch_nonce: None,
9626            })
9627            .expect("HELLO serializes"),
9628        )
9629        .expect("HELLO frame builds");
9630        handler
9631            .handle_control(ConnectionId::new(78), hello)
9632            .expect("consumer registers");
9633        let describe = Frame::build(
9634            FrameType::Request,
9635            control_flags(),
9636            0,
9637            0,
9638            79,
9639            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9640                .expect("request serializes"),
9641        )
9642        .expect("describe frame builds");
9643        let response = handler
9644            .handle_server_describe(describe)
9645            .expect("server.describe succeeds");
9646        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9647        let requirement = &rendered["capability_requirements"][0];
9648        assert_eq!(requirement["consumer"], "consumer");
9649        assert_eq!(requirement["verdict"], "never_provided");
9650        assert_eq!(requirement["episode_seq"], 1);
9651        assert_eq!(requirement["config_satisfiable"], false);
9652        assert_eq!(requirement["runtime_available"], false);
9653        assert!(requirement["detail"]
9654            .as_str()
9655            .expect("detail string")
9656            .contains("credentials-provider/v1"));
9657    }
9658
9659    /// Register a consumer that requires `project-identity/v1` and the provider
9660    /// of it, optionally mark daemon shutdown as begun, then deregister the
9661    /// provider as its connection closes. Returns the `capability.requirement`
9662    /// events that deregistration logged and the verdict the evaluator then
9663    /// holds for the requirement (the one `route.open` refusals read).
9664    fn deregister_required_provider(
9665        daemon_shutdown_begun: bool,
9666    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
9667        const CAPABILITY: &str = "project-identity/v1";
9668        let registry = Arc::new(Registry::default());
9669        let daemon_shutdown = DaemonShutdownFlag::default();
9670        let handler = ControlHandler::new(Arc::clone(&registry))
9671            .with_capability_config(
9672                [
9673                    ("consumer".to_string(), true),
9674                    ("provider".to_string(), true),
9675                ],
9676                BTreeMap::new(),
9677            )
9678            .with_daemon_shutdown_flag(daemon_shutdown.clone());
9679        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9680        consumer.capabilities = Some(CapabilityDeclarations {
9681            provides: Vec::new(),
9682            requires: vec![subc_protocol::manifest::CapabilityRequirement {
9683                capability: CAPABILITY.to_string(),
9684                need: subc_protocol::manifest::CapabilityNeed::Required,
9685            }],
9686            must_never_reach: Vec::new(),
9687        });
9688        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
9689        let provider_conn = ConnectionId::new(182);
9690        for (conn, manifest, corr) in [
9691            (ConnectionId::new(181), consumer, 181),
9692            (provider_conn, provider, 182),
9693        ] {
9694            let hello = handler
9695                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
9696                .expect("module registers");
9697            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9698        }
9699        assert_eq!(
9700            handler.capability_evaluator.verdict("consumer", CAPABILITY),
9701            Some(CapabilityVerdict::Provided),
9702            "the registered provider must satisfy the requirement first"
9703        );
9704        if daemon_shutdown_begun {
9705            daemon_shutdown.set_for_test();
9706        }
9707
9708        let capture = EventCapture::default();
9709        let _subscriber =
9710            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9711        handler
9712            .cleanup_connection(provider_conn)
9713            .expect("provider deregisters");
9714        let events = capture
9715            .events()
9716            .into_iter()
9717            .filter(|event| {
9718                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
9719            })
9720            .collect();
9721        (
9722            events,
9723            handler.capability_evaluator.verdict("consumer", CAPABILITY),
9724        )
9725    }
9726
9727    #[test]
9728    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
9729        let (events, verdict) = deregister_required_provider(true);
9730        assert_eq!(
9731            verdict,
9732            Some(CapabilityVerdict::NeverProvided),
9733            "shutdown must not change the verdict route.open refusals read"
9734        );
9735        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9736        let event = &events[0];
9737        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
9738        assert_eq!(
9739            event.fields.get("cause").map(String::as_str),
9740            Some("\"daemon_shutdown\""),
9741            "{event:?}"
9742        );
9743        assert_eq!(
9744            event.fields.get("verdict").map(String::as_str),
9745            Some("\"never_provided\""),
9746            "{event:?}"
9747        );
9748    }
9749
9750    #[test]
9751    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
9752        let (events, verdict) = deregister_required_provider(false);
9753        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
9754        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9755        let event = &events[0];
9756        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
9757        assert_eq!(event.fields.get("cause"), None, "{event:?}");
9758        assert_eq!(
9759            event.fields.get("verdict").map(String::as_str),
9760            Some("\"never_provided\""),
9761            "{event:?}"
9762        );
9763    }
9764
9765    #[test]
9766    fn catalog_list_omits_capabilities_for_legacy_manifest() {
9767        let registry = Arc::new(Registry::default());
9768        let handler = ControlHandler::new(Arc::clone(&registry));
9769        let hello = handler
9770            .handle_control(
9771                ConnectionId::new(101),
9772                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9773            )
9774            .expect("legacy HELLO registers");
9775        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9776
9777        let request = Frame::build(
9778            FrameType::Request,
9779            control_flags(),
9780            0,
9781            0,
9782            102,
9783            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9784                .expect("catalog request serializes"),
9785        )
9786        .expect("catalog request frame builds");
9787        let response = handler
9788            .handle_catalog_list(request, None)
9789            .expect("catalog list succeeds");
9790        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9791        assert!(
9792            body["modules"][0].get("capabilities").is_none(),
9793            "legacy manifest must retain an absent capabilities field on catalog.list"
9794        );
9795    }
9796
9797    #[test]
9798    fn hello_ack_omits_storage_when_no_storage_config() {
9799        let registry = Arc::new(Registry::default());
9800        let handler = ControlHandler::new(Arc::clone(&registry));
9801        let responses = handler
9802            .handle_control(
9803                ConnectionId::new(1),
9804                hello_frame("aft", PROTOCOL_VERSION, 7),
9805            )
9806            .unwrap();
9807        let ack = parse_ack(&responses[0]);
9808        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9809        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9810    }
9811
9812    #[tokio::test]
9813    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9814        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9815        let registry = Arc::new(Registry::default());
9816        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
9817        let responses = handler
9818            .handle_control(
9819                ConnectionId::new(1),
9820                hello_frame("aft", PROTOCOL_VERSION, 7),
9821            )
9822            .unwrap();
9823        let ack = parse_ack(&responses[0]);
9824        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9825
9826        let described = handler
9827            .handle_control_frame(
9828                &route_ctx(ConnectionId::new(2)).0,
9829                Frame::build(
9830                    FrameType::Request,
9831                    control_flags(),
9832                    0,
9833                    0,
9834                    9,
9835                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9836                )
9837                .unwrap(),
9838            )
9839            .await
9840            .unwrap();
9841        let ClientControlResponse::ServerDescribe { machine_id, .. } =
9842            serde_json::from_slice(&described[0].body).unwrap()
9843        else {
9844            panic!("server.describe answered with another shape");
9845        };
9846        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9847    }
9848
9849    #[test]
9850    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9851        // With a central sqlite storage policy, each registering module gets its
9852        // own resolved descriptor in HELLO_ACK, keyed by its module id.
9853        let registry = Arc::new(Registry::default());
9854        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
9855            crate::daemon_config::StorageConfig::Sqlite {
9856                data_home: std::path::PathBuf::from("/data"),
9857            },
9858        ));
9859
9860        let responses = handler
9861            .handle_control(
9862                ConnectionId::new(1),
9863                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9864            )
9865            .unwrap();
9866        let ack = parse_ack(&responses[0]);
9867        assert_eq!(
9868            ack.storage,
9869            Some(serde_json::json!({
9870                "module_id": "alfonso-routing",
9871                "storage_namespace": "default",
9872                "isolation": { "kind": "module" },
9873                "backend": {
9874                    "backend": "sqlite",
9875                    "path": "/data/cortexkit/alfonso-routing/store.db"
9876                }
9877            })),
9878            "the delivered descriptor is the module's own sqlite store path"
9879        );
9880    }
9881
9882    #[test]
9883    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9884        let registry = Arc::new(Registry::default());
9885        let handler = ControlHandler::new(Arc::clone(&registry));
9886        let conn = ConnectionId::new(1);
9887        let responses = handler
9888            .handle_control(
9889                conn,
9890                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9891            )
9892            .unwrap();
9893        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9894        let registration = registry.get_module("aft").unwrap().unwrap();
9895        assert_eq!(registration.control_ops, module_baseline_control_ops());
9896
9897        let frame =
9898            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9899        assert!(handler
9900            .guard_module_control_op(&frame, "aft", "route.bind")
9901            .unwrap()
9902            .is_none());
9903        let error = handler
9904            .guard_module_control_op(&frame, "aft", "test.synthetic")
9905            .unwrap()
9906            .expect("synthetic ungranted op should be rejected");
9907        assert_eq!(error.header.ty, FrameType::Error);
9908        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9909    }
9910
9911    #[test]
9912    fn hello_control_ops_some_adds_optional_grants() {
9913        let registry = Arc::new(Registry::default());
9914        let handler = ControlHandler::new(Arc::clone(&registry));
9915        handler
9916            .handle_control(
9917                ConnectionId::new(1),
9918                hello_frame_with_control_ops(
9919                    "aft",
9920                    PROTOCOL_VERSION,
9921                    7,
9922                    Some(vec![
9923                        "future.synthetic".to_string(),
9924                        "route.bind".to_string(),
9925                    ]),
9926                ),
9927            )
9928            .unwrap();
9929        let registration = registry.get_module("aft").unwrap().unwrap();
9930        assert_eq!(
9931            registration.control_ops,
9932            vec![
9933                "route.bind".to_string(),
9934                "route.status".to_string(),
9935                "future.synthetic".to_string(),
9936            ]
9937        );
9938        let frame =
9939            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9940        assert!(handler
9941            .guard_module_control_op(&frame, "aft", "future.synthetic")
9942            .unwrap()
9943            .is_none());
9944    }
9945
9946    #[tokio::test]
9947    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9948        let registry = Arc::new(Registry::default());
9949        let forwarding = Arc::new(ForwardingTable::default());
9950        let handler =
9951            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9952        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9953        hello_via_sink(
9954            &handler,
9955            &module_ctx,
9956            &mut module_rx,
9957            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9958        )
9959        .await;
9960
9961        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9962        let responses = handler
9963            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9964            .await
9965            .unwrap();
9966        assert_eq!(responses.len(), 1);
9967        assert_eq!(responses[0].header.ty, FrameType::Error);
9968        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9969        assert!(module_rx.try_recv().is_err());
9970    }
9971
9972    #[tokio::test]
9973    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9974        let registry = Arc::new(Registry::default());
9975        let forwarding = Arc::new(ForwardingTable::default());
9976        let handler =
9977            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9978        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9979        hello_via_sink(
9980            &handler,
9981            &module_ctx,
9982            &mut module_rx,
9983            hello_frame_with_control_ops(
9984                "aft",
9985                PROTOCOL_VERSION,
9986                7,
9987                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9988            ),
9989        )
9990        .await;
9991
9992        let project_root = unique_project_root("demux");
9993        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9994        let route_handler = handler.clone();
9995        let route_task = tokio::spawn(async move {
9996            route_handler
9997                .handle_control_frame(
9998                    &route_client_ctx,
9999                    route_open_frame(100, "aft", project_root),
10000                )
10001                .await
10002                .unwrap()
10003        });
10004        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10005            .await
10006            .unwrap()
10007            .unwrap();
10008        assert!(matches!(
10009            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10010            ModuleControlRequest::RouteBind { .. }
10011        ));
10012
10013        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10014        let health_handler = handler.clone();
10015        let health_task = tokio::spawn(async move {
10016            health_handler
10017                .handle_control_frame(
10018                    &health_client_ctx,
10019                    supervisor_health_probe_frame(101, "aft"),
10020                )
10021                .await
10022                .unwrap()
10023        });
10024        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10025            .await
10026            .unwrap()
10027            .unwrap();
10028        assert_eq!(
10029            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10030            ModuleControlRequest::HealthCheck {}
10031        );
10032
10033        handler
10034            .handle_control_frame(
10035                &module_ctx,
10036                health_response(health_frame.header.corr, HealthStatus::Degraded),
10037            )
10038            .await
10039            .unwrap();
10040        let health_response = health_task.await.unwrap();
10041        assert_eq!(health_response.len(), 1);
10042        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10043            ClientControlResponse::SupervisorHealthProbe {
10044                module_id,
10045                status,
10046                detail,
10047                metrics,
10048            } => {
10049                assert_eq!(module_id, "aft");
10050                assert_eq!(status, HealthStatus::Degraded);
10051                assert_eq!(detail.as_deref(), Some("warming"));
10052                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10053            }
10054            other => panic!("unexpected health response: {other:?}"),
10055        }
10056
10057        handler
10058            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10059            .await
10060            .unwrap();
10061        let route_response = route_task.await.unwrap();
10062        assert!(route_response.is_empty());
10063        let published = route_client_rx.recv().await.unwrap();
10064        assert!(matches!(
10065            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10066            ClientControlResponse::RouteOpen { .. }
10067        ));
10068    }
10069
10070    /// Start one `route.open` on `client_connection` and return its still-running
10071    /// handler task together with the `route.bind` the module received for it.
10072    /// The handler blocks until the module answers, so it has to run as a task
10073    /// while the test drives the module side.
10074    async fn relay_route_open(
10075        handler: &ControlHandler,
10076        client_connection: ConnectionId,
10077        client_egress: &FrameSink,
10078        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10079        corr: u64,
10080        module_id: &str,
10081        project_root_label: &str,
10082    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10083        let ctx = RouteCtx {
10084            connection_id: client_connection,
10085            egress: client_egress.clone(),
10086        };
10087        let handler = handler.clone();
10088        let project_root = unique_project_root(project_root_label);
10089        let module_id = module_id.to_string();
10090        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10091        let task = tokio::spawn(async move {
10092            let _guard = tracing::dispatcher::set_default(&dispatch);
10093            handler
10094                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10095                .await
10096                .unwrap()
10097        });
10098        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10099            .await
10100            .expect("module receives the relayed route.bind")
10101            .expect("module egress is open");
10102        (task, bind.frame)
10103    }
10104
10105    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10106        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10107            ModuleControlRequest::RouteBind {
10108                route_channel,
10109                epoch,
10110                ..
10111            } => (route_channel, epoch),
10112            other => panic!("expected a route.bind request, got {other:?}"),
10113        }
10114    }
10115
10116    fn published_route(frame: &Frame) -> (u16, u32) {
10117        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10118            ClientControlResponse::RouteOpen {
10119                route_channel,
10120                route_epoch,
10121            } => (route_channel, route_epoch),
10122            other => panic!("expected a route.open response, got {other:?}"),
10123        }
10124    }
10125
10126    /// Reproduction of a production outage. A client had `route.open`s in
10127    /// flight to a module and was already marked closing -- its egress had refused a
10128    /// module frame, so the daemon asked its connection to end -- while its sink
10129    /// was still open. When the module acked those binds, the daemon refused to
10130    /// commit a route for a closing client, and that refusal was returned from
10131    /// the MODULE connection's frame handler, where a router error that has no
10132    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
10133    /// the supervisor correctly did not respawn a clean exit, and every seat lost
10134    /// its tools for hours -- one client's teardown took down a connection
10135    /// carrying ~170 other routes.
10136    ///
10137    /// The window is opened here by calling the production path that opens it
10138    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
10139    /// state that matters is "in `closing_connections`, sink still open, relay
10140    /// still pending", and it lasts only from the close request until the
10141    /// connection loop reacts to it; a socket-level test can flood a client into
10142    /// that escalation but cannot pin the module's ack inside the window. Closing
10143    /// the socket instead takes the other path entirely -- connection teardown
10144    /// removes the pending relay under the same lock, so the ack finds nothing.
10145    #[tokio::test]
10146    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10147        let registry = Arc::new(Registry::default());
10148        let forwarding = Arc::new(ForwardingTable::default());
10149        let handler =
10150            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10151
10152        let module_connection = ConnectionId::new(30);
10153        let (module_ctx, mut module_rx) = route_ctx(module_connection);
10154        hello_via_sink(
10155            &handler,
10156            &module_ctx,
10157            &mut module_rx,
10158            hello_frame("aft", PROTOCOL_VERSION, 7),
10159        )
10160        .await;
10161
10162        let dying_client = ConnectionId::new(31);
10163        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10164
10165        // A published route on the dying client. The escalation below only marks
10166        // a connection closing for a route it has already published.
10167        let (first_task, first_bind) = relay_route_open(
10168            &handler,
10169            dying_client,
10170            &dying_ctx.egress,
10171            &mut module_rx,
10172            100,
10173            "aft",
10174            "closing-first",
10175        )
10176        .await;
10177        handler
10178            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10179            .await
10180            .unwrap();
10181        assert!(first_task.await.unwrap().is_empty());
10182        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10183
10184        // A second route.open from the same client, relayed and awaiting its ack.
10185        let (second_task, second_bind) = relay_route_open(
10186            &handler,
10187            dying_client,
10188            &dying_ctx.egress,
10189            &mut module_rx,
10190            101,
10191            "aft",
10192            "closing-second",
10193        )
10194        .await;
10195        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10196
10197        // The window: the client is closing, its sink is still open, and its
10198        // second bind is still pending.
10199        assert!(forwarding
10200            .escalate_client_delivery_failure(
10201                dying_client,
10202                first_channel,
10203                first_epoch,
10204                CloseReason::new(
10205                    "module_to_client_delivery_failed",
10206                    "client egress refused a module frame",
10207                ),
10208                crate::forwarding::UndeliveredFrame {
10209                    module_id: None,
10210                    sink: &dying_ctx.egress,
10211                },
10212            )
10213            .unwrap());
10214        assert!(!dying_ctx.egress.is_closed());
10215
10216        // The frame that used to end the module connection.
10217        let ack = handler
10218            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10219            .await;
10220        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10221        if module_loop_error.is_some() {
10222            // What the server's connection loop does with a router error that has
10223            // no ERROR-frame translation: end the connection, which releases the
10224            // module's registration and every route on it.
10225            handler.cleanup_connection(module_connection).unwrap();
10226        }
10227        // Read the module's next frame before opening the co-tenant's route, so
10228        // the GOODBYE assertion below is about THIS ack and not about later
10229        // traffic. `None` means the module was told nothing.
10230        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10231            .await
10232            .ok()
10233            .flatten();
10234
10235        // 1. The module connection is still registered.
10236        assert!(
10237            registry
10238                .get_module_by_connection(module_connection)
10239                .unwrap()
10240                .is_some(),
10241            "one client's closing connection ended the shared module connection: \
10242             {module_loop_error:?}"
10243        );
10244        // ...and still serving: another client can open and use a route on it.
10245        let cotenant = ConnectionId::new(32);
10246        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10247        let (cotenant_task, cotenant_bind) = relay_route_open(
10248            &handler,
10249            cotenant,
10250            &cotenant_ctx.egress,
10251            &mut module_rx,
10252            102,
10253            "aft",
10254            "closing-cotenant",
10255        )
10256        .await;
10257        handler
10258            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10259            .await
10260            .unwrap();
10261        assert!(cotenant_task.await.unwrap().is_empty());
10262        let (cotenant_channel, cotenant_epoch) =
10263            published_route(&cotenant_rx.recv().await.unwrap());
10264        assert!(matches!(
10265            forwarding
10266                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10267                .unwrap(),
10268            DataRoute::Client(DataRouteState::Bound(_))
10269        ));
10270
10271        // 2. The module was told to drop the binding it created for the route
10272        //    that will never be published.
10273        let goodbye = post_ack_module_frame
10274            .expect("module receives a GOODBYE for the abandoned route channel");
10275        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10276        assert_eq!(goodbye.header.channel, abandoned_channel);
10277        assert_eq!(goodbye.header.epoch, abandoned_epoch);
10278
10279        // 3. The dying client received nothing: no route was ever published to
10280        //    it. Its route.open is answered as unavailable, which the connection
10281        //    loop would write to a socket that is already going away.
10282        assert!(dying_rx.try_recv().is_err());
10283        let second_response = second_task.await.unwrap();
10284        assert_eq!(second_response.len(), 1);
10285        assert_eq!(
10286            parse_error(&second_response[0])["code"],
10287            "target_unavailable"
10288        );
10289    }
10290
10291    /// One `catalog.list` as a client sees it: the generation and the listed
10292    /// module ids with their readiness.
10293    async fn catalog_snapshot(
10294        handler: &ControlHandler,
10295        client_ctx: &RouteCtx,
10296        corr: u64,
10297    ) -> (u64, Vec<(String, bool)>) {
10298        let body =
10299            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10300        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10301        let reply = handler
10302            .handle_control_frame(client_ctx, request)
10303            .await
10304            .unwrap()
10305            .pop()
10306            .unwrap();
10307        let ClientControlResponse::CatalogList {
10308            generation,
10309            modules,
10310            ..
10311        } = serde_json::from_slice(&reply.body).unwrap()
10312        else {
10313            panic!("not a catalog.list reply");
10314        };
10315        (
10316            generation,
10317            modules
10318                .into_iter()
10319                .map(|entry| (entry.module_id, entry.ready))
10320                .collect(),
10321        )
10322    }
10323
10324    /// A handler whose HELLO for module "aft" on connection 40 is parked
10325    /// between reserving its registry entry and installing its forwarding
10326    /// endpoint. Returns the handler, the two-party barrier (wait once to know
10327    /// the HELLO is parked, once more to release it), the HELLO's thread, and
10328    /// the module's context and receiver.
10329    fn hello_held_before_forwarding() -> (
10330        ControlHandler,
10331        Arc<std::sync::Barrier>,
10332        std::thread::JoinHandle<Vec<Frame>>,
10333        RouteCtx,
10334        mpsc::Receiver<crate::router::OutboundFrame>,
10335    ) {
10336        let hold = Arc::new(std::sync::Barrier::new(2));
10337        let handler = ControlHandler::with_forwarding(
10338            Arc::new(Registry::default()),
10339            Arc::new(ForwardingTable::default()),
10340        )
10341        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10342        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10343        let hello = {
10344            let handler = handler.clone();
10345            let module_ctx = module_ctx.clone();
10346            std::thread::spawn(move || {
10347                handler
10348                    .handle_hello(
10349                        module_ctx.connection_id,
10350                        Some(module_ctx.egress.clone()),
10351                        hello_frame("aft", PROTOCOL_VERSION, 7),
10352                    )
10353                    .unwrap()
10354            })
10355        };
10356        (handler, hold, hello, module_ctx, module_rx)
10357    }
10358
10359    /// While a HELLO has reserved its registry entry but its forwarding
10360    /// endpoint is not installed yet, `route.open` to the module can only be
10361    /// refused. `catalog.list` must not list the module in that window: a
10362    /// client that opens a route as soon as it sees a module listed and ready
10363    /// would be refused for a module the daemon had just advertised. The
10364    /// refusal in the window stays the retryable `target_unavailable`, never
10365    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
10366    /// Once the HELLO completes, the module is listed and a route to it opens.
10367    #[tokio::test]
10368    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10369        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10370        let client = ConnectionId::new(41);
10371        let (client_ctx, mut client_rx) = route_ctx(client);
10372
10373        hold.wait();
10374        // Parked: the registry holds "aft", forwarding has no endpoint for it.
10375        assert!(handler.registry.get_module("aft").unwrap().is_some());
10376        assert!(!handler
10377            .forwarding
10378            .has_live_module_connection("aft")
10379            .unwrap());
10380        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10381        assert!(
10382            listed.is_empty(),
10383            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10384        );
10385        let refused = handler
10386            .handle_control_frame(
10387                &client_ctx,
10388                route_open_frame(301, "aft", unique_project_root("listed-window")),
10389            )
10390            .await
10391            .unwrap();
10392        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10393        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10394        let duplicate = handler
10395            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10396            .await
10397            .unwrap();
10398        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10399
10400        hold.wait();
10401        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10402        let ack = module_rx.try_recv().unwrap().frame;
10403        assert_eq!(ack.header.ty, FrameType::HelloAck);
10404        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10405        assert_eq!(listed, vec![("aft".to_string(), true)]);
10406
10407        let (open_task, bind) = relay_route_open(
10408            &handler,
10409            client,
10410            &client_ctx.egress,
10411            &mut module_rx,
10412            303,
10413            "aft",
10414            "listed-after",
10415        )
10416        .await;
10417        assert!(handler
10418            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10419            .await
10420            .unwrap()
10421            .is_empty());
10422        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10423        published_route(&client_rx.try_recv().unwrap().frame);
10424    }
10425
10426    /// A catalog watcher keys on the generation: when it changes, it re-reads
10427    /// the listing. So the generation must advance exactly when the module
10428    /// appears. If it advanced while the module was still hidden, a watcher
10429    /// would read the new generation without the module, and the module would
10430    /// later appear under that same generation with nothing telling the
10431    /// watcher to look again.
10432    #[tokio::test]
10433    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10434        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10435        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10436        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10437
10438        hold.wait();
10439        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10440        assert!(listed.is_empty());
10441        assert_eq!(
10442            during, before,
10443            "the generation moved while the module was not listed"
10444        );
10445
10446        hold.wait();
10447        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10448        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10449        assert_eq!(listed, vec![("aft".to_string(), true)]);
10450        assert_eq!(
10451            after,
10452            before + 1,
10453            "the module appeared without exactly one generation bump"
10454        );
10455    }
10456
10457    /// A HELLO whose forwarding endpoint cannot be installed (here the
10458    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
10459    /// refused and leaves nothing behind: no registry entry, no catalog entry,
10460    /// and the catalog generation where it was.
10461    #[tokio::test]
10462    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10463        let registry = Arc::new(Registry::default());
10464        let handler = ControlHandler::with_forwarding(
10465            Arc::clone(&registry),
10466            Arc::new(ForwardingTable::default()),
10467        );
10468        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10469        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10470        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10471        drop(module_rx);
10472
10473        let replies = handler
10474            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10475            .await
10476            .unwrap();
10477        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10478        assert!(registry.get_module("aft").unwrap().is_none());
10479        assert!(registry
10480            .get_module_by_connection(module_ctx.connection_id)
10481            .unwrap()
10482            .is_none());
10483        assert_eq!(
10484            catalog_snapshot(&handler, &client_ctx, 301).await,
10485            (before, Vec::new())
10486        );
10487    }
10488
10489    /// A handler whose `route.open` parks after queueing the route.bind and
10490    /// before marking it enqueued, with module "aft" registered on connection
10491    /// 40. Returns the two-party barrier that parks and releases it.
10492    async fn route_open_held_after_bind_send() -> (
10493        ControlHandler,
10494        Arc<tokio::sync::Barrier>,
10495        RouteCtx,
10496        mpsc::Receiver<crate::router::OutboundFrame>,
10497    ) {
10498        let hold = Arc::new(tokio::sync::Barrier::new(2));
10499        let handler = ControlHandler::with_forwarding(
10500            Arc::new(Registry::default()),
10501            Arc::new(ForwardingTable::default()),
10502        )
10503        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10504        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10505        hello_via_sink(
10506            &handler,
10507            &module_ctx,
10508            &mut module_rx,
10509            hello_frame("aft", PROTOCOL_VERSION, 7),
10510        )
10511        .await;
10512        (handler, hold, module_ctx, module_rx)
10513    }
10514
10515    /// A fast module acks the route.bind, and the reader commits the route,
10516    /// before `route.open` has marked the relay enqueued. The daemon used to
10517    /// read the missing relay as an abandoned bind and send the module a
10518    /// GOODBYE for the route it had just committed: the client was told its
10519    /// route was open, the module unbound the channel, and the client's first
10520    /// request was dropped on the module side as unbound.
10521    #[tokio::test]
10522    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10523        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10524        let client = ConnectionId::new(41);
10525        let (client_ctx, mut client_rx) = route_ctx(client);
10526
10527        let (open_task, bind) = relay_route_open(
10528            &handler,
10529            client,
10530            &client_ctx.egress,
10531            &mut module_rx,
10532            200,
10533            "aft",
10534            "acked-before-mark",
10535        )
10536        .await;
10537        let (module_channel, module_epoch) = route_bind_channel(&bind);
10538        // route.open is now parked between queueing the bind and marking it.
10539        hold.wait().await;
10540
10541        // The module answers inside that window, and the route commits.
10542        assert!(handler
10543            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10544            .await
10545            .unwrap()
10546            .is_empty());
10547        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10548
10549        hold.wait().await;
10550        assert!(
10551            open_task.await.unwrap().is_empty(),
10552            "route.open answers accepted"
10553        );
10554
10555        // Every frame route.open could have sent is already queued, because it
10556        // has returned; the module must have been sent nothing.
10557        match module_rx.try_recv() {
10558            Err(mpsc::error::TryRecvError::Empty) => {}
10559            Ok(outbound) => panic!(
10560                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10561                outbound.frame.header.ty,
10562                outbound.frame.header.channel,
10563                outbound.frame.header.epoch
10564            ),
10565            Err(err) => panic!("module egress closed: {err}"),
10566        }
10567
10568        // The client's first request on the route is the next thing the module
10569        // reads, on the binding it just created.
10570        let router = Router::with_control_handler(Arc::new(handler.clone()));
10571        let request = Frame::build(
10572            FrameType::Request,
10573            Flags::new(true, Priority::Interactive, false),
10574            client_channel,
10575            client_epoch,
10576            201,
10577            b"first request".to_vec(),
10578        )
10579        .unwrap();
10580        router
10581            .route_for_connection(&client_ctx, request)
10582            .await
10583            .unwrap();
10584        let delivered = module_rx.try_recv().unwrap().frame;
10585        assert_eq!(delivered.header.ty, FrameType::Request);
10586        assert_eq!(
10587            (delivered.header.channel, delivered.header.epoch),
10588            (module_channel, module_epoch)
10589        );
10590        assert_eq!(delivered.body, b"first request");
10591    }
10592
10593    /// The client goes away while `route.open` is still queueing the bind, so
10594    /// the relay is dropped with no module reply. The teardown cannot send the
10595    /// GOODBYE itself (it could reach the module ahead of the bind), so
10596    /// `route.open` must send it once the bind is queued; otherwise the module
10597    /// keeps a binding the daemon will never commit.
10598    #[tokio::test]
10599    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10600        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10601        let client = ConnectionId::new(42);
10602        let (client_ctx, _client_rx) = route_ctx(client);
10603
10604        let (open_task, bind) = relay_route_open(
10605            &handler,
10606            client,
10607            &client_ctx.egress,
10608            &mut module_rx,
10609            300,
10610            "aft",
10611            "released-before-mark",
10612        )
10613        .await;
10614        let (module_channel, module_epoch) = route_bind_channel(&bind);
10615        hold.wait().await;
10616
10617        handler.cleanup_connection(client).unwrap();
10618        assert!(
10619            module_rx.try_recv().is_err(),
10620            "client teardown must not GOODBYE a bind route.open is still queueing"
10621        );
10622
10623        hold.wait().await;
10624        let response = open_task.await.unwrap();
10625        assert_eq!(response.len(), 1);
10626        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10627
10628        let goodbye = module_rx
10629            .try_recv()
10630            .expect("module is told to drop the binding the daemon abandoned")
10631            .frame;
10632        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10633        assert_eq!(
10634            (goodbye.header.channel, goodbye.header.epoch),
10635            (module_channel, module_epoch)
10636        );
10637        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10638    }
10639
10640    /// The module acks inside the same window, but the client's egress closed
10641    /// first, so the reader refuses to commit. The ack proves the module holds
10642    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
10643    /// send a second one.
10644    #[tokio::test]
10645    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
10646        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10647        let client = ConnectionId::new(43);
10648        let (client_ctx, client_rx) = route_ctx(client);
10649
10650        let (open_task, bind) = relay_route_open(
10651            &handler,
10652            client,
10653            &client_ctx.egress,
10654            &mut module_rx,
10655            400,
10656            "aft",
10657            "gone-client-before-mark",
10658        )
10659        .await;
10660        let (module_channel, module_epoch) = route_bind_channel(&bind);
10661        hold.wait().await;
10662
10663        drop(client_rx);
10664        assert!(client_ctx.egress.is_closed());
10665        handler
10666            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10667            .await
10668            .unwrap();
10669
10670        hold.wait().await;
10671        let response = open_task.await.unwrap();
10672        assert_eq!(response.len(), 1);
10673        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10674
10675        let goodbye = module_rx
10676            .try_recv()
10677            .expect("module is told to drop the binding it acked")
10678            .frame;
10679        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10680        assert_eq!(
10681            (goodbye.header.channel, goodbye.header.epoch),
10682            (module_channel, module_epoch)
10683        );
10684        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10685        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
10686    }
10687
10688    /// The fence at the module-loop boundary, stated as its own contract: which
10689    /// forwarding failures are allowed to end the module connection that is being
10690    /// served. A `ConnectionClosing` naming some client is about that client, and
10691    /// a module connection is shared; the same error naming the module's own
10692    /// connection is about this connection and must stay fatal, as must failures
10693    /// that are about the forwarding table itself.
10694    #[test]
10695    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
10696        let handler = ControlHandler::default();
10697        let module_connection = ConnectionId::new(30);
10698        let client_connection = ConnectionId::new(31);
10699
10700        handler
10701            .refuse_to_end_module_connection_for_a_client(
10702                module_connection,
10703                77,
10704                ForwardingError::ConnectionClosing {
10705                    connection_id: client_connection,
10706                },
10707            )
10708            .expect("a closing client must never end the module connection");
10709
10710        assert!(matches!(
10711            handler.refuse_to_end_module_connection_for_a_client(
10712                module_connection,
10713                78,
10714                ForwardingError::ConnectionClosing {
10715                    connection_id: module_connection,
10716                },
10717            ),
10718            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
10719                connection_id
10720            })) if connection_id == module_connection
10721        ));
10722        assert!(matches!(
10723            handler.refuse_to_end_module_connection_for_a_client(
10724                module_connection,
10725                79,
10726                ForwardingError::Poisoned,
10727            ),
10728            Err(RouterError::Forwarding(ForwardingError::Poisoned))
10729        ));
10730        assert!(matches!(
10731            handler.refuse_to_end_module_connection_for_a_client(
10732                module_connection,
10733                80,
10734                ForwardingError::StaleModuleEndpoint,
10735            ),
10736            Err(RouterError::Forwarding(
10737                ForwardingError::StaleModuleEndpoint
10738            ))
10739        ));
10740    }
10741
10742    /// The spawn-attestation guard is what stops a connected module from claiming
10743    /// another module's identity and being stamped `Reserved` for it. Every other
10744    /// test that supplies a consumer_identity supplies a CORRECT one, because a
10745    /// correct one is what the rest of the flow needs -- so the guard's rejection
10746    /// branch was never the subject of an assertion, only its acceptance branch.
10747    ///
10748    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
10749    /// whole subc-core library suite green; only the forwarding integration tests
10750    /// notice, and they notice for unrelated reasons. This test exists so the
10751    /// refusal itself is asserted where the guard lives: it fails if the identity
10752    /// check stops refusing, which is the direction that matters, since a guard
10753    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
10754    #[tokio::test]
10755    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
10756        let registry = Arc::new(Registry::default());
10757        let forwarding = Arc::new(ForwardingTable::default());
10758        let supervisor = SupervisorHandle::new();
10759        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10760        let handler =
10761            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10762                .with_supervisor(supervisor);
10763
10764        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10765        hello_via_sink(
10766            &handler,
10767            &target_ctx,
10768            &mut target_rx,
10769            hello_frame("target", PROTOCOL_VERSION, 1),
10770        )
10771        .await;
10772
10773        // A real supervised module id presenting the wrong nonce. This is the
10774        // impersonation case: the attacker knows a privileged module_id, which is
10775        // public, and guesses at the nonce, which is not.
10776        let wrong_nonce = handler
10777            .handle_control_frame(
10778                &route_ctx(ConnectionId::new(91)).0,
10779                route_open_frame_with_admission_facts(
10780                    20,
10781                    "target",
10782                    unique_project_root("admission-facts"),
10783                    Some(subc_control::ConsumerIdentity {
10784                        module_id: "fed".to_string(),
10785                        launch_nonce: "not-the-real-nonce".to_string(),
10786                    }),
10787                    None,
10788                ),
10789            )
10790            .await
10791            .unwrap();
10792        assert_eq!(
10793            parse_error(&wrong_nonce[0])["code"],
10794            "bad_consumer_identity",
10795            "a mismatched launch nonce must be refused, not stamped Reserved"
10796        );
10797
10798        // A module id the supervisor never spawned at all, so no nonce exists to
10799        // compare against. An implementation that treats "no record" as "nothing
10800        // to check" fails open here while passing the case above.
10801        let never_spawned = handler
10802            .handle_control_frame(
10803                &route_ctx(ConnectionId::new(92)).0,
10804                route_open_frame_with_admission_facts(
10805                    21,
10806                    "target",
10807                    unique_project_root("admission-facts"),
10808                    Some(subc_control::ConsumerIdentity {
10809                        module_id: "never-spawned".to_string(),
10810                        launch_nonce: "any-nonce".to_string(),
10811                    }),
10812                    None,
10813                ),
10814            )
10815            .await
10816            .unwrap();
10817        assert_eq!(
10818            parse_error(&never_spawned[0])["code"],
10819            "bad_consumer_identity",
10820            "an unspawned module_id must be refused rather than accepted for lack of a record"
10821        );
10822    }
10823
10824    /// The refusal test above proves the guard says NO. Nothing proved it can say
10825    /// YES, and the difference is not academic: replacing the whole authorization
10826    /// with `false` -- admitting no consumer identity at all, revoking Reserved
10827    /// standing for every supervised module in the fleet -- leaves 110 of the 111
10828    /// library tests GREEN. The one that notices does so by HANGING, because it
10829    /// waits for a bind that can no longer happen.
10830    ///
10831    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
10832    /// or flaky test, invites a RETRY rather than an investigation, and the retry
10833    /// hangs too and gets blamed on the runner. So a total revocation of the
10834    /// daemon's trust grant would have shipped behind a symptom nobody attributes
10835    /// to code.
10836    ///
10837    /// The bias is structural rather than accidental. A REFUSAL looks like a
10838    /// failure someone writes a test for; a GRANT looks like the happy path. Every
10839    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
10840    /// suite for that reason, and this one is the purest case in the daemon.
10841    ///
10842    /// This test asserts the EFFECT rather than the absence of an error: the module
10843    /// receives a RouteBind and it carries `Reserved` naming the attested module.
10844    /// A guard that admitted nobody would produce no bind at all; one that admitted
10845    /// everybody would stamp the wrong principal, which the refusal test catches.
10846    #[tokio::test]
10847    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10848        let registry = Arc::new(Registry::default());
10849        let forwarding = Arc::new(ForwardingTable::default());
10850        let supervisor = SupervisorHandle::new();
10851        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10852        let handler =
10853            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10854                .with_supervisor(supervisor);
10855
10856        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10857        hello_via_sink(
10858            &handler,
10859            &target_ctx,
10860            &mut target_rx,
10861            hello_frame("target", PROTOCOL_VERSION, 1),
10862        )
10863        .await;
10864
10865        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10866        let route_handler = handler.clone();
10867        let route_task = tokio::spawn(async move {
10868            route_handler
10869                .handle_control_frame(
10870                    &client_ctx,
10871                    route_open_frame_with_admission_facts(
10872                        30,
10873                        "target",
10874                        unique_project_root("admission-facts"),
10875                        Some(subc_control::ConsumerIdentity {
10876                            module_id: "fed".to_string(),
10877                            launch_nonce: "fed-nonce".to_string(),
10878                        }),
10879                        None,
10880                    ),
10881                )
10882                .await
10883                .unwrap()
10884        });
10885
10886        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
10887        // the very mutation it exists to catch -- a guard that admits nobody -- no
10888        // bind is ever sent, so it HUNG rather than failing. That reproduces the
10889        // exact defect being fixed: a total revocation detected only as a stalled
10890        // suite, which reads as flakiness and invites a retry. An acceptance test
10891        // that waits for an effect must bound the wait, or a red becomes a hang.
10892        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10893            .await
10894            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10895            .expect("module control channel closed before route.bind");
10896        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10897        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10898            panic!("expected route.bind")
10899        };
10900        assert_eq!(
10901            principal,
10902            Some(Principal::Reserved {
10903                module_id: "fed".to_string()
10904            }),
10905            "a correctly attested consumer must be stamped Reserved for its own id"
10906        );
10907
10908        handler
10909            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10910            .await
10911            .unwrap();
10912        assert!(route_task.await.unwrap().is_empty());
10913        assert!(
10914            matches!(
10915                serde_json::from_slice::<ClientControlResponse>(
10916                    &client_rx.recv().await.unwrap().body
10917                )
10918                .unwrap(),
10919                ClientControlResponse::RouteOpen { .. }
10920            ),
10921            "the route must actually open, not merely avoid an error"
10922        );
10923    }
10924
10925    #[tokio::test(start_paused = true)]
10926    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10927        let registry = Arc::new(Registry::default());
10928        let forwarding = Arc::new(ForwardingTable::default());
10929        let supervisor = SupervisorHandle::new();
10930        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10931        let handler =
10932            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10933                .with_supervisor(supervisor);
10934
10935        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10936        hello_via_sink(
10937            &handler,
10938            &target_ctx,
10939            &mut target_rx,
10940            hello_frame("target", PROTOCOL_VERSION, 1),
10941        )
10942        .await;
10943
10944        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10945        let direct_handler = handler.clone();
10946        let direct_open = tokio::spawn(async move {
10947            direct_handler
10948                .handle_control_frame(
10949                    &direct_ctx,
10950                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
10951                )
10952                .await
10953                .unwrap()
10954        });
10955        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10956            .await
10957            .expect("no direct route.bind within 5s")
10958            .expect("target control channel closed before direct route.bind");
10959        handler
10960            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10961            .await
10962            .unwrap();
10963        assert!(direct_open.await.unwrap().is_empty());
10964        let _ = direct_rx.recv().await.unwrap();
10965
10966        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10967        let reserved_handler = handler.clone();
10968        let reserved_open = tokio::spawn(async move {
10969            reserved_handler
10970                .handle_control_frame(
10971                    &reserved_ctx,
10972                    route_open_frame_with_admission_facts(
10973                        3,
10974                        "target",
10975                        unique_project_root("admission-facts"),
10976                        Some(ConsumerIdentity {
10977                            module_id: "fed".to_string(),
10978                            launch_nonce: "fed-nonce".to_string(),
10979                        }),
10980                        None,
10981                    ),
10982                )
10983                .await
10984                .unwrap()
10985        });
10986        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10987            .await
10988            .expect("no reserved route.bind within 5s")
10989            .expect("target control channel closed before reserved route.bind");
10990        handler
10991            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10992            .await
10993            .unwrap();
10994        assert!(reserved_open.await.unwrap().is_empty());
10995        let _ = reserved_rx.recv().await.unwrap();
10996
10997        forwarding
10998            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10999            .unwrap();
11000        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11001        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11002            module_id: Some("target".to_string()),
11003        })
11004        .unwrap();
11005        let census_frame =
11006            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11007        let response = handler
11008            .handle_control_frame(&census_ctx, census_frame)
11009            .await
11010            .unwrap()
11011            .pop()
11012            .unwrap();
11013        let actual: Value = serde_json::from_slice(&response.body).unwrap();
11014        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11015        assert!(matches!(
11016            decoded,
11017            ClientControlResponse::SupervisorRoutes { .. }
11018        ));
11019        let routes = actual["modules"][0]["routes"].as_array().unwrap();
11020        assert_eq!(routes.len(), 2);
11021        assert!(routes.iter().all(|route| route["draining"] == true));
11022        // The census carries WHY: the reason the drain was begun with, in the
11023        // route.closing vocabulary, on every draining route this drain marked.
11024        assert!(
11025            routes.iter().all(|route| route["drain_reason"] == "reload"),
11026            "draining routes must name the drain's reason: {routes:?}"
11027        );
11028        assert!(routes.iter().any(|route| {
11029            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11030        }));
11031        assert!(routes.iter().any(|route| {
11032            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11033        }));
11034
11035        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11036            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11037        if std::env::var_os("UPDATE_GOLDEN").is_some() {
11038            std::fs::write(
11039                &golden_path,
11040                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11041            )
11042            .unwrap();
11043        }
11044        let expected: Value =
11045            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11046        assert_eq!(actual, expected);
11047    }
11048
11049    async fn query_live_roots(
11050        handler: &ControlHandler,
11051        module_ctx: &RouteCtx,
11052    ) -> ModuleControlResponseToModule {
11053        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11054        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11055        let response = handler
11056            .handle_control_frame(module_ctx, frame)
11057            .await
11058            .unwrap()
11059            .pop()
11060            .unwrap();
11061        serde_json::from_slice(&response.body).unwrap()
11062    }
11063
11064    #[tokio::test(start_paused = true)]
11065    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11066        let registry = Arc::new(Registry::default());
11067        let forwarding = Arc::new(ForwardingTable::default());
11068        let handler = ControlHandler::with_forwarding(registry, forwarding);
11069        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11070        hello_via_sink(
11071            &handler,
11072            &target_ctx,
11073            &mut target_rx,
11074            hello_frame("target", PROTOCOL_VERSION, 1),
11075        )
11076        .await;
11077        let root = unique_project_root("live-roots-known");
11078        let path = ProjectRootId::from_path_allowing_missing(root.path())
11079            .unwrap()
11080            .as_path()
11081            .to_path_buf();
11082        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11083        let open_handler = handler.clone();
11084        let opened = tokio::spawn(async move {
11085            open_handler
11086                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11087                .await
11088                .unwrap()
11089        });
11090        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11091            .await
11092            .unwrap()
11093            .unwrap();
11094        handler
11095            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11096            .await
11097            .unwrap();
11098        assert!(opened.await.unwrap().is_empty());
11099        let _ = client_rx.recv().await.unwrap();
11100
11101        let root = unique_project_root("live-roots-pending");
11102        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11103            .unwrap()
11104            .as_path()
11105            .to_path_buf();
11106        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11107        let open_handler = handler.clone();
11108        let pending = tokio::spawn(async move {
11109            open_handler
11110                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11111                .await
11112                .unwrap()
11113        });
11114        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11115            .await
11116            .unwrap()
11117            .unwrap();
11118        let actual = query_live_roots(&handler, &target_ctx).await;
11119        let ModuleControlResponseToModule::LiveRoots {
11120            roots,
11121            unknown_root_bindings,
11122            total_bindings,
11123        } = actual
11124        else {
11125            panic!("expected live roots")
11126        };
11127        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11128        assert_eq!(unknown_root_bindings, 0);
11129        assert_eq!(
11130            roots.len(),
11131            2,
11132            "root-known arm must retain each canonical root"
11133        );
11134        assert_eq!(
11135            total_bindings,
11136            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11137        );
11138        let counts = roots
11139            .iter()
11140            .map(|root| (root.project_root.clone(), root.bound, root.pending))
11141            .collect::<Vec<_>>();
11142        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11143        expected.sort_by(|a, b| a.0.cmp(&b.0));
11144        assert_eq!(
11145            counts, expected,
11146            "roots must sort by path and count pending separately"
11147        );
11148        handler
11149            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11150            .await
11151            .unwrap();
11152        assert!(pending.await.unwrap().is_empty());
11153    }
11154
11155    #[tokio::test(start_paused = true)]
11156    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11157        let forwarding = Arc::new(ForwardingTable::default());
11158        let handler =
11159            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11160        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11161        hello_via_sink(
11162            &handler,
11163            &target_ctx,
11164            &mut target_rx,
11165            hello_frame("target", PROTOCOL_VERSION, 1),
11166        )
11167        .await;
11168        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11169        let pending = forwarding
11170            .begin_route_bind_relay_for_test(
11171                client_ctx.connection_id,
11172                client_ctx.egress.clone(),
11173                2,
11174                "target",
11175            )
11176            .unwrap();
11177        forwarding
11178            .complete_pending_relay(
11179                target_ctx.connection_id,
11180                pending.corr,
11181                RouteBindRelayOutcome::Accepted,
11182            )
11183            .unwrap();
11184        let actual = query_live_roots(&handler, &target_ctx).await;
11185        let ModuleControlResponseToModule::LiveRoots {
11186            roots,
11187            unknown_root_bindings,
11188            total_bindings,
11189        } = actual
11190        else {
11191            panic!("expected live roots")
11192        };
11193        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11194        assert_eq!(
11195            unknown_root_bindings, 1,
11196            "unknown-root arm must not read as no bindings"
11197        );
11198        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11199        assert_eq!(
11200            total_bindings,
11201            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11202        );
11203    }
11204
11205    /// A module reads HELLO_ACK as its first frame and exits on anything else,
11206    /// so the ack has to be on its outbound queue before the module is
11207    /// routable. The connection loop writes a handler's replies only after the
11208    /// handler returns; this test stops in exactly that gap, runs a real
11209    /// route.open from another connection, and only then writes whatever the
11210    /// HELLO handler returned, the way the loop would. If the ack were still a
11211    /// reply, the route.bind request would reach the module first.
11212    #[tokio::test(start_paused = true)]
11213    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11214        let forwarding = Arc::new(ForwardingTable::default());
11215        let handler =
11216            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11217        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11218        let replies = handler
11219            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11220            .await
11221            .unwrap();
11222        let queued_by_hello = module_rx.len();
11223
11224        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11225        let open_handler = handler.clone();
11226        let open = tokio::spawn(async move {
11227            open_handler
11228                .handle_control_frame(
11229                    &client_ctx,
11230                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11231                )
11232                .await
11233                .unwrap()
11234        });
11235        // Let the route.open run until its route.bind is on the module's queue.
11236        let mut spins = 0;
11237        while module_rx.len() == queued_by_hello {
11238            spins += 1;
11239            assert!(spins < 10_000, "route.open never queued a route.bind");
11240            tokio::task::yield_now().await;
11241        }
11242
11243        // Now the connection loop's half: write the HELLO handler's replies.
11244        for reply in replies {
11245            module_ctx.egress.send(reply).await.unwrap();
11246        }
11247
11248        let first = module_rx.recv().await.unwrap().frame;
11249        assert_eq!(
11250            first.header.ty,
11251            FrameType::HelloAck,
11252            "the first frame a registering module reads must be its HELLO_ACK"
11253        );
11254        assert_eq!(first.header.corr, 7);
11255        let second = module_rx.recv().await.unwrap().frame;
11256        assert_eq!(second.header.ty, FrameType::Request);
11257        assert!(
11258            matches!(
11259                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11260                ModuleControlRequest::RouteBind { .. }
11261            ),
11262            "the route.bind follows the ack"
11263        );
11264        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11265
11266        handler
11267            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11268            .await
11269            .unwrap();
11270        assert!(open.await.unwrap().is_empty());
11271        let _ = client_rx.recv().await.unwrap();
11272    }
11273
11274    #[tokio::test(start_paused = true)]
11275    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11276        let handler = ControlHandler::with_forwarding(
11277            Arc::new(Registry::default()),
11278            Arc::new(ForwardingTable::default()),
11279        );
11280        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11281        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11282        hello_via_sink(
11283            &handler,
11284            &first_ctx,
11285            &mut first_rx,
11286            hello_frame("first", PROTOCOL_VERSION, 1),
11287        )
11288        .await;
11289        hello_via_sink(
11290            &handler,
11291            &second_ctx,
11292            &mut second_rx,
11293            hello_frame("second", PROTOCOL_VERSION, 2),
11294        )
11295        .await;
11296        let root = unique_project_root("second-only");
11297        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11298        let cloned = handler.clone();
11299        let open = tokio::spawn(async move {
11300            cloned
11301                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11302                .await
11303                .unwrap()
11304        });
11305        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11306            .await
11307            .unwrap()
11308            .unwrap();
11309        let first = query_live_roots(&handler, &first_ctx).await;
11310        let second = query_live_roots(&handler, &second_ctx).await;
11311        assert!(
11312            matches!(
11313                first,
11314                ModuleControlResponseToModule::LiveRoots {
11315                    total_bindings: 0,
11316                    ..
11317                }
11318            ),
11319            "cross-module scope must not expose another module's roots"
11320        );
11321        assert!(
11322            matches!(
11323                second,
11324                ModuleControlResponseToModule::LiveRoots {
11325                    total_bindings: 1,
11326                    ..
11327                }
11328            ),
11329            "second module must see its pending route"
11330        );
11331        handler
11332            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11333            .await
11334            .unwrap();
11335        assert!(open.await.unwrap().is_empty());
11336    }
11337
11338    #[tokio::test(start_paused = true)]
11339    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11340        let handler = ControlHandler::with_forwarding(
11341            Arc::new(Registry::default()),
11342            Arc::new(ForwardingTable::default()),
11343        );
11344        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11345        hello_via_sink(
11346            &handler,
11347            &target_ctx,
11348            &mut target_rx,
11349            hello_frame("target", PROTOCOL_VERSION, 1),
11350        )
11351        .await;
11352        let actual = query_live_roots(&handler, &target_ctx).await;
11353        let ModuleControlResponseToModule::LiveRoots {
11354            roots,
11355            unknown_root_bindings,
11356            total_bindings,
11357        } = actual
11358        else {
11359            panic!("expected live roots")
11360        };
11361        assert!(roots.is_empty());
11362        assert_eq!(unknown_root_bindings, 0);
11363        assert_eq!(total_bindings, 0);
11364        assert_eq!(
11365            total_bindings,
11366            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11367        );
11368    }
11369
11370    /// Read the vendored fed corpus rather than hand-building a package.
11371    ///
11372    /// A hand-built object encodes what the test author believed the carrier
11373    /// emits. These vectors are what it actually emits, and one of them exists
11374    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
11375    /// ignores additive unknown fields at the traversal emit terminus."
11376    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11377        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11378            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11379        let text = std::fs::read_to_string(&path)
11380            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11381        let vectors: Vec<(String, Value)> = text
11382            .lines()
11383            .filter(|line| !line.trim().is_empty())
11384            .map(|line| {
11385                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11386                let id = entry["corpus_id"]
11387                    .as_str()
11388                    .expect("every vector carries a corpus_id")
11389                    .to_string();
11390                (id, entry["package"].clone())
11391            })
11392            .collect();
11393        // Pin the count: a corpus that silently shrinks would take its coverage
11394        // with it, and a suite reading N-1 vectors reports the same clean pass
11395        // as one reading N.
11396        assert_eq!(
11397            vectors.len(),
11398            3,
11399            "vendored fed corpus changed size; re-sync from subc-federation"
11400        );
11401
11402        // Pin what makes the corpus DISCRIMINATING, not just present.
11403        //
11404        // The relay test below takes its expected value from the corpus, so the
11405        // corpus supplies the test's power to detect a lossy relay rather than
11406        // its correctness. A relay that dropped unrecognised fields would still
11407        // be caught -- but only by a package carrying fields it does not know.
11408        // Shrink every package to the handful of keys any implementation would
11409        // recognise and the test keeps passing over an input that can no longer
11410        // fail, which is the same clean green as a corpus that shrank away.
11411        //
11412        // So assert the precondition rather than duplicating the packages here:
11413        // at least one vector must carry a field beyond the small common set.
11414        // That is one claim to maintain instead of nine, and it fails loudly if
11415        // a re-sync ever flattens the corpus.
11416        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11417        let richest = vectors
11418            .iter()
11419            .filter_map(|(_, package)| package.as_object())
11420            .map(|object| {
11421                object
11422                    .keys()
11423                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11424                    .count()
11425            })
11426            .max()
11427            .unwrap_or(0);
11428        assert!(
11429            richest >= 2,
11430            "vendored corpus no longer carries a package with unmodelled fields, \
11431             so the relay test can no longer distinguish a verbatim relay from a lossy one"
11432        );
11433
11434        vectors
11435    }
11436
11437    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
11438    ///
11439    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
11440    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
11441    /// three-key object, so a relay that quietly dropped fields it did not
11442    /// recognise would satisfy it. These vectors carry nine keys including ones
11443    /// this crate has no type for, so a typed relay fails here and only here.
11444    #[tokio::test]
11445    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11446        for (corpus_id, package) in fed_admission_facts_vectors() {
11447            let registry = Arc::new(Registry::default());
11448            let forwarding = Arc::new(ForwardingTable::default());
11449            let supervisor = SupervisorHandle::new();
11450            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11451            let handler =
11452                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11453                    .with_supervisor(supervisor)
11454                    .with_admission_facts_config(
11455                        Some("fed".to_string()),
11456                        Some(vec!["target".to_string()]),
11457                    );
11458
11459            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11460            hello_via_sink(
11461                &handler,
11462                &target_ctx,
11463                &mut target_rx,
11464                hello_frame("target", PROTOCOL_VERSION, 1),
11465            )
11466            .await;
11467
11468            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11469            let route_handler = handler.clone();
11470            let expected = package.clone();
11471            let route_task = tokio::spawn(async move {
11472                route_handler
11473                    .handle_control_frame(
11474                        &client_ctx,
11475                        route_open_frame_with_admission_facts(
11476                            20,
11477                            "target",
11478                            unique_project_root("admission-facts"),
11479                            Some(subc_control::ConsumerIdentity {
11480                                module_id: "fed".to_string(),
11481                                launch_nonce: "fed-nonce".to_string(),
11482                            }),
11483                            Some(package),
11484                        ),
11485                    )
11486                    .await
11487                    .unwrap()
11488            });
11489
11490            let bind_frame = target_rx.recv().await.unwrap();
11491            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11492            let ModuleControlRequest::RouteBind {
11493                admission_facts, ..
11494            } = bind
11495            else {
11496                panic!("{corpus_id}: expected route.bind")
11497            };
11498            assert_eq!(
11499                admission_facts,
11500                Some(expected),
11501                "{corpus_id}: relay must not add, drop or reshape any field"
11502            );
11503
11504            handler
11505                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11506                .await
11507                .unwrap();
11508            route_task.await.unwrap();
11509        }
11510    }
11511
11512    #[tokio::test]
11513    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11514        let registry = Arc::new(Registry::default());
11515        let forwarding = Arc::new(ForwardingTable::default());
11516        let supervisor = SupervisorHandle::new();
11517        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11518        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11519        let handler =
11520            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11521                .with_supervisor(supervisor)
11522                .with_admission_facts_config(
11523                    Some("fed".to_string()),
11524                    Some(vec!["target".to_string()]),
11525                );
11526
11527        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11528        hello_via_sink(
11529            &handler,
11530            &target_ctx,
11531            &mut target_rx,
11532            hello_frame("target", PROTOCOL_VERSION, 1),
11533        )
11534        .await;
11535        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11536        hello_via_sink(
11537            &handler,
11538            &other_ctx,
11539            &mut other_rx,
11540            hello_frame("other", PROTOCOL_VERSION, 2),
11541        )
11542        .await;
11543
11544        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11545        let expected_facts = facts.clone();
11546        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11547        let route_handler = handler.clone();
11548        let route_task = tokio::spawn(async move {
11549            route_handler
11550                .handle_control_frame(
11551                    &client_ctx,
11552                    route_open_frame_with_admission_facts(
11553                        10,
11554                        "target",
11555                        unique_project_root("admission-facts"),
11556                        Some(subc_control::ConsumerIdentity {
11557                            module_id: "fed".to_string(),
11558                            launch_nonce: "fed-nonce".to_string(),
11559                        }),
11560                        Some(facts.clone()),
11561                    ),
11562                )
11563                .await
11564                .unwrap()
11565        });
11566        let bind_frame = target_rx.recv().await.unwrap();
11567        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11568        let ModuleControlRequest::RouteBind {
11569            admission_facts, ..
11570        } = bind
11571        else {
11572            panic!("expected route.bind")
11573        };
11574        assert_eq!(admission_facts, Some(expected_facts));
11575        handler
11576            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11577            .await
11578            .unwrap();
11579        assert!(route_task.await.unwrap().is_empty());
11580        assert!(matches!(
11581            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11582                .unwrap(),
11583            ClientControlResponse::RouteOpen { .. }
11584        ));
11585
11586        let direct = handler
11587            .handle_control_frame(
11588                &route_ctx(ConnectionId::new(73)).0,
11589                route_open_frame_with_admission_facts(
11590                    11,
11591                    "target",
11592                    unique_project_root("admission-facts"),
11593                    None,
11594                    Some(json!({"x": 1})),
11595                ),
11596            )
11597            .await
11598            .unwrap();
11599        assert_eq!(
11600            parse_error(&direct[0])["code"],
11601            "admission_facts_not_permitted"
11602        );
11603
11604        let different_reserved = handler
11605            .handle_control_frame(
11606                &route_ctx(ConnectionId::new(77)).0,
11607                route_open_frame_with_admission_facts(
11608                    15,
11609                    "target",
11610                    unique_project_root("admission-facts"),
11611                    Some(subc_control::ConsumerIdentity {
11612                        module_id: "other".to_string(),
11613                        launch_nonce: "other-nonce".to_string(),
11614                    }),
11615                    Some(json!({"x": 1})),
11616                ),
11617            )
11618            .await
11619            .unwrap();
11620        assert_eq!(
11621            parse_error(&different_reserved[0])["code"],
11622            "admission_facts_not_permitted"
11623        );
11624
11625        let other_target = handler
11626            .handle_control_frame(
11627                &route_ctx(ConnectionId::new(74)).0,
11628                route_open_frame_with_admission_facts(
11629                    12,
11630                    "other",
11631                    unique_project_root("admission-facts"),
11632                    Some(subc_control::ConsumerIdentity {
11633                        module_id: "fed".to_string(),
11634                        launch_nonce: "fed-nonce".to_string(),
11635                    }),
11636                    Some(json!({"x": 1})),
11637                ),
11638            )
11639            .await
11640            .unwrap();
11641        assert_eq!(
11642            parse_error(&other_target[0])["code"],
11643            "admission_facts_target_not_allowed"
11644        );
11645
11646        let nonexistent = handler
11647            .handle_control_frame(
11648                &route_ctx(ConnectionId::new(75)).0,
11649                route_open_frame_with_admission_facts(
11650                    13,
11651                    "missing",
11652                    unique_project_root("admission-facts"),
11653                    None,
11654                    Some(json!({"x": 1})),
11655                ),
11656            )
11657            .await
11658            .unwrap();
11659        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
11660
11661        let described = handler
11662            .handle_control_frame(
11663                &route_ctx(ConnectionId::new(76)).0,
11664                Frame::build(
11665                    FrameType::Request,
11666                    control_flags(),
11667                    0,
11668                    0,
11669                    14,
11670                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11671                )
11672                .unwrap(),
11673            )
11674            .await
11675            .unwrap();
11676        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11677            serde_json::from_slice(&described[0].body).unwrap()
11678        else {
11679            panic!("expected server.describe response")
11680        };
11681        assert!(capabilities
11682            .iter()
11683            .any(|cap| cap == "admission_facts_relay_v1"));
11684    }
11685
11686    #[tokio::test]
11687    async fn admission_facts_without_configured_carrier_are_rejected() {
11688        let registry = Arc::new(Registry::default());
11689        let forwarding = Arc::new(ForwardingTable::default());
11690        let handler = ControlHandler::with_forwarding(registry, forwarding);
11691        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
11692        hello_via_sink(
11693            &handler,
11694            &target_ctx,
11695            &mut target_rx,
11696            hello_frame("target", PROTOCOL_VERSION, 1),
11697        )
11698        .await;
11699
11700        let responses = handler
11701            .handle_control_frame(
11702                &route_ctx(ConnectionId::new(79)).0,
11703                route_open_frame_with_admission_facts(
11704                    16,
11705                    "target",
11706                    unique_project_root("admission-facts"),
11707                    None,
11708                    Some(json!({"x": 1})),
11709                ),
11710            )
11711            .await
11712            .unwrap();
11713        assert_eq!(
11714            parse_error(&responses[0])["code"],
11715            "admission_facts_not_permitted"
11716        );
11717    }
11718
11719    #[tokio::test]
11720    async fn route_open_relays_consumer_capabilities_verbatim() {
11721        let registry = Arc::new(Registry::default());
11722        let forwarding = Arc::new(ForwardingTable::default());
11723        let handler =
11724            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11725        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
11726        hello_via_sink(
11727            &handler,
11728            &module_ctx,
11729            &mut module_rx,
11730            hello_frame("aft", PROTOCOL_VERSION, 7),
11731        )
11732        .await;
11733
11734        let expected = vec!["elicitation".to_string(), "roots".to_string()];
11735        let expected_for_request = expected.clone();
11736        let project_root = unique_project_root("consumer-capabilities-present");
11737        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
11738        let route_handler = handler.clone();
11739        let route_task = tokio::spawn(async move {
11740            route_handler
11741                .handle_control_frame(
11742                    &client_ctx,
11743                    route_open_frame_with_consumer_capabilities(
11744                        401,
11745                        "aft",
11746                        project_root,
11747                        Some(expected_for_request),
11748                    ),
11749                )
11750                .await
11751                .unwrap()
11752        });
11753        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11754            .await
11755            .unwrap()
11756            .unwrap();
11757        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11758        let ModuleControlRequest::RouteBind {
11759            consumer_capabilities,
11760            ..
11761        } = bind
11762        else {
11763            panic!("expected route.bind request, got {bind:?}");
11764        };
11765        assert_eq!(consumer_capabilities, Some(expected.clone()));
11766
11767        handler
11768            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11769            .await
11770            .unwrap();
11771        let route_response = route_task.await.unwrap();
11772        assert!(route_response.is_empty());
11773        let published = client_rx.recv().await.unwrap();
11774        assert!(matches!(
11775            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11776            ClientControlResponse::RouteOpen { .. }
11777        ));
11778    }
11779
11780    #[tokio::test]
11781    async fn route_open_without_consumer_capabilities_relays_none() {
11782        let registry = Arc::new(Registry::default());
11783        let forwarding = Arc::new(ForwardingTable::default());
11784        let handler =
11785            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11786        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11787        hello_via_sink(
11788            &handler,
11789            &module_ctx,
11790            &mut module_rx,
11791            hello_frame("aft", PROTOCOL_VERSION, 7),
11792        )
11793        .await;
11794
11795        let project_root = unique_project_root("consumer-capabilities-absent");
11796        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11797        let route_handler = handler.clone();
11798        let route_task = tokio::spawn(async move {
11799            route_handler
11800                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11801                .await
11802                .unwrap()
11803        });
11804        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11805            .await
11806            .unwrap()
11807            .unwrap();
11808        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11809        let ModuleControlRequest::RouteBind {
11810            consumer_capabilities,
11811            ..
11812        } = bind
11813        else {
11814            panic!("expected route.bind request, got {bind:?}");
11815        };
11816        assert_eq!(consumer_capabilities, None);
11817
11818        handler
11819            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11820            .await
11821            .unwrap();
11822        let route_response = route_task.await.unwrap();
11823        assert!(route_response.is_empty());
11824        let published = client_rx.recv().await.unwrap();
11825        assert!(matches!(
11826            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11827            ClientControlResponse::RouteOpen { .. }
11828        ));
11829    }
11830
11831    /// Opens a route to a freshly registered `aft` with `sent` as the
11832    /// route.open's role_versions, acks the bind, and returns the role_versions
11833    /// the module's bind carried.
11834    async fn bind_role_versions_for(
11835        sent: Option<BTreeMap<String, String>>,
11836        connection: u64,
11837    ) -> Option<BTreeMap<String, String>> {
11838        let registry = Arc::new(Registry::default());
11839        let forwarding = Arc::new(ForwardingTable::default());
11840        let handler =
11841            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11842        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11843        hello_via_sink(
11844            &handler,
11845            &module_ctx,
11846            &mut module_rx,
11847            hello_frame("aft", PROTOCOL_VERSION, 7),
11848        )
11849        .await;
11850        let project_root = unique_project_root("role-versions");
11851        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11852        let route_handler = handler.clone();
11853        let route_task = tokio::spawn(async move {
11854            route_handler
11855                .handle_control_frame(
11856                    &client_ctx,
11857                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
11858                )
11859                .await
11860                .unwrap()
11861        });
11862        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11863            .await
11864            .expect("a well-formed route.open reaches the module as a bind")
11865            .unwrap();
11866        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11867        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11868            panic!("expected route.bind request, got {bind:?}");
11869        };
11870        handler
11871            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11872            .await
11873            .unwrap();
11874        assert!(route_task.await.unwrap().is_empty());
11875        let published = client_rx.recv().await.unwrap();
11876        assert!(matches!(
11877            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11878            ClientControlResponse::RouteOpen { .. }
11879        ));
11880        role_versions
11881    }
11882
11883    #[tokio::test]
11884    async fn route_open_relays_role_versions_verbatim() {
11885        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11886        assert_eq!(
11887            bind_role_versions_for(Some(sent.clone()), 141).await,
11888            Some(sent)
11889        );
11890    }
11891
11892    /// An empty map declares nothing, so the provider sees no field rather
11893    /// than an empty object it would have to treat as a second "none".
11894    #[tokio::test]
11895    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11896        assert_eq!(bind_role_versions_for(None, 143).await, None);
11897        assert_eq!(
11898            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11899            None
11900        );
11901    }
11902
11903    #[tokio::test]
11904    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11905        let registry = Arc::new(Registry::default());
11906        let forwarding = Arc::new(ForwardingTable::default());
11907        let handler =
11908            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11909        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11910        hello_via_sink(
11911            &handler,
11912            &module_ctx,
11913            &mut module_rx,
11914            hello_frame("aft", PROTOCOL_VERSION, 7),
11915        )
11916        .await;
11917
11918        let nine: BTreeMap<String, String> = (0..9)
11919            .map(|index| (format!("role-{index}"), "v1".to_string()))
11920            .collect();
11921        for (label, malformed) in [
11922            (
11923                "invalid role name",
11924                role_versions(&[("Tool_Provider", "v1")]),
11925            ),
11926            ("invalid version", role_versions(&[("tool-provider", "v0")])),
11927            ("nine entries", nine),
11928        ] {
11929            // A refused open answers at once; one that reached the module
11930            // would wait for its bind ack and trip this timeout.
11931            let responses = tokio::time::timeout(
11932                Duration::from_secs(1),
11933                handler.handle_control_frame(
11934                    &route_ctx(ConnectionId::new(148)).0,
11935                    route_open_frame_with_role_versions(
11936                        404,
11937                        "aft",
11938                        unique_project_root("role-versions-malformed"),
11939                        Some(malformed),
11940                    ),
11941                ),
11942            )
11943            .await
11944            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11945            .unwrap();
11946            assert_eq!(responses.len(), 1, "{label}");
11947            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11948            let error = parse_error(&responses[0]);
11949            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11950            assert_eq!(
11951                error["detail"]["field"], "role_versions",
11952                "{label}: {error}"
11953            );
11954            assert!(
11955                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11956                "{label}: a malformed declaration is terminal"
11957            );
11958            assert!(
11959                module_rx.try_recv().is_err(),
11960                "{label}: the module must never see a bind"
11961            );
11962        }
11963    }
11964
11965    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
11966    /// consumer can tell this daemon forwards the field from one that would
11967    /// drop it.
11968    #[tokio::test]
11969    async fn route_role_versions_capability_is_advertised() {
11970        let handler = ControlHandler::new(Arc::new(Registry::default()));
11971        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11972        let ack = hello_via_sink(
11973            &handler,
11974            &ctx,
11975            &mut rx,
11976            hello_frame("m", PROTOCOL_VERSION, 1),
11977        )
11978        .await;
11979        let ack = parse_ack(&ack);
11980        assert!(
11981            ack.subc_capabilities
11982                .iter()
11983                .any(|c| c == "route-role-versions/v1"),
11984            "{:?}",
11985            ack.subc_capabilities
11986        );
11987
11988        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11989        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11990        let reply = handler
11991            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11992            .await
11993            .unwrap()
11994            .pop()
11995            .unwrap();
11996        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11997            serde_json::from_slice(&reply.body).unwrap()
11998        else {
11999            panic!("not a server.describe reply");
12000        };
12001        assert!(
12002            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12003            "{capabilities:?}"
12004        );
12005    }
12006
12007    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
12008    /// catalog entry without `events` (the module declares none) from a daemon
12009    /// that never reports declared events.
12010    #[tokio::test]
12011    async fn catalog_events_capability_is_advertised() {
12012        let handler = ControlHandler::new(Arc::new(Registry::default()));
12013        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12014        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12015        let reply = handler
12016            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12017            .await
12018            .unwrap()
12019            .pop()
12020            .unwrap();
12021        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12022            serde_json::from_slice(&reply.body).unwrap()
12023        else {
12024            panic!("not a server.describe reply");
12025        };
12026        assert!(
12027            capabilities
12028                .iter()
12029                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12030            "{capabilities:?}"
12031        );
12032    }
12033
12034    #[tokio::test]
12035    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12036        let handler = ControlHandler::new(Arc::new(Registry::default()));
12037        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12038        let ack = hello_via_sink(
12039            &handler,
12040            &module,
12041            &mut module_rx,
12042            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12043        )
12044        .await;
12045        let ack = parse_ack(&ack);
12046        assert!(
12047            ack.subc_capabilities
12048                .iter()
12049                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12050            "{:?}",
12051            ack.subc_capabilities
12052        );
12053        assert!(ack
12054            .subc_capabilities
12055            .iter()
12056            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12057
12058        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12059        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12060        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12061        let reply = handler
12062            .handle_control_frame(&client, frame)
12063            .await
12064            .unwrap()
12065            .pop()
12066            .unwrap();
12067        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12068            serde_json::from_slice(&reply.body).unwrap()
12069        else {
12070            panic!("not a server.describe reply");
12071        };
12072        assert!(
12073            capabilities
12074                .iter()
12075                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12076            "{capabilities:?}"
12077        );
12078        assert!(capabilities
12079            .iter()
12080            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12081    }
12082
12083    #[tokio::test]
12084    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12085        let handler = ControlHandler::new(Arc::new(Registry::default()));
12086        let (module, mut rx) = route_ctx(ConnectionId::new(156));
12087        let ack = hello_via_sink(
12088            &handler,
12089            &module,
12090            &mut rx,
12091            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12092        )
12093        .await;
12094        assert!(parse_ack(&ack)
12095            .subc_capabilities
12096            .iter()
12097            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12098        assert!(parse_ack(&ack)
12099            .subc_capabilities
12100            .iter()
12101            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12102        let (client, _rx) = route_ctx(ConnectionId::new(157));
12103        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12104        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12105        let reply = handler
12106            .handle_control_frame(&client, frame)
12107            .await
12108            .unwrap()
12109            .pop()
12110            .unwrap();
12111        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12112            serde_json::from_slice(&reply.body).unwrap()
12113        else {
12114            panic!("server.describe reply")
12115        };
12116        assert!(capabilities
12117            .iter()
12118            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12119        assert!(capabilities
12120            .iter()
12121            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12122    }
12123
12124    #[tokio::test]
12125    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12126        let handler = ControlHandler::new(Arc::new(Registry::default()));
12127        let (module, mut rx) = route_ctx(ConnectionId::new(158));
12128        let ack = hello_via_sink(
12129            &handler,
12130            &module,
12131            &mut rx,
12132            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12133        )
12134        .await;
12135        assert!(parse_ack(&ack)
12136            .subc_capabilities
12137            .iter()
12138            .any(|cap| cap == "scope-workdir/v1"));
12139        let (client, _rx) = route_ctx(ConnectionId::new(159));
12140        let frame = Frame::build(
12141            FrameType::Request,
12142            control_flags(),
12143            0,
12144            0,
12145            5,
12146            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12147        )
12148        .unwrap();
12149        let reply = handler
12150            .handle_control_frame(&client, frame)
12151            .await
12152            .unwrap()
12153            .remove(0);
12154        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12155            serde_json::from_slice(&reply.body).unwrap()
12156        else {
12157            panic!("server.describe reply")
12158        };
12159        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12160    }
12161
12162    #[tokio::test]
12163    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12164        let handler = ControlHandler::new(Arc::new(Registry::default()));
12165        let (module, mut rx) = route_ctx(ConnectionId::new(154));
12166        let ack = hello_via_sink(
12167            &handler,
12168            &module,
12169            &mut rx,
12170            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12171        )
12172        .await;
12173        assert!(parse_ack(&ack)
12174            .subc_capabilities
12175            .iter()
12176            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12177        let (client, _rx) = route_ctx(ConnectionId::new(155));
12178        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12179        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12180        let reply = handler
12181            .handle_control_frame(&client, frame)
12182            .await
12183            .unwrap()
12184            .pop()
12185            .unwrap();
12186        let ClientControlResponse::ServerDescribe { capabilities, .. } =
12187            serde_json::from_slice(&reply.body).unwrap()
12188        else {
12189            panic!("not a server.describe reply")
12190        };
12191        assert!(capabilities
12192            .iter()
12193            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12194    }
12195
12196    #[tokio::test]
12197    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12198        let registry = Arc::new(Registry::default());
12199        let forwarding = Arc::new(ForwardingTable::default());
12200        let handler =
12201            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12202                .with_health_probe_timeout(Duration::from_secs(5));
12203        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12204        hello_via_sink(
12205            &handler,
12206            &module_ctx,
12207            &mut module_rx,
12208            non_routable_hello_frame_with_control_ops(
12209                "mcp",
12210                300,
12211                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12212            ),
12213        )
12214        .await;
12215        assert!(registry
12216            .get_module("mcp")
12217            .unwrap()
12218            .unwrap()
12219            .manifest
12220            .provides
12221            .is_empty());
12222
12223        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12224        let route_response = handler
12225            .handle_control_frame(
12226                &route_client_ctx,
12227                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12228            )
12229            .await
12230            .unwrap();
12231        assert_eq!(route_response[0].header.ty, FrameType::Error);
12232        assert_eq!(
12233            parse_error(&route_response[0])["code"],
12234            "target_unavailable"
12235        );
12236        assert!(parse_error(&route_response[0])["message"]
12237            .as_str()
12238            .unwrap()
12239            .contains("does not provide the requested target"));
12240        assert!(module_rx.try_recv().is_err());
12241
12242        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
12243        let health_handler = handler.clone();
12244        let health_task = tokio::spawn(async move {
12245            health_handler
12246                .handle_control_frame(
12247                    &health_client_ctx,
12248                    supervisor_health_probe_frame(302, "mcp"),
12249                )
12250                .await
12251                .unwrap()
12252        });
12253        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12254            .await
12255            .unwrap()
12256            .unwrap();
12257        assert_eq!(
12258            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
12259            ModuleControlRequest::HealthCheck {}
12260        );
12261        handler
12262            .handle_control_frame(
12263                &module_ctx,
12264                health_response(health_frame.header.corr, HealthStatus::Ok),
12265            )
12266            .await
12267            .unwrap();
12268        let health_response = health_task.await.unwrap();
12269        assert_eq!(health_response[0].header.ty, FrameType::Response);
12270        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
12271            ClientControlResponse::SupervisorHealthProbe {
12272                module_id, status, ..
12273            } => {
12274                assert_eq!(module_id, "mcp");
12275                assert_eq!(status, HealthStatus::Ok);
12276            }
12277            other => panic!("unexpected health response: {other:?}"),
12278        }
12279
12280        // Exercise the forwarding cleanup path directly while leaving the registry
12281        // advertisement in place. If cleanup leaves a stale control sink behind,
12282        // the next probe will enqueue onto it and wait for the long probe timeout
12283        // instead of returning an immediate no-connection error.
12284        forwarding
12285            .cleanup_connection(module_ctx.connection_id)
12286            .unwrap();
12287        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
12288        let cleanup_response = tokio::time::timeout(
12289            Duration::from_millis(200),
12290            handler.handle_control_frame(
12291                &cleanup_probe_ctx,
12292                supervisor_health_probe_frame(303, "mcp"),
12293            ),
12294        )
12295        .await
12296        .expect("probe should fail immediately when the control lane is gone")
12297        .unwrap();
12298        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
12299        assert_eq!(
12300            parse_error(&cleanup_response[0])["code"],
12301            "target_unavailable"
12302        );
12303        assert!(parse_error(&cleanup_response[0])["message"]
12304            .as_str()
12305            .unwrap()
12306            .contains("no module connection"));
12307
12308        handler
12309            .cleanup_connection(module_ctx.connection_id)
12310            .unwrap();
12311    }
12312
12313    #[tokio::test]
12314    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
12315        let registry = Arc::new(Registry::default());
12316        let supervisor_handle = SupervisorHandle::new();
12317        let supervisor =
12318            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12319                .with_handle(supervisor_handle.clone())
12320                .with_connection_file_path(
12321                    std::env::temp_dir()
12322                        .join(format!("subc-route-open-warming-{}", std::process::id())),
12323                );
12324        let module = supervisor
12325            .supervise_configured(
12326                ModuleSpec {
12327                    module_id: "warming".to_string(),
12328                    program: fake_aft_stub_path(),
12329                    args: Vec::new(),
12330                    env: Vec::new(),
12331                    reserved: false,
12332                    reserved_prefixes: Vec::new(),
12333                    protocol: ModuleProtocol::Subc,
12334                    overlap: Default::default(),
12335                },
12336                true,
12337            )
12338            .unwrap();
12339        assert_eq!(module.state().unwrap(), ModuleState::Running);
12340
12341        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12342        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
12343        let response = handler
12344            .handle_control_frame(
12345                &ctx,
12346                route_open_frame(304, "warming", unique_project_root("warming")),
12347            )
12348            .await
12349            .unwrap();
12350        module.stop().await.unwrap();
12351
12352        assert_eq!(response[0].header.ty, FrameType::Error);
12353        let error = parse_error(&response[0]);
12354        assert_eq!(error["code"], "module_warming");
12355        assert!(error["message"]
12356            .as_str()
12357            .unwrap()
12358            .contains("state=running, enabled=true, live=false"));
12359    }
12360
12361    #[test]
12362    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
12363        let handler = ControlHandler::new(Arc::new(Registry::default()));
12364        let capture = EventCapture::default();
12365        let _subscriber =
12366            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12367        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
12368        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
12369        let pending = (0..limit).collect::<Vec<_>>();
12370        let response = handler
12371            .route_open_capacity_refusal(
12372                &ctx,
12373                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
12374                "busy",
12375                pending.len(),
12376                limit,
12377            )
12378            .unwrap();
12379        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12380        let event = capture
12381            .events()
12382            .into_iter()
12383            .find(|event| {
12384                event.target == "control"
12385                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12386            })
12387            .expect("connection admission refusal event");
12388        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12389        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12390    }
12391
12392    #[test]
12393    fn route_open_target_cap_logs_admission_reason_and_capacity() {
12394        let handler = ControlHandler::new(Arc::new(Registry::default()));
12395        let capture = EventCapture::default();
12396        let _subscriber =
12397            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12398        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12399        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12400        let guards = (0..limit)
12401            .map(|_| {
12402                handler
12403                    .route_bind_concurrency
12404                    .try_admit("busy", limit)
12405                    .unwrap()
12406            })
12407            .collect::<Vec<_>>();
12408        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12409            Err(in_flight) => in_flight,
12410            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12411        };
12412        let response = handler
12413            .route_open_target_capacity_refusal(
12414                &ctx,
12415                &route_open_frame(397, "busy", unique_project_root("target-cap")),
12416                "busy",
12417                in_flight,
12418            )
12419            .unwrap();
12420        assert_eq!(parse_error(&response)["code"], "target_unavailable");
12421        let event = capture
12422            .events()
12423            .into_iter()
12424            .find(|event| {
12425                event.target == "control"
12426                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12427            })
12428            .expect("target admission refusal event");
12429        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12430        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12431        drop(guards);
12432    }
12433
12434    #[tokio::test]
12435    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
12436        let registry = Arc::new(Registry::default());
12437        let handle = SupervisorHandle::new();
12438        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
12439            .with_handle(handle.clone());
12440        supervisor
12441            .supervise_configured(
12442                crate::ModuleSpec {
12443                    module_id: "drift".into(),
12444                    program: PathBuf::from("disabled-module"),
12445                    args: Vec::new(),
12446                    env: Vec::new(),
12447                    reserved: false,
12448                    reserved_prefixes: Vec::new(),
12449                    protocol: ModuleProtocol::Subc,
12450                    overlap: Default::default(),
12451                },
12452                false,
12453            )
12454            .unwrap();
12455        let handler = ControlHandler::new(registry).with_supervisor(handle);
12456        for configured in [None, Some(false), Some(true)] {
12457            handler.capability_evaluator.configure(
12458                configured.map(|enabled| ("drift".to_string(), enabled)),
12459                BTreeMap::new(),
12460            );
12461            let frame =
12462                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12463            let replies = handler
12464                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
12465                .await
12466                .unwrap();
12467            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12468            assert!(body.get("configured_enabled").is_none());
12469            assert_eq!(body["op"], "supervisor.ack");
12470            let frame =
12471                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12472            let replies = handler.handle_supervisor_list(frame).await.unwrap();
12473            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
12474            assert_eq!(
12475                body["modules"][0].get("configured_enabled"),
12476                configured.map(Value::Bool).as_ref()
12477            );
12478        }
12479    }
12480
12481    #[tokio::test(flavor = "current_thread")]
12482    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12483        let handler = ControlHandler::new(Arc::new(Registry::default()));
12484        let capture = EventCapture::default();
12485        let _subscriber =
12486            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12487        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12488        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12489
12490        let _ = handler
12491            .handle_client_control_request(
12492                &ctx,
12493                frame,
12494                ClientControlRequest::SupervisorSetEnabled {
12495                    module_id: "broca".to_string(),
12496                    enabled: false,
12497                },
12498                &mut None,
12499            )
12500            .await
12501            .unwrap();
12502
12503        let events = capture
12504            .events()
12505            .into_iter()
12506            .filter(|event| {
12507                event.target == "control"
12508                    && event.fields.get("message").map(String::as_str)
12509                        == Some("supervisor request received")
12510            })
12511            .collect::<Vec<_>>();
12512        assert_eq!(events.len(), 1, "one supervisor request log line");
12513        let event = &events[0];
12514        assert_eq!(event.level, tracing::Level::INFO);
12515        assert_eq!(
12516            event.fields.get("op"),
12517            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12518        );
12519        assert_eq!(
12520            event.fields.get("module_id"),
12521            Some(&"\"broca\"".to_string())
12522        );
12523        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12524        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12525        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12526    }
12527
12528    #[tokio::test(flavor = "current_thread")]
12529    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12530        let registry = Arc::new(Registry::default());
12531        let module_connection = ConnectionId::new(303);
12532        registry
12533            .register_with_control_ops(
12534                manifest("aft", PROTOCOL_VERSION),
12535                PROTOCOL_VERSION,
12536                module_connection,
12537                module_baseline_control_ops(),
12538            )
12539            .unwrap();
12540        let handler = ControlHandler::new(registry);
12541        let capture = EventCapture::default();
12542        let _subscriber =
12543            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12544        let (ctx, _rx) = route_ctx(module_connection);
12545        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12546
12547        let _ = handler
12548            .handle_client_control_request(
12549                &ctx,
12550                frame,
12551                ClientControlRequest::SupervisorSetEnabled {
12552                    module_id: "broca".to_string(),
12553                    enabled: false,
12554                },
12555                &mut None,
12556            )
12557            .await
12558            .unwrap();
12559
12560        let callers = capture
12561            .events()
12562            .into_iter()
12563            .filter(|event| {
12564                event.fields.get("message").map(String::as_str)
12565                    == Some("supervisor request received")
12566            })
12567            .map(|event| event.fields.get("caller").cloned())
12568            .collect::<Vec<_>>();
12569        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12570    }
12571
12572    #[tokio::test(flavor = "current_thread")]
12573    async fn supervisor_list_does_not_log_request_received() {
12574        let handler = ControlHandler::new(Arc::new(Registry::default()));
12575        let capture = EventCapture::default();
12576        let _subscriber =
12577            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12578        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12579        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12580
12581        handler
12582            .handle_client_control_request(
12583                &ctx,
12584                frame,
12585                ClientControlRequest::SupervisorList {},
12586                &mut None,
12587            )
12588            .await
12589            .unwrap();
12590
12591        assert!(capture.events().into_iter().all(|event| {
12592            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12593        }));
12594    }
12595
12596    #[tokio::test(flavor = "current_thread")]
12597    async fn supervisor_rescan_preview_does_not_log_request_received() {
12598        let handler = ControlHandler::new(Arc::new(Registry::default()));
12599        let capture = EventCapture::default();
12600        let _subscriber =
12601            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12602        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
12603        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12604
12605        handler
12606            .handle_client_control_request(
12607                &ctx,
12608                frame,
12609                ClientControlRequest::SupervisorRescan { preview: true },
12610                &mut None,
12611            )
12612            .await
12613            .unwrap();
12614
12615        assert!(capture.events().into_iter().all(|event| {
12616            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12617        }));
12618    }
12619
12620    struct ApproveOperator;
12621    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
12622        fn prompt(
12623            &self,
12624            _: &str,
12625            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
12626        ) -> crate::operator_confirm::ProviderResult {
12627            crate::operator_confirm::ProviderResult::Approved
12628        }
12629    }
12630
12631    #[tokio::test]
12632    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
12633        assert_eq!(
12634            ControlHandler::default().privileged_operator_prompt,
12635            PrivilegedOperatorPrompt::Log
12636        );
12637        for (initial, changed, running) in [
12638            ("log", "enforce", PrivilegedOperatorPrompt::Log),
12639            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
12640        ] {
12641            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
12642            let path = dir.join("subc.jsonc");
12643            std::fs::write(
12644                &path,
12645                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
12646            )
12647            .unwrap();
12648            let config = crate::daemon_config::load(&path).unwrap().unwrap();
12649            let registry = Arc::new(Registry::default());
12650            let handle = SupervisorHandle::new();
12651            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
12652                .with_handle(handle.clone());
12653            let handler = ControlHandler::new(registry)
12654                .with_supervisor(handle)
12655                .with_privileged_operator_prompt(config.privileged_operator_prompt)
12656                .with_supervisor_rescan(supervisor, &path, config.port);
12657            handler.forwarding.operator_confirms().configure(
12658                Arc::new(ApproveOperator),
12659                crate::operator_confirm::OperatorLimits::default(),
12660            );
12661            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
12662
12663            for (disk_mode, needs_restart) in [
12664                (Some(initial), false),
12665                (Some(changed), true),
12666                (Some(changed), true),
12667                (None, running == PrivilegedOperatorPrompt::Enforce),
12668                (Some(initial), false),
12669            ] {
12670                let mut doc = json!({"version":1});
12671                if let Some(mode) = disk_mode {
12672                    doc["privileged_operator_prompt"] = json!(mode);
12673                }
12674                std::fs::write(&path, doc.to_string()).unwrap();
12675                for preview in [true, false] {
12676                    let request = ClientControlRequest::SupervisorRescan { preview };
12677                    let frame = Frame::build(
12678                        FrameType::Request,
12679                        control_flags(),
12680                        0,
12681                        0,
12682                        1,
12683                        serde_json::to_vec(&request).unwrap(),
12684                    )
12685                    .unwrap();
12686                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12687                    assert_eq!(frames.len(), 1);
12688                    let ClientControlResponse::SupervisorRescan { result } =
12689                        serde_json::from_slice(&frames[0].body).unwrap()
12690                    else {
12691                        panic!("expected a rescan response");
12692                    };
12693                    let expected = if needs_restart {
12694                        vec!["privileged_operator_prompt"]
12695                    } else {
12696                        Vec::new()
12697                    };
12698                    assert_eq!(result.restart_required, expected);
12699                    assert_eq!(handler.privileged_operator_prompt, running);
12700                    assert_eq!(
12701                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
12702                        running
12703                    );
12704                }
12705            }
12706        }
12707    }
12708
12709    #[tokio::test]
12710    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
12711        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
12712        let path = dir.join("subc.jsonc");
12713        let registry = Arc::new(Registry::default());
12714        let handle = SupervisorHandle::new();
12715        let supervisor =
12716            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
12717        let handler = ControlHandler::new(registry)
12718            .with_supervisor(handle)
12719            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
12720            .with_supervisor_rescan(supervisor, &path, None);
12721        handler.forwarding.operator_confirms().configure(
12722            Arc::new(ApproveOperator),
12723            crate::operator_confirm::OperatorLimits::default(),
12724        );
12725        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
12726        for preview in [true, false] {
12727            std::fs::write(
12728                &path,
12729                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
12730            )
12731            .unwrap();
12732            let request = ClientControlRequest::SupervisorRescan { preview };
12733            let frame = Frame::build(
12734                FrameType::Request,
12735                control_flags(),
12736                0,
12737                0,
12738                1,
12739                serde_json::to_vec(&request).unwrap(),
12740            )
12741            .unwrap();
12742            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12743            assert_eq!(frames.len(), 1);
12744            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
12745            assert_eq!(
12746                handler.privileged_operator_prompt,
12747                PrivilegedOperatorPrompt::Enforce
12748            );
12749        }
12750    }
12751
12752    /// One wire code has several senders, so the refusal line names the check
12753    /// that refused. This drives the shared refusal path for ordinary refusals
12754    /// with an unregistered
12755    /// target and requires the branch label on the event.
12756    #[tokio::test]
12757    async fn route_open_refusal_names_the_check_that_refused() {
12758        let handler = ControlHandler::new(Arc::new(Registry::default()));
12759        let capture = EventCapture::default();
12760        let _subscriber =
12761            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12762        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12763        let response = handler
12764            .handle_control_frame(
12765                &ctx,
12766                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
12767            )
12768            .await
12769            .unwrap();
12770
12771        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12772        let event = capture
12773            .events()
12774            .into_iter()
12775            .find(|event| {
12776                event.target == "control"
12777                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12778            })
12779            .expect("route.open refusal event");
12780        assert_eq!(
12781            event.fields.get("reason"),
12782            Some(&"\"not_registered\"".to_string())
12783        );
12784    }
12785
12786    #[tokio::test]
12787    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
12788        let registry = Arc::new(Registry::default());
12789        let supervisor_handle = SupervisorHandle::new();
12790        let supervisor =
12791            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12792                .with_handle(supervisor_handle.clone())
12793                .with_connection_file_path(std::env::temp_dir().join(format!(
12794                    "subc-route-open-refusal-info-{}",
12795                    std::process::id()
12796                )));
12797        let module = supervisor
12798            .supervise_configured(
12799                ModuleSpec {
12800                    module_id: "warming".to_string(),
12801                    program: fake_aft_stub_path(),
12802                    args: Vec::new(),
12803                    env: Vec::new(),
12804                    reserved: false,
12805                    reserved_prefixes: Vec::new(),
12806                    protocol: ModuleProtocol::Subc,
12807                    overlap: Default::default(),
12808                },
12809                true,
12810            )
12811            .unwrap();
12812        assert_eq!(module.state().unwrap(), ModuleState::Running);
12813
12814        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12815        assert!(handler
12816            .counters()
12817            .snapshot()
12818            .get("route_open_refused_by_code")
12819            .is_none());
12820        let capture = EventCapture::default();
12821        let _subscriber =
12822            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12823        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
12824        let response = handler
12825            .handle_control_frame(
12826                &ctx,
12827                route_open_frame(394, "warming", unique_project_root("refusal-info")),
12828            )
12829            .await
12830            .unwrap();
12831        module.stop().await.unwrap();
12832
12833        assert_eq!(parse_error(&response[0])["code"], "module_warming");
12834        let event = capture
12835            .events()
12836            .into_iter()
12837            .find(|event| {
12838                event.target == "control"
12839                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
12840            })
12841            .expect("route.open refusal event");
12842        assert_eq!(
12843            event.fields.get("module_id"),
12844            Some(&"\"warming\"".to_string())
12845        );
12846        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
12847        assert_eq!(
12848            event.fields.get("reason"),
12849            Some(&"\"supervised_not_registered\"".to_string())
12850        );
12851        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
12852        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
12853        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
12854        assert_eq!(
12855            handler.counters().snapshot()["route_open_refused_by_code"],
12856            json!({ "module_warming": 1 })
12857        );
12858    }
12859
12860    const OUTAGE_START: &str = "route.open refusing module: not serving";
12861    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
12862
12863    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
12864        capture
12865            .events()
12866            .into_iter()
12867            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
12868            .collect()
12869    }
12870
12871    fn supervise_stub(
12872        registry: &Arc<Registry>,
12873        module_id: &str,
12874        enabled: bool,
12875    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
12876        let supervisor_handle = SupervisorHandle::new();
12877        let supervisor =
12878            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
12879                .with_handle(supervisor_handle.clone())
12880                .with_connection_file_path(std::env::temp_dir().join(format!(
12881                    "subc-route-outage-{module_id}-{}",
12882                    std::process::id()
12883                )));
12884        let module = supervisor
12885            .supervise_configured(
12886                ModuleSpec {
12887                    module_id: module_id.to_string(),
12888                    program: fake_aft_stub_path(),
12889                    args: Vec::new(),
12890                    env: Vec::new(),
12891                    reserved: false,
12892                    reserved_prefixes: Vec::new(),
12893                    protocol: ModuleProtocol::Subc,
12894                    overlap: Default::default(),
12895                },
12896                enabled,
12897            )
12898            .unwrap();
12899        (supervisor_handle, module)
12900    }
12901
12902    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
12903        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
12904            module_id: module_id.to_string(),
12905            drain_timeout_ms: Some(50),
12906        })
12907        .unwrap();
12908        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
12909    }
12910
12911    /// Two handlers built over one forwarding table must share one outage
12912    /// tracker; separate trackers would each log their own opening line for
12913    /// the same outage.
12914    #[test]
12915    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
12916        let registry = Arc::new(Registry::default());
12917        let forwarding = Arc::new(ForwardingTable::default());
12918        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12919        let second = ControlHandler::with_forwarding(registry, forwarding);
12920        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
12921    }
12922
12923    /// A client can name any module id it likes. Refusing an unknown one,
12924    /// however often, must not create outage state or outage lines, or the
12925    /// tracker would be a memory sink any client could fill.
12926    #[tokio::test(flavor = "current_thread")]
12927    async fn route_open_unknown_module_refusals_add_no_outage_state() {
12928        let handler = ControlHandler::new(Arc::new(Registry::default()));
12929        let capture = EventCapture::default();
12930        let _subscriber =
12931            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12932        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12933        for corr in 0..8 {
12934            let response = handler
12935                .handle_control_frame(
12936                    &ctx,
12937                    route_open_frame(
12938                        380 + corr,
12939                        &format!("nobody-{corr}"),
12940                        unique_project_root("outage-unknown"),
12941                    ),
12942                )
12943                .await
12944                .unwrap();
12945            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12946        }
12947
12948        assert_eq!(handler.route_outages.tracked_module_count(), 0);
12949        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12950        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12951    }
12952
12953    /// Drives the refusal path end to end: a supervised module that served
12954    /// before and stopped being registered with no instruction to stop is a
12955    /// WARN, and the same module refused after an operator `supervisor.restart`
12956    /// is an INFO.
12957    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12958    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12959        let registry = Arc::new(Registry::default());
12960        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
12961        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12962        let capture = EventCapture::default();
12963        let _subscriber =
12964            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12965        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12966        // The stub never registers, so pretend it served once: otherwise every
12967        // refusal would fall in its startup window.
12968        handler.route_outages.record_accepted("outage-restart");
12969
12970        let response = handler
12971            .handle_control_frame(
12972                &ctx,
12973                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12974            )
12975            .await
12976            .unwrap();
12977        assert_eq!(response[0].header.ty, FrameType::Error);
12978        let starts = outage_lines(&capture, OUTAGE_START);
12979        assert_eq!(starts.len(), 1, "{starts:?}");
12980        assert_eq!(starts[0].level, tracing::Level::WARN);
12981        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12982        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12983        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12984        handler.route_outages.record_accepted("outage-restart");
12985        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12986
12987        let restart = handler
12988            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12989            .await
12990            .unwrap();
12991        assert_eq!(
12992            restart[0].header.ty,
12993            FrameType::Response,
12994            "{:?}",
12995            parse_error(&restart[0])
12996        );
12997        handler
12998            .handle_control_frame(
12999                &ctx,
13000                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13001            )
13002            .await
13003            .unwrap();
13004        module.stop().await.unwrap();
13005
13006        let starts = outage_lines(&capture, OUTAGE_START);
13007        assert_eq!(starts.len(), 2, "{starts:?}");
13008        assert_eq!(starts[1].level, tracing::Level::INFO);
13009        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13010    }
13011
13012    /// A restart refused before it touched the module (here: the module is
13013    /// disabled) must clear its operator mark, so the next real outage is
13014    /// still reported as a warning.
13015    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13016    async fn failed_operator_restart_leaves_no_operator_mark() {
13017        let registry = Arc::new(Registry::default());
13018        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
13019        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13020        let capture = EventCapture::default();
13021        let _subscriber =
13022            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13023        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13024        handler.route_outages.record_accepted("outage-disabled");
13025
13026        let restart = handler
13027            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13028            .await
13029            .unwrap();
13030        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13031        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13032
13033        handler
13034            .handle_control_frame(
13035                &ctx,
13036                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13037            )
13038            .await
13039            .unwrap();
13040        let starts = outage_lines(&capture, OUTAGE_START);
13041        assert_eq!(starts.len(), 1, "{starts:?}");
13042        assert_eq!(starts[0].level, tracing::Level::WARN);
13043    }
13044
13045    #[tokio::test(flavor = "current_thread")]
13046    async fn route_open_unknown_module_escapes_target_module_id() {
13047        let handler = ControlHandler::new(Arc::new(Registry::default()));
13048        let capture = EventCapture::default();
13049        let _subscriber =
13050            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13051        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13052        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13053        let response = handler
13054            .handle_control_frame(
13055                &ctx,
13056                route_open_frame(
13057                    395,
13058                    hostile_module_id,
13059                    unique_project_root("hostile-target-module-id"),
13060                ),
13061            )
13062            .await
13063            .unwrap();
13064
13065        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13066        let event = capture
13067            .events()
13068            .into_iter()
13069            .find(|event| {
13070                event.target == "control"
13071                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13072            })
13073            .expect("route.open unknown-module refusal event");
13074        let logged = event.fields.get("module_id").expect("module_id field");
13075        assert!(!logged.bytes().any(|byte| byte < 0x20));
13076        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13077    }
13078
13079    #[tokio::test(flavor = "current_thread")]
13080    async fn route_open_module_rejection_uses_daemon_counter_key() {
13081        let registry = Arc::new(Registry::default());
13082        let forwarding = Arc::new(ForwardingTable::default());
13083        let handler =
13084            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13085        let module_connection = ConnectionId::new(95);
13086        let (module_ctx, mut module_rx) = route_ctx(module_connection);
13087        hello_via_sink(
13088            &handler,
13089            &module_ctx,
13090            &mut module_rx,
13091            hello_frame("aft", PROTOCOL_VERSION, 395),
13092        )
13093        .await;
13094
13095        let client_connection = ConnectionId::new(96);
13096        let (client_ctx, _client_rx) = route_ctx(client_connection);
13097        let capture = EventCapture::default();
13098        let _subscriber =
13099            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13100        let (route_task, bind) = relay_route_open(
13101            &handler,
13102            client_connection,
13103            &client_ctx.egress,
13104            &mut module_rx,
13105            396,
13106            "aft",
13107            "hostile-module-code",
13108        )
13109        .await;
13110        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
13111        let rejection = Frame::build(
13112            FrameType::Error,
13113            control_flags(),
13114            0,
13115            0,
13116            bind.header.corr,
13117            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
13118        )
13119        .unwrap();
13120        handler
13121            .handle_control_frame(&module_ctx, rejection)
13122            .await
13123            .unwrap();
13124
13125        let response = route_task.await.unwrap();
13126        assert_eq!(parse_error(&response[0])["code"], hostile_code);
13127        let counters = handler.counters().snapshot();
13128        assert_eq!(
13129            counters["route_open_refused_by_code"],
13130            json!({ "module_rejected": 1 })
13131        );
13132        assert!(counters["route_open_refused_by_code"]
13133            .get(hostile_code)
13134            .is_none());
13135
13136        let event = capture
13137            .events()
13138            .into_iter()
13139            .find(|event| {
13140                event.target == "control"
13141                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
13142            })
13143            .expect("route.open module-rejection refusal event");
13144        let logged = event.fields.get("module_code").expect("module_code field");
13145        assert!(!logged.bytes().any(|byte| byte < 0x20));
13146        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13147    }
13148
13149    #[tokio::test]
13150    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
13151        let registry = Arc::new(Registry::default());
13152        let supervisor_handle = SupervisorHandle::new();
13153        let missing_program = std::env::temp_dir().join(format!(
13154            "subc-route-open-missing-program-{}",
13155            std::process::id()
13156        ));
13157        let supervisor =
13158            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
13159                .with_handle(supervisor_handle.clone());
13160        let module = supervisor
13161            .supervise_configured(
13162                ModuleSpec {
13163                    module_id: "failed".to_string(),
13164                    program: missing_program,
13165                    args: Vec::new(),
13166                    env: Vec::new(),
13167                    reserved: false,
13168                    reserved_prefixes: Vec::new(),
13169                    protocol: ModuleProtocol::Subc,
13170                    overlap: Default::default(),
13171                },
13172                true,
13173            )
13174            .unwrap();
13175        assert_eq!(module.state().unwrap(), ModuleState::Failed);
13176
13177        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13178        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
13179        let response = handler
13180            .handle_control_frame(
13181                &ctx,
13182                route_open_frame(305, "failed", unique_project_root("failed")),
13183            )
13184            .await
13185            .unwrap();
13186
13187        assert_eq!(response[0].header.ty, FrameType::Error);
13188        let error = parse_error(&response[0]);
13189        assert_eq!(error["code"], "target_unavailable");
13190        assert!(error["message"]
13191            .as_str()
13192            .unwrap()
13193            .contains("state=failed, enabled=true, live=false"));
13194    }
13195
13196    #[tokio::test]
13197    async fn route_open_role_mismatch_remains_target_unavailable() {
13198        let registry = Arc::new(Registry::default());
13199        let handler = ControlHandler::new(Arc::clone(&registry));
13200        handler
13201            .handle_control(
13202                ConnectionId::new(41),
13203                non_routable_hello_frame_with_control_ops("health-only", 306, None),
13204            )
13205            .unwrap();
13206
13207        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
13208        let response = handler
13209            .handle_control_frame(
13210                &ctx,
13211                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
13212            )
13213            .await
13214            .unwrap();
13215
13216        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13217        assert!(parse_error(&response[0])["message"]
13218            .as_str()
13219            .unwrap()
13220            .contains("does not provide the requested target"));
13221    }
13222
13223    #[tokio::test]
13224    async fn route_open_inactive_registration_remains_target_unavailable() {
13225        let registry = Arc::new(Registry::default());
13226        let handler = ControlHandler::new(Arc::clone(&registry));
13227        handler
13228            .handle_control(
13229                ConnectionId::new(43),
13230                hello_frame("inactive", PROTOCOL_VERSION, 308),
13231            )
13232            .unwrap();
13233        assert!(registry
13234            .set_module_state_for_test("inactive", ChannelState::Closed)
13235            .unwrap());
13236
13237        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
13238        let response = handler
13239            .handle_control_frame(
13240                &ctx,
13241                route_open_frame(309, "inactive", unique_project_root("inactive")),
13242            )
13243            .await
13244            .unwrap();
13245
13246        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
13247        assert!(parse_error(&response[0])["message"]
13248            .as_str()
13249            .unwrap()
13250            .contains("is not active"));
13251    }
13252
13253    #[tokio::test]
13254    async fn late_health_reply_is_recorded_through_the_module_response_path() {
13255        let registry = Arc::new(Registry::default());
13256        let forwarding = Arc::new(ForwardingTable::default());
13257        let supervisor_handle = SupervisorHandle::new();
13258        let supervisor =
13259            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
13260                .with_forwarding(Arc::clone(&forwarding))
13261                .with_handle(supervisor_handle.clone());
13262        let module = supervisor
13263            .supervise_configured(
13264                crate::ModuleSpec {
13265                    module_id: "late-health-response".to_string(),
13266                    program: PathBuf::from("disabled-module"),
13267                    args: Vec::new(),
13268                    env: Vec::new(),
13269                    reserved: false,
13270                    reserved_prefixes: Vec::new(),
13271                    protocol: ModuleProtocol::Subc,
13272                    overlap: Default::default(),
13273                },
13274                false,
13275            )
13276            .unwrap();
13277        let handler =
13278            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13279                .with_supervisor(supervisor_handle);
13280        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
13281        handler
13282            .handle_control_frame(
13283                &module_ctx,
13284                hello_frame_with_control_ops(
13285                    "late-health-response",
13286                    PROTOCOL_VERSION,
13287                    7,
13288                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13289                ),
13290            )
13291            .await
13292            .unwrap();
13293        let probe_started_at = Instant::now() - Duration::from_millis(80);
13294        let pending = forwarding
13295            .begin_health_probe_rpc_for(
13296                "late-health-response",
13297                MODULE_CONTROL_OP_HEALTH_CHECK,
13298                probe_started_at,
13299                Instant::now() - Duration::from_millis(1),
13300            )
13301            .unwrap();
13302        assert!(forwarding
13303            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
13304            .unwrap());
13305
13306        let responses = handler
13307            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
13308            .await
13309            .unwrap();
13310
13311        assert!(responses.is_empty());
13312        let health = module.status().unwrap().health;
13313        assert_eq!(health.late_answer_count, 1);
13314        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
13315    }
13316
13317    #[tokio::test]
13318    async fn health_probe_timeout_and_module_death_are_typed() {
13319        let registry = Arc::new(Registry::default());
13320        let forwarding = Arc::new(ForwardingTable::default());
13321        let handler =
13322            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13323                .with_health_probe_timeout(Duration::from_millis(50));
13324        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
13325        hello_via_sink(
13326            &handler,
13327            &module_ctx,
13328            &mut module_rx,
13329            hello_frame_with_control_ops(
13330                "aft",
13331                PROTOCOL_VERSION,
13332                7,
13333                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13334            ),
13335        )
13336        .await;
13337
13338        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
13339        let responses = handler
13340            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
13341            .await
13342            .unwrap();
13343        assert_eq!(responses[0].header.ty, FrameType::Error);
13344        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
13345        let _ = module_rx.try_recv();
13346
13347        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
13348        let health_handler = handler.clone();
13349        let death_task = tokio::spawn(async move {
13350            health_handler
13351                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
13352                .await
13353                .unwrap()
13354        });
13355        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13356            .await
13357            .unwrap()
13358            .unwrap();
13359        handler
13360            .cleanup_connection(module_ctx.connection_id)
13361            .unwrap();
13362        let responses = death_task.await.unwrap();
13363        assert_eq!(responses[0].header.ty, FrameType::Error);
13364        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
13365    }
13366
13367    #[test]
13368    fn hello_requires_exact_protocol_version() {
13369        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
13370            let registry = Arc::new(Registry::default());
13371            let handler = ControlHandler::new(Arc::clone(&registry));
13372            let responses = handler
13373                .handle_control(
13374                    ConnectionId::new(connection),
13375                    hello_frame("aft", offered, 9),
13376                )
13377                .unwrap();
13378
13379            assert_eq!(responses.len(), 1);
13380            assert_eq!(responses[0].header.ty, FrameType::Error);
13381            let error = parse_error(&responses[0]);
13382            assert_eq!(error["code"], "version_unsupported");
13383            assert!(registry.get_module("aft").unwrap().is_none());
13384            assert_eq!(registry.active_registration_count().unwrap(), 0);
13385        }
13386    }
13387
13388    #[test]
13389    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
13390        let registry = Arc::new(Registry::default());
13391        let forwarding = Arc::new(ForwardingTable::default());
13392        let handler =
13393            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13394        let module_connection = ConnectionId::new(301);
13395        let registration = registry
13396            .register_with_control_ops(
13397                manifest("aft-push", PROTOCOL_VERSION),
13398                PROTOCOL_VERSION,
13399                module_connection,
13400                module_baseline_control_ops(),
13401            )
13402            .unwrap();
13403        let (module_tx, _module_rx) = mpsc::channel(8);
13404        let endpoint = forwarding
13405            .register_module_connection(
13406                module_connection,
13407                "aft-push".to_string(),
13408                PROTOCOL_VERSION,
13409                manifest_concurrency(&registration.manifest),
13410                FrameSink::new(module_tx),
13411            )
13412            .unwrap();
13413
13414        // A push op this version does not know is ignored (forward-compat), not errored.
13415        let unknown = Frame::build(
13416            FrameType::Push,
13417            control_flags(),
13418            0,
13419            0,
13420            5,
13421            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
13422        )
13423        .unwrap();
13424        let out = handler.handle_status_update(endpoint, unknown).unwrap();
13425        assert!(
13426            out.is_empty(),
13427            "unknown push op must be ignored, got {out:?}"
13428        );
13429
13430        // A malformed body for a KNOWN op is a real error worth surfacing.
13431        let malformed = Frame::build(
13432            FrameType::Push,
13433            control_flags(),
13434            0,
13435            0,
13436            6,
13437            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13438        )
13439        .unwrap();
13440        let out = handler.handle_status_update(endpoint, malformed).unwrap();
13441        assert_eq!(out.len(), 1);
13442        assert_eq!(out[0].header.ty, FrameType::Error);
13443        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13444    }
13445
13446    #[test]
13447    fn hello_rejected_when_connection_already_owns_client_routes() {
13448        let registry = Arc::new(Registry::default());
13449        let forwarding = Arc::new(ForwardingTable::default());
13450        let handler =
13451            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13452        // Commits a client route on connection 202 (bound to a module on conn 101).
13453        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
13454        let client_connection = ConnectionId::new(202);
13455
13456        // That same connection now tries to register as a module: rejected, so one
13457        // connection never holds both client-route and module-endpoint state.
13458        let responses = handler
13459            .handle_control(
13460                client_connection,
13461                hello_frame("aft-second", PROTOCOL_VERSION, 9),
13462            )
13463            .unwrap();
13464        assert_eq!(responses[0].header.ty, FrameType::Error);
13465        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13466        assert!(registry.get_module("aft-second").unwrap().is_none());
13467    }
13468
13469    #[tokio::test]
13470    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13471        let registry = Arc::new(Registry::default());
13472        let forwarding = Arc::new(ForwardingTable::default());
13473        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13474        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13475        hello_via_sink(
13476            &handler,
13477            &module_ctx,
13478            &mut module_rx,
13479            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13480        )
13481        .await;
13482        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13483        let pending = forwarding
13484            .begin_route_bind_relay_for_test(
13485                client_ctx.connection_id,
13486                client_ctx.egress.clone(),
13487                2,
13488                "alpha",
13489            )
13490            .unwrap();
13491        forwarding
13492            .complete_pending_relay(
13493                module_ctx.connection_id,
13494                pending.corr,
13495                RouteBindRelayOutcome::Accepted,
13496            )
13497            .unwrap();
13498        client_rx.try_recv().unwrap();
13499        for module_id in ["beta", "alpha"] {
13500            let replies = handler
13501                .handle_control_frame(
13502                    &module_ctx,
13503                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13504                )
13505                .await
13506                .unwrap();
13507            assert_eq!(replies.len(), 1, "second HELLO must be refused");
13508            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13509        }
13510        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13511        assert!(registry.get_module("beta").unwrap().is_none());
13512        assert!(matches!(
13513            forwarding
13514                .lookup_data_route(
13515                    client_ctx.connection_id,
13516                    pending.client_channel,
13517                    pending.client_epoch,
13518                )
13519                .unwrap(),
13520            DataRoute::Client(DataRouteState::Bound(_))
13521        ));
13522        assert!(handler
13523            .hello_launch_nonces
13524            .lock()
13525            .unwrap()
13526            .presented(module_ctx.connection_id, Some("alpha-nonce")));
13527        assert!(module_rx.try_recv().is_err());
13528    }
13529
13530    #[test]
13531    fn reserved_module_hello_requires_matching_launch_nonce() {
13532        let registry = Arc::new(Registry::default());
13533        let supervisor = SupervisorHandle::new();
13534        // The supervisor recorded the nonce it injected when it spawned the reserved
13535        // module; the HELLO verifier checks against the same shared handle.
13536        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13537        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13538
13539        // A HELLO with NO nonce is rejected.
13540        let no_nonce = handler
13541            .handle_control(
13542                ConnectionId::new(1),
13543                hello_frame("vault", PROTOCOL_VERSION, 1),
13544            )
13545            .unwrap();
13546        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13547        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13548        assert!(registry.get_module("vault").unwrap().is_none());
13549
13550        // A HELLO with the WRONG nonce is rejected.
13551        let wrong = handler
13552            .handle_control(
13553                ConnectionId::new(2),
13554                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13555            )
13556            .unwrap();
13557        assert_eq!(wrong[0].header.ty, FrameType::Error);
13558        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13559        assert!(registry.get_module("vault").unwrap().is_none());
13560
13561        // A HELLO with the CORRECT nonce registers.
13562        let ok = handler
13563            .handle_control(
13564                ConnectionId::new(3),
13565                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13566            )
13567            .unwrap();
13568        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13569        assert!(registry.get_module("vault").unwrap().is_some());
13570    }
13571
13572    #[test]
13573    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13574        let registry = Arc::new(Registry::default());
13575        let supervisor = SupervisorHandle::new();
13576        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13577        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13578        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13579
13580        let squat = handler
13581            .handle_control(
13582                ConnectionId::new(1),
13583                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
13584            )
13585            .unwrap();
13586        assert_eq!(squat[0].header.ty, FrameType::Error);
13587        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
13588        assert!(parse_error(&squat[0])["message"]
13589            .as_str()
13590            .unwrap()
13591            .contains("fed:"));
13592
13593        let accepted_peer = handler
13594            .handle_control(
13595                ConnectionId::new(2),
13596                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
13597            )
13598            .unwrap();
13599        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
13600
13601        let accepted_short = handler
13602            .handle_control(
13603                ConnectionId::new(3),
13604                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
13605            )
13606            .unwrap();
13607        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
13608
13609        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
13610            let response = handler
13611                .handle_control(
13612                    ConnectionId::new(conn),
13613                    hello_frame(module_id, PROTOCOL_VERSION, conn),
13614                )
13615                .unwrap();
13616            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
13617        }
13618    }
13619
13620    #[test]
13621    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
13622        let registry = Arc::new(Registry::default());
13623        let supervisor = SupervisorHandle::new();
13624        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13625        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13626        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
13627        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13628
13629        let owner_nonce = handler
13630            .handle_control(
13631                ConnectionId::new(1),
13632                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
13633            )
13634            .unwrap();
13635        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
13636        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
13637        assert!(registry.get_module("fed:special").unwrap().is_none());
13638
13639        let exact_nonce = handler
13640            .handle_control(
13641                ConnectionId::new(2),
13642                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
13643            )
13644            .unwrap();
13645        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
13646        assert!(registry.get_module("fed:special").unwrap().is_some());
13647    }
13648
13649    #[test]
13650    fn non_reserved_module_ignores_launch_nonce() {
13651        let registry = Arc::new(Registry::default());
13652        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
13653        // registration succeeds whether a spawned process echoes a nonce or not.
13654        let handler = ControlHandler::new(Arc::clone(&registry));
13655        let no_nonce = handler
13656            .handle_control(
13657                ConnectionId::new(1),
13658                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
13659            )
13660            .unwrap();
13661        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
13662        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
13663
13664        let echoed_nonce = handler
13665            .handle_control(
13666                ConnectionId::new(2),
13667                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
13668            )
13669            .unwrap();
13670        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
13671        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
13672    }
13673
13674    #[test]
13675    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
13676        let handler = ControlHandler::default();
13677        let conn = ConnectionId::new(1);
13678        let malformed = Frame::build(
13679            FrameType::Hello,
13680            control_flags(),
13681            0,
13682            0,
13683            3,
13684            b"{not json".to_vec(),
13685        )
13686        .unwrap();
13687
13688        let error = handler.handle_control(conn, malformed).unwrap();
13689        assert_eq!(error[0].header.ty, FrameType::Error);
13690        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
13691
13692        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13693        let pong = handler.handle_control(conn, ping).unwrap();
13694        assert_eq!(pong[0].header.ty, FrameType::Pong);
13695        assert_eq!(pong[0].header.corr, 4);
13696    }
13697
13698    #[test]
13699    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
13700        let registry = Arc::new(Registry::default());
13701        let handler = ControlHandler::new(Arc::clone(&registry));
13702
13703        handler
13704            .handle_control(
13705                ConnectionId::new(1),
13706                hello_frame("aft", PROTOCOL_VERSION, 1),
13707            )
13708            .unwrap();
13709        let duplicate = handler
13710            .handle_control(
13711                ConnectionId::new(2),
13712                hello_frame("aft", PROTOCOL_VERSION, 2),
13713            )
13714            .unwrap();
13715
13716        assert_eq!(duplicate[0].header.ty, FrameType::Error);
13717        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13718        let registration = registry.get_module("aft").unwrap().unwrap();
13719        assert_eq!(registration.connection_id, ConnectionId::new(1));
13720    }
13721
13722    #[test]
13723    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
13724        let registry = Arc::new(Registry::default());
13725        let forwarding = Arc::new(ForwardingTable::default());
13726        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
13727        let handler =
13728            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13729                .with_process_liveness(process_liveness);
13730        let (ctx, route_channel, route_epoch) =
13731            bind_liveness_route(&registry, &forwarding, "aft-dead");
13732        let responses = handler
13733            .handle_route_poll(
13734                &ctx,
13735                route_poll_frame(41, PollKind::Liveness, route_channel),
13736                route_channel,
13737                route_epoch,
13738                PollKind::Liveness,
13739            )
13740            .unwrap();
13741
13742        assert_eq!(responses.len(), 1);
13743        assert_eq!(responses[0].header.ty, FrameType::Response);
13744        assert_route_poll_liveness(&responses[0], false);
13745    }
13746
13747    #[test]
13748    fn liveness_poll_without_process_source_uses_bound_route() {
13749        let registry = Arc::new(Registry::default());
13750        let forwarding = Arc::new(ForwardingTable::default());
13751        let handler =
13752            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13753        let (ctx, route_channel, route_epoch) =
13754            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
13755        let responses = handler
13756            .handle_route_poll(
13757                &ctx,
13758                route_poll_frame(42, PollKind::Liveness, route_channel),
13759                route_channel,
13760                route_epoch,
13761                PollKind::Liveness,
13762            )
13763            .unwrap();
13764
13765        assert_route_poll_liveness(&responses[0], true);
13766    }
13767
13768    #[test]
13769    fn liveness_poll_untracked_process_source_uses_bound_route() {
13770        let registry = Arc::new(Registry::default());
13771        let forwarding = Arc::new(ForwardingTable::default());
13772        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
13773        let handler =
13774            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13775                .with_process_liveness(process_liveness);
13776        let (ctx, route_channel, route_epoch) =
13777            bind_liveness_route(&registry, &forwarding, "aft-untracked");
13778        let responses = handler
13779            .handle_route_poll(
13780                &ctx,
13781                route_poll_frame(43, PollKind::Liveness, route_channel),
13782                route_channel,
13783                route_epoch,
13784                PollKind::Liveness,
13785            )
13786            .unwrap();
13787
13788        assert_route_poll_liveness(&responses[0], true);
13789    }
13790
13791    #[tokio::test]
13792    async fn unknown_op_returns_unknown_control_op() {
13793        let handler = ControlHandler::default();
13794        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
13795        let request = Frame::build(
13796            FrameType::Request,
13797            control_flags(),
13798            0,
13799            0,
13800            55,
13801            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
13802        )
13803        .unwrap();
13804
13805        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13806
13807        assert_eq!(response.len(), 1);
13808        assert_eq!(response[0].header.ty, FrameType::Error);
13809        assert_eq!(response[0].header.corr, 55);
13810        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
13811    }
13812
13813    #[tokio::test]
13814    async fn supervisor_provenance_rejects_unknown_exact_module() {
13815        let handler = ControlHandler::default();
13816        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
13817        let request = Frame::build(
13818            FrameType::Request,
13819            control_flags(),
13820            0,
13821            0,
13822            57,
13823            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
13824        )
13825        .unwrap();
13826
13827        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13828
13829        assert_eq!(response.len(), 1);
13830        assert_eq!(response[0].header.ty, FrameType::Error);
13831        assert_eq!(response[0].header.corr, 57);
13832        let error = parse_error(&response[0]);
13833        assert_eq!(error["code"], "unknown_module");
13834        assert_eq!(error["message"], "module_id 'missing' is not supervised");
13835    }
13836
13837    #[test]
13838    fn provenance_probe_override_keeps_handler_tests_deterministic() {
13839        let expected = subc_control::RunningImageAgreement::Unavailable {
13840            reason: subc_control::RunningImageUnavailableReason::HashFailed,
13841        };
13842        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
13843        assert_eq!(handler.provenance_probe_override, Some(expected));
13844    }
13845
13846    #[test]
13847    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
13848        let verdict = reload_verdict(
13849            std::path::Path::new("/bin/new"),
13850            Some(std::path::Path::new("/bin/old")),
13851            subc_control::RunningImageAgreement::Unavailable {
13852                reason: subc_control::RunningImageUnavailableReason::HashFailed,
13853            },
13854        );
13855        assert!(matches!(
13856            verdict.path,
13857            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
13858                if configured == std::path::Path::new("/bin/new")
13859                    && spawned_from == std::path::Path::new("/bin/old")
13860        ));
13861    }
13862
13863    #[test]
13864    fn launch_spec_without_running_digest_is_unavailable_not_match() {
13865        use subc_control::{
13866            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
13867        };
13868        let configured = crate::supervise::ModuleSpec {
13869            module_id: "legacy".into(),
13870            program: "/module".into(),
13871            args: Vec::new(),
13872            env: Vec::new(),
13873            reserved: false,
13874            reserved_prefixes: Vec::new(),
13875            protocol: subc_control::ModuleProtocol::Subc,
13876            overlap: Default::default(),
13877        };
13878        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
13879        assert_eq!(
13880            launch_spec_agreement(&configured, false, Some(&digest)),
13881            Agreement::Unavailable {
13882                reason: Reason::NotRunning
13883            }
13884        );
13885        assert_eq!(
13886            launch_spec_agreement(&configured, true, None),
13887            Agreement::Unavailable {
13888                reason: Reason::SpawnedSpecUnavailable
13889            }
13890        );
13891        assert_eq!(
13892            launch_spec_agreement(&configured, true, Some(&digest)),
13893            Agreement::Match
13894        );
13895    }
13896
13897    #[cfg(windows)]
13898    #[test]
13899    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
13900        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
13901        let configured = dir.join("module.exe");
13902        std::fs::write(&configured, b"executable path fixture").unwrap();
13903        let spawned = std::fs::canonicalize(&configured).unwrap();
13904        assert_ne!(
13905            configured, spawned,
13906            "fixture must exercise different path spellings"
13907        );
13908        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
13909        let lower_case = configured.to_string_lossy().to_lowercase();
13910        for spelling in [
13911            &configured,
13912            Path::new(&forward_slashes),
13913            Path::new(&lower_case),
13914        ] {
13915            let verdict = reload_verdict(
13916                spelling,
13917                Some(&spawned),
13918                subc_control::RunningImageAgreement::Unavailable {
13919                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13920                },
13921            );
13922            assert_eq!(
13923                verdict.path,
13924                ReloadPathAgreement::Match,
13925                "configured {spelling:?}, spawned {spawned:?}"
13926            );
13927        }
13928    }
13929
13930    #[cfg(windows)]
13931    #[test]
13932    fn windows_reload_path_keeps_different_existing_program_pending() {
13933        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
13934        let old = dir.join("old.exe");
13935        let new = dir.join("new.exe");
13936        std::fs::write(&old, b"same executable content").unwrap();
13937        std::fs::copy(&old, &new).unwrap();
13938        let spawned = std::fs::canonicalize(&old).unwrap();
13939        let image = subc_control::RunningImageAgreement::Unavailable {
13940            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13941        };
13942        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
13943        assert!(
13944            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
13945            if configured == new && spawned_from == spawned)
13946        );
13947        let missing = dir.join("missing.exe");
13948        assert!(matches!(
13949            reload_verdict(&missing, Some(&spawned), image).path,
13950            ReloadPathAgreement::Mismatch { .. }
13951        ));
13952    }
13953
13954    #[test]
13955    fn reload_verdict_detects_replaced_image_at_same_path() {
13956        let image = subc_control::RunningImageAgreement::Mismatch {
13957            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13958                digest: "old".into(),
13959            },
13960            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13961                digest: "new".into(),
13962            },
13963        };
13964        let verdict = reload_verdict(
13965            std::path::Path::new("/bin/same"),
13966            Some(std::path::Path::new("/bin/same")),
13967            image.clone(),
13968        );
13969        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13970        assert_eq!(verdict.image, image);
13971    }
13972
13973    #[test]
13974    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13975        let image = subc_control::RunningImageAgreement::Unavailable {
13976            reason: subc_control::RunningImageUnavailableReason::NotRunning,
13977        };
13978        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13979        assert_eq!(
13980            verdict.path,
13981            subc_control::ReloadPathAgreement::Unavailable {
13982                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13983            }
13984        );
13985        assert_eq!(verdict.image, image);
13986
13987        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13988            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13989        };
13990        let verdict = reload_verdict(
13991            std::path::Path::new("/bin/same"),
13992            Some(std::path::Path::new("/bin/same")),
13993            unconfirmed.clone(),
13994        );
13995        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13996        assert_eq!(verdict.image, unconfirmed);
13997    }
13998
13999    #[test]
14000    fn reload_verdict_preserves_each_image_unavailability_reason() {
14001        use subc_control::RunningImageUnavailableReason as Reason;
14002
14003        for reason in [
14004            Reason::NotRunning,
14005            Reason::UnsupportedPlatform,
14006            Reason::RunningExecutableUnreadable,
14007            Reason::SpawnedPathUnreadable,
14008            Reason::HashFailed,
14009            Reason::ProcessIdentityUnconfirmed,
14010            Reason::Unknown("future_probe_reason".to_string()),
14011        ] {
14012            let image = subc_control::RunningImageAgreement::Unavailable {
14013                reason: reason.clone(),
14014            };
14015            let verdict = reload_verdict(
14016                std::path::Path::new("/bin/same"),
14017                Some(std::path::Path::new("/bin/same")),
14018                image.clone(),
14019            );
14020            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14021            assert_eq!(verdict.image, image, "{reason:?}");
14022        }
14023    }
14024
14025    #[tokio::test]
14026    async fn malformed_control_bodies_return_invalid_control_body() {
14027        let handler = ControlHandler::default();
14028        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14029
14030        for (corr, body) in [
14031            (56, br#"{"route_channel":1}"#.as_slice()),
14032            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14033            (
14034                58,
14035                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14036            ),
14037        ] {
14038            let request = Frame::build(
14039                FrameType::Request,
14040                control_flags(),
14041                0,
14042                0,
14043                corr,
14044                body.to_vec(),
14045            )
14046            .unwrap();
14047            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14048
14049            assert_eq!(response.len(), 1);
14050            assert_eq!(response[0].header.ty, FrameType::Error);
14051            assert_eq!(response[0].header.corr, corr);
14052            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14053        }
14054    }
14055
14056    #[tokio::test]
14057    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
14058        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14059        let registry = Arc::new(Registry::default());
14060        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14061        let router = Router::with_control_handler(Arc::clone(&control));
14062        let connection = router.begin_connection();
14063        let (ctx, mut rx) = route_ctx(connection.id());
14064
14065        router
14066            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
14067            .await
14068            .unwrap();
14069        let response = rx.recv().await.unwrap();
14070        let ack = parse_ack(&response);
14071        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14072        let channel = 1;
14073
14074        let goodbye =
14075            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
14076        router.route_for_connection(&ctx, goodbye).await.unwrap();
14077        assert!(rx.try_recv().is_err());
14078        assert!(registry.get_module("aft").unwrap().is_none());
14079
14080        router
14081            .route_for_connection(&ctx, channel_request(channel, 13))
14082            .await
14083            .unwrap();
14084        let error_frame = rx.recv().await.unwrap();
14085        assert_eq!(error_frame.header.ty, FrameType::Error);
14086        assert_eq!(error_frame.header.channel, channel);
14087        let captured = crate::router::test_log::captured_logs(&logs);
14088        assert_eq!(
14089            captured
14090                .lines()
14091                .filter(|line| {
14092                    line.contains("module_id=aft")
14093                        && line.contains("reason=explicit_goodbye")
14094                        && line.contains("module registration ended")
14095                })
14096                .count(),
14097            1,
14098            "unexpected GOODBYE registry log: {captured}"
14099        );
14100    }
14101
14102    #[tokio::test]
14103    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
14104        let registry = Arc::new(Registry::default());
14105        let handler = ControlHandler::new(registry).with_capability_config(
14106            [("prov".to_string(), true), ("cons".to_string(), true)],
14107            BTreeMap::new(),
14108        );
14109        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
14110        register_capability_manifest(
14111            &handler,
14112            &provider_ctx,
14113            &mut provider_rx,
14114            capability_manifest("prov", &["thing/v1"], &[]),
14115            1,
14116        )
14117        .await;
14118        let mut consumer = capability_manifest("cons", &[], &[]);
14119        consumer.capabilities.as_mut().unwrap().requires.push(
14120            subc_protocol::manifest::CapabilityRequirement {
14121                capability: "thing/v1".to_string(),
14122                need: subc_protocol::manifest::CapabilityNeed::Required,
14123            },
14124        );
14125        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
14126        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
14127        assert_eq!(
14128            handler.capability_evaluator.verdict("cons", "thing/v1"),
14129            Some(CapabilityVerdict::Provided)
14130        );
14131        let (mut client_rx, _) = open_route_for_capability_test(
14132            &handler,
14133            &provider_ctx,
14134            &mut provider_rx,
14135            703,
14136            3,
14137            "prov",
14138            None,
14139        )
14140        .await;
14141        let goodbye =
14142            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14143        handler
14144            .handle_control_frame(&provider_ctx, goodbye)
14145            .await
14146            .unwrap();
14147        assert_eq!(
14148            handler.capability_evaluator.verdict("cons", "thing/v1"),
14149            Some(CapabilityVerdict::NeverProvided)
14150        );
14151        let closed = client_rx
14152            .try_recv()
14153            .expect("GOODBYE pushes route.closed before route GOODBYE");
14154        assert!(
14155            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
14156            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
14157        );
14158        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
14159        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14160    }
14161
14162    struct RouteClosedFixture {
14163        handler: ControlHandler,
14164        #[cfg(unix)]
14165        supervisor: Supervisor,
14166        module_ctx: RouteCtx,
14167        #[cfg(unix)]
14168        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14169        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14170        route: Arc<crate::forwarding::RouteBinding>,
14171    }
14172
14173    async fn route_closed_fixture() -> RouteClosedFixture {
14174        let registry = Arc::new(Registry::default());
14175        let handler = ControlHandler::new(Arc::clone(&registry));
14176        #[cfg(unix)]
14177        let supervisor = Supervisor::new(registry, RestartPolicy::default())
14178            .with_forwarding(Arc::clone(&handler.forwarding));
14179        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
14180        register_capability_manifest(
14181            &handler,
14182            &module_ctx,
14183            &mut module_rx,
14184            capability_manifest("prov", &[], &[]),
14185            1,
14186        )
14187        .await;
14188        let (client_rx, bind) = open_route_for_capability_test(
14189            &handler,
14190            &module_ctx,
14191            &mut module_rx,
14192            702,
14193            2,
14194            "prov",
14195            None,
14196        )
14197        .await;
14198        let ModuleControlRequest::RouteBind {
14199            route_channel: channel,
14200            epoch: route_epoch,
14201            ..
14202        } = bind
14203        else {
14204            panic!("expected route.bind");
14205        };
14206        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
14207            handler
14208                .forwarding
14209                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
14210                .unwrap()
14211        else {
14212            panic!("the acknowledged route must be bound");
14213        };
14214        RouteClosedFixture {
14215            handler,
14216            #[cfg(unix)]
14217            supervisor,
14218            module_ctx,
14219            #[cfg(unix)]
14220            module_rx,
14221            client_rx,
14222            route,
14223        }
14224    }
14225
14226    impl RouteClosedFixture {
14227        #[cfg(unix)]
14228        async fn cut(&mut self) {
14229            let notices = async {
14230                let mut draining = self.module_rx.recv().await.unwrap();
14231                assert!(matches!(
14232                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
14233                        &draining.body
14234                    )
14235                    .unwrap(),
14236                    subc_protocol::session::ModuleControlCommand::Draining {
14237                        reason: RouteCloseReason::Restart,
14238                        ..
14239                    }
14240                ));
14241                draining.flushed.take().unwrap().send(()).unwrap();
14242                let mut closing = self.client_rx.recv().await.unwrap();
14243                assert!(matches!(
14244                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
14245                    ClientControlPush::RouteClosing {
14246                        reason: RouteCloseReason::Restart,
14247                        ..
14248                    }
14249                ));
14250                closing.flushed.take().unwrap().send(()).unwrap();
14251            };
14252            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
14253            result.unwrap();
14254        }
14255
14256        fn disconnect(&mut self) -> ClientControlPush {
14257            self.handler
14258                .cleanup_connection(self.module_ctx.connection_id)
14259                .unwrap();
14260            let closed = self.client_rx.try_recv().unwrap();
14261            assert_eq!(closed.header.ty, FrameType::Push);
14262            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
14263            assert!(matches!(
14264                &push,
14265                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
14266                    if channels == &[self.route.client_channel]
14267            ));
14268            assert_eq!(
14269                self.client_rx.try_recv().unwrap().header.ty,
14270                FrameType::Goodbye
14271            );
14272            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
14273            push
14274        }
14275    }
14276
14277    #[cfg(unix)]
14278    #[tokio::test(start_paused = true)]
14279    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
14280        let mut fixture = route_closed_fixture().await;
14281        fixture.cut().await;
14282        assert!(matches!(
14283            fixture.disconnect(),
14284            ClientControlPush::RouteClosed {
14285                reason: RouteCloseReason::Restart,
14286                drained: true,
14287                ..
14288            }
14289        ));
14290    }
14291
14292    #[cfg(unix)]
14293    #[tokio::test(start_paused = true)]
14294    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
14295        let mut fixture = route_closed_fixture().await;
14296        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
14297        let started = tokio::time::Instant::now();
14298        fixture.cut().await;
14299        assert!(started.elapsed() >= Duration::from_secs(2));
14300        assert_eq!(fixture.route.flow.in_flight(), 1);
14301        assert!(matches!(
14302            fixture.disconnect(),
14303            ClientControlPush::RouteClosed {
14304                reason: RouteCloseReason::Restart,
14305                drained: false,
14306                ..
14307            }
14308        ));
14309    }
14310
14311    #[tokio::test]
14312    async fn module_connection_crash_reports_crash_and_not_drained() {
14313        let mut fixture = route_closed_fixture().await;
14314        // The server calls cleanup_connection on module EOF. Without a planned
14315        // daemon shutdown, that disconnect must remain a crash with no prior
14316        // route.closing notice, even when the route has no in-flight calls.
14317        assert!(matches!(
14318            fixture.disconnect(),
14319            ClientControlPush::RouteClosed {
14320                reason: RouteCloseReason::Crash,
14321                drained: false,
14322                ..
14323            }
14324        ));
14325    }
14326
14327    #[tokio::test]
14328    async fn dropping_router_connection_releases_registration() {
14329        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14330        let registry = Arc::new(Registry::default());
14331        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
14332        let router = Router::with_control_handler(Arc::clone(&control));
14333        let connection = router.begin_connection();
14334        let connection_id = connection.id();
14335        let (ctx, mut rx) = route_ctx(connection_id);
14336
14337        router
14338            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
14339            .await
14340            .unwrap();
14341        let response = rx.recv().await.unwrap();
14342        let ack = parse_ack(&response);
14343        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
14344        assert!(registry.get_module("aft").unwrap().is_some());
14345
14346        drop(connection);
14347
14348        assert!(registry.get_module("aft").unwrap().is_none());
14349        assert_eq!(registry.active_registration_count().unwrap(), 0);
14350
14351        control
14352            .cleanup_connection(ConnectionId::new(u64::MAX))
14353            .unwrap();
14354        let captured = crate::router::test_log::captured_logs(&logs);
14355        println!("captured registry lifecycle logs:\n{captured}");
14356        let events: Vec<_> = captured
14357            .lines()
14358            .filter(|line| {
14359                line.contains("module registered module_id=aft ")
14360                    || line.contains("module registration ended")
14361            })
14362            .collect();
14363        assert_eq!(
14364            events.len(),
14365            2,
14366            "unexpected registry lifecycle logs: {captured}"
14367        );
14368        assert!(events[0].contains("module registered module_id=aft "));
14369        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
14370        assert!(events[1].contains(&format!(
14371            "module_id=aft connection_id={}",
14372            connection_id.get()
14373        )));
14374        assert!(events[1].contains("reason=connection_closed"));
14375        assert!(events[1].contains("module registration ended"));
14376        assert_eq!(
14377            captured
14378                .lines()
14379                .filter(|line| line.contains("module registered module_id=aft "))
14380                .count(),
14381            1,
14382            "legacy registration admission line must appear once: {captured}"
14383        );
14384    }
14385
14386    #[tokio::test]
14387    async fn hello_registration_keeps_legacy_module_registered_line_once() {
14388        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14389        let registry = Arc::new(Registry::default());
14390        let control = ControlHandler::new(Arc::clone(&registry));
14391        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
14392        hello_via_sink(
14393            &control,
14394            &ctx,
14395            &mut rx,
14396            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
14397        )
14398        .await;
14399
14400        let captured = crate::router::test_log::captured_logs(&logs);
14401        let admissions: Vec<_> = captured
14402            .lines()
14403            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
14404            .collect();
14405        assert_eq!(
14406            admissions.len(),
14407            1,
14408            "legacy admission line must remain exactly once: {captured}"
14409        );
14410        assert!(admissions[0].contains("routable_provider=true"));
14411        assert!(admissions[0].contains("connection_id=777"));
14412    }
14413
14414    fn capability_manifest(
14415        module_id: &str,
14416        provides: &[&str],
14417        must_never_reach: &[&str],
14418    ) -> ModuleManifest {
14419        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
14420        manifest.capabilities = Some(CapabilityDeclarations {
14421            provides: provides
14422                .iter()
14423                .map(|capability| (*capability).to_string())
14424                .collect(),
14425            requires: Vec::new(),
14426            must_never_reach: must_never_reach
14427                .iter()
14428                .map(|capability| (*capability).to_string())
14429                .collect(),
14430        });
14431        manifest
14432    }
14433
14434    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14435        Frame::build(
14436            FrameType::Hello,
14437            control_flags(),
14438            0,
14439            0,
14440            corr,
14441            serde_json::to_vec(&ModuleHelloBody {
14442                protocol_ver: manifest.protocol_ver,
14443                manifest,
14444                control_ops: None,
14445                launch_nonce: None,
14446            })
14447            .expect("capability test HELLO serializes"),
14448        )
14449        .expect("capability test HELLO frame builds")
14450    }
14451
14452    fn catalog_update_with_capabilities_frame(
14453        corr: u64,
14454        capabilities: CapabilityDeclarations,
14455    ) -> Frame {
14456        Frame::build(
14457            FrameType::Request,
14458            control_flags(),
14459            0,
14460            0,
14461            corr,
14462            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14463                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14464                capabilities: Some(capabilities),
14465                ready: None,
14466            })
14467            .expect("capability catalog.update serializes"),
14468        )
14469        .expect("capability catalog.update frame builds")
14470    }
14471
14472    async fn register_capability_manifest(
14473        handler: &ControlHandler,
14474        ctx: &RouteCtx,
14475        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14476        manifest: ModuleManifest,
14477        corr: u64,
14478    ) {
14479        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14480    }
14481
14482    async fn open_route_for_capability_test(
14483        handler: &ControlHandler,
14484        target_ctx: &RouteCtx,
14485        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14486        client_connection_id: u64,
14487        corr: u64,
14488        target_module_id: &str,
14489        consumer_identity: Option<ConsumerIdentity>,
14490    ) -> (
14491        mpsc::Receiver<crate::router::OutboundFrame>,
14492        ModuleControlRequest,
14493    ) {
14494        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14495        let route_handler = handler.clone();
14496        let target_module_id = target_module_id.to_string();
14497        let route_task = tokio::spawn(async move {
14498            route_handler
14499                .handle_control_frame(
14500                    &client_ctx,
14501                    route_open_frame_with_admission_facts(
14502                        corr,
14503                        &target_module_id,
14504                        unique_project_root("admission-facts"),
14505                        consumer_identity,
14506                        None,
14507                    ),
14508                )
14509                .await
14510                .expect("capability test route.open succeeds")
14511        });
14512        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14513            .await
14514            .expect("capability test route.open must reach route.bind")
14515            .expect("target control receiver stays open");
14516        let bind_request: ModuleControlRequest =
14517            serde_json::from_slice(&bind.body).expect("route.bind decodes");
14518        handler
14519            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14520            .await
14521            .expect("capability test route.bind ACK succeeds");
14522        assert!(route_task.await.expect("route.open task joins").is_empty());
14523        let opened = client_rx
14524            .recv()
14525            .await
14526            .expect("successful route.open publishes a response");
14527        assert!(matches!(
14528            serde_json::from_slice::<ClientControlResponse>(&opened.body),
14529            Ok(ClientControlResponse::RouteOpen { .. })
14530        ));
14531        (client_rx, bind_request)
14532    }
14533
14534    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
14535        assert_eq!(frame.header.ty, FrameType::Push);
14536        assert_eq!(frame.header.channel, 0);
14537        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14538            .expect("route.closed control push decodes");
14539        let ClientControlPush::RouteClosed { channels, .. } = &push else {
14540            panic!("expected route.closed");
14541        };
14542        assert_eq!(channels.len(), 1, "exactly one violating route closed");
14543        let channels = channels.clone();
14544        assert_eq!(
14545            push,
14546            ClientControlPush::RouteClosed {
14547                module_id: target_module_id.to_string(),
14548                channels,
14549                reason: RouteCloseReason::CapabilityDenied,
14550                drained: false,
14551                abandoned: 0,
14552                excluded_subscriptions: 0,
14553                terminal: Some(false),
14554            }
14555        );
14556    }
14557
14558    #[tokio::test]
14559    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14560        let registry = Arc::new(Registry::default());
14561        let forwarding = Arc::new(ForwardingTable::default());
14562        let supervisor = SupervisorHandle::new();
14563        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14564        let handler =
14565            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14566                .with_supervisor(supervisor);
14567        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14568        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14569        register_capability_manifest(
14570            &handler,
14571            &target_ctx,
14572            &mut target_rx,
14573            capability_manifest("target", &["credentials-provider/v1"], &[]),
14574            1,
14575        )
14576        .await;
14577        register_capability_manifest(
14578            &handler,
14579            &opener_ctx,
14580            &mut opener_rx,
14581            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14582            2,
14583        )
14584        .await;
14585
14586        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
14587        let replies = handler
14588            .handle_control_frame(
14589                &client_ctx,
14590                route_open_frame_with_admission_facts(
14591                    3,
14592                    "target",
14593                    unique_project_root("admission-facts"),
14594                    Some(ConsumerIdentity {
14595                        module_id: "opener".to_string(),
14596                        launch_nonce: "opener-nonce".to_string(),
14597                    }),
14598                    None,
14599                ),
14600            )
14601            .await
14602            .expect("denied route.open returns a typed frame");
14603        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14604        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14605        assert!(
14606            target_rx.try_recv().is_err(),
14607            "forbidden route.open must not relay route.bind"
14608        );
14609    }
14610
14611    #[tokio::test]
14612    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
14613        let registry = Arc::new(Registry::default());
14614        let forwarding = Arc::new(ForwardingTable::default());
14615        let supervisor = SupervisorHandle::new();
14616        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14617        let handler =
14618            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14619                .with_supervisor(supervisor);
14620        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
14621        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
14622        register_capability_manifest(
14623            &handler,
14624            &target_ctx,
14625            &mut target_rx,
14626            capability_manifest("target", &["credentials-provider/v1"], &[]),
14627            1,
14628        )
14629        .await;
14630        register_capability_manifest(
14631            &handler,
14632            &old_opener_ctx,
14633            &mut old_opener_rx,
14634            capability_manifest("opener", &[], &[]),
14635            2,
14636        )
14637        .await;
14638        let (mut client_rx, _) = open_route_for_capability_test(
14639            &handler,
14640            &target_ctx,
14641            &mut target_rx,
14642            712,
14643            3,
14644            "target",
14645            Some(ConsumerIdentity {
14646                module_id: "opener".to_string(),
14647                launch_nonce: "opener-nonce".to_string(),
14648            }),
14649        )
14650        .await;
14651        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14652
14653        handler
14654            .cleanup_connection(old_opener_ctx.connection_id)
14655            .expect("old opener registration cleans up");
14656        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
14657        register_capability_manifest(
14658            &handler,
14659            &new_opener_ctx,
14660            &mut new_opener_rx,
14661            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14662            4,
14663        )
14664        .await;
14665
14666        assert_capability_denied_push(
14667            client_rx
14668                .try_recv()
14669                .expect("HELLO deny addition must emit route.closed")
14670                .frame,
14671            "target",
14672        );
14673        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14674        assert!(matches!(
14675            target_rx.try_recv(),
14676            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14677        ));
14678    }
14679
14680    #[tokio::test]
14681    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
14682        let registry = Arc::new(Registry::default());
14683        let forwarding = Arc::new(ForwardingTable::default());
14684        let supervisor = SupervisorHandle::new();
14685        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14686        let handler =
14687            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14688                .with_supervisor(supervisor);
14689        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
14690        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
14691        register_capability_manifest(
14692            &handler,
14693            &target_ctx,
14694            &mut target_rx,
14695            capability_manifest("target", &[], &[]),
14696            1,
14697        )
14698        .await;
14699        register_capability_manifest(
14700            &handler,
14701            &opener_ctx,
14702            &mut opener_rx,
14703            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14704            2,
14705        )
14706        .await;
14707        let (mut client_rx, _) = open_route_for_capability_test(
14708            &handler,
14709            &target_ctx,
14710            &mut target_rx,
14711            722,
14712            3,
14713            "target",
14714            Some(ConsumerIdentity {
14715                module_id: "opener".to_string(),
14716                launch_nonce: "opener-nonce".to_string(),
14717            }),
14718        )
14719        .await;
14720        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14721
14722        let replies = handler
14723            .handle_control_frame(
14724                &target_ctx,
14725                catalog_update_with_capabilities_frame(
14726                    4,
14727                    CapabilityDeclarations {
14728                        provides: vec!["credentials-provider/v1".to_string()],
14729                        requires: Vec::new(),
14730                        must_never_reach: Vec::new(),
14731                    },
14732                ),
14733            )
14734            .await
14735            .expect("claim catalog.update succeeds");
14736        assert!(matches!(
14737            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
14738            Ok(ModuleControlResponseToModule::CatalogUpdate {})
14739        ));
14740        assert_capability_denied_push(
14741            client_rx
14742                .try_recv()
14743                .expect("claim addition must emit route.closed")
14744                .frame,
14745            "target",
14746        );
14747        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14748        assert!(matches!(
14749            target_rx.try_recv(),
14750            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14751        ));
14752    }
14753
14754    #[tokio::test]
14755    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
14756        let registry = Arc::new(Registry::default());
14757        let forwarding = Arc::new(ForwardingTable::default());
14758        let supervisor = SupervisorHandle::new();
14759        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14760        let handler =
14761            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14762                .with_supervisor(supervisor);
14763        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
14764        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
14765        register_capability_manifest(
14766            &handler,
14767            &target_ctx,
14768            &mut target_rx,
14769            capability_manifest("target", &["credentials-provider/v1"], &[]),
14770            1,
14771        )
14772        .await;
14773        register_capability_manifest(
14774            &handler,
14775            &opener_ctx,
14776            &mut opener_rx,
14777            capability_manifest("opener", &[], &[]),
14778            2,
14779        )
14780        .await;
14781        let (mut client_rx, _) = open_route_for_capability_test(
14782            &handler,
14783            &target_ctx,
14784            &mut target_rx,
14785            732,
14786            3,
14787            "target",
14788            Some(ConsumerIdentity {
14789                module_id: "opener".to_string(),
14790                launch_nonce: "opener-nonce".to_string(),
14791            }),
14792        )
14793        .await;
14794        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14795
14796        handler
14797            .handle_control_frame(
14798                &target_ctx,
14799                catalog_update_with_capabilities_frame(
14800                    4,
14801                    CapabilityDeclarations {
14802                        provides: Vec::new(),
14803                        requires: Vec::new(),
14804                        must_never_reach: Vec::new(),
14805                    },
14806                ),
14807            )
14808            .await
14809            .expect("claim removal catalog.update succeeds");
14810        assert_eq!(
14811            forwarding.active_binding_count().unwrap(),
14812            1,
14813            "removing an attested target claim must leave the route census unchanged"
14814        );
14815        assert!(
14816            client_rx.try_recv().is_err(),
14817            "claim removal must not emit route.closed capability_denied"
14818        );
14819        assert!(
14820            target_rx.try_recv().is_err(),
14821            "claim removal must not send the target a route GOODBYE"
14822        );
14823    }
14824
14825    /// A direct client may open a route to a denied capability provider; this
14826    /// policy applies only to attested supervised module origins, not to direct clients.
14827    #[tokio::test]
14828    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
14829        let registry = Arc::new(Registry::default());
14830        let forwarding = Arc::new(ForwardingTable::default());
14831        let supervisor = SupervisorHandle::new();
14832        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14833        let handler =
14834            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14835                .with_supervisor(supervisor);
14836        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
14837        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
14838        register_capability_manifest(
14839            &handler,
14840            &target_ctx,
14841            &mut target_rx,
14842            capability_manifest("target", &["credentials-provider/v1"], &[]),
14843            1,
14844        )
14845        .await;
14846        register_capability_manifest(
14847            &handler,
14848            &opener_ctx,
14849            &mut opener_rx,
14850            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14851            2,
14852        )
14853        .await;
14854
14855        let (_client_rx, bind) = open_route_for_capability_test(
14856            &handler,
14857            &target_ctx,
14858            &mut target_rx,
14859            742,
14860            3,
14861            "target",
14862            None,
14863        )
14864        .await;
14865        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14866            panic!("direct scope-honesty route must bind");
14867        };
14868        assert_eq!(principal, Some(Principal::Direct));
14869        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14870    }
14871
14872    /// A module that denies a capability receives no self-route exemption when it
14873    /// also attestedly provides that capability.
14874    #[tokio::test]
14875    async fn must_never_reach_self_route_is_capability_forbidden() {
14876        let registry = Arc::new(Registry::default());
14877        let forwarding = Arc::new(ForwardingTable::default());
14878        let supervisor = SupervisorHandle::new();
14879        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
14880        let handler =
14881            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14882                .with_supervisor(supervisor);
14883        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
14884        register_capability_manifest(
14885            &handler,
14886            &self_ctx,
14887            &mut self_rx,
14888            capability_manifest(
14889                "self-provider",
14890                &["credentials-provider/v1"],
14891                &["credentials-provider/v1"],
14892            ),
14893            1,
14894        )
14895        .await;
14896
14897        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
14898        let replies = handler
14899            .handle_control_frame(
14900                &client_ctx,
14901                route_open_frame_with_admission_facts(
14902                    2,
14903                    "self-provider",
14904                    unique_project_root("admission-facts"),
14905                    Some(ConsumerIdentity {
14906                        module_id: "self-provider".to_string(),
14907                        launch_nonce: "self-nonce".to_string(),
14908                    }),
14909                    None,
14910                ),
14911            )
14912            .await
14913            .expect("self-route refusal returns a typed frame");
14914        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14915        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14916        assert!(
14917            self_rx.try_recv().is_err(),
14918            "self denial must not relay route.bind"
14919        );
14920    }
14921
14922    #[test]
14923    fn unsupported_channel_zero_frame_returns_error() {
14924        let handler = ControlHandler::default();
14925        let request = Frame::build(
14926            FrameType::Request,
14927            control_flags(),
14928            0,
14929            0,
14930            21,
14931            b"opaque".to_vec(),
14932        )
14933        .unwrap();
14934
14935        let response = handler
14936            .handle_control(ConnectionId::new(1), request)
14937            .unwrap();
14938
14939        assert_eq!(response[0].header.ty, FrameType::Error);
14940        assert_eq!(
14941            parse_error(&response[0])["code"],
14942            "unsupported_control_frame"
14943        );
14944    }
14945
14946    /// Blue/green swap at the control-plane boundary. The supervisor that opens
14947    /// a swap is not wired yet, so the candidate is registered here directly
14948    /// into the registry and forwarding candidate slots, the way the swap's
14949    /// HELLO admission will.
14950    mod swap {
14951        use super::*;
14952
14953        const INCUMBENT: ConnectionId = ConnectionId::new(30);
14954        const CANDIDATE: ConnectionId = ConnectionId::new(40);
14955
14956        struct Swap {
14957            registry: Arc<Registry>,
14958            forwarding: Arc<ForwardingTable>,
14959            handler: ControlHandler,
14960            incumbent_ctx: RouteCtx,
14961            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14962            candidate_ctx: RouteCtx,
14963            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14964        }
14965
14966        async fn swap_with_incumbent() -> Swap {
14967            let registry = Arc::new(Registry::default());
14968            let forwarding = Arc::new(ForwardingTable::default());
14969            let handler =
14970                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14971            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14972            hello_via_sink(
14973                &handler,
14974                &incumbent_ctx,
14975                &mut incumbent_rx,
14976                hello_frame("aft", PROTOCOL_VERSION, 7),
14977            )
14978            .await;
14979            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14980            Swap {
14981                registry,
14982                forwarding,
14983                handler,
14984                incumbent_ctx,
14985                incumbent_rx,
14986                candidate_ctx,
14987                candidate_rx,
14988            }
14989        }
14990
14991        fn register_candidate(swap: &Swap, ready: Option<bool>) {
14992            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14993            candidate_manifest.ready = ready;
14994            let registration = swap
14995                .registry
14996                .register_candidate_with_control_ops(
14997                    candidate_manifest,
14998                    PROTOCOL_VERSION,
14999                    CANDIDATE,
15000                    module_baseline_control_ops(),
15001                )
15002                .unwrap();
15003            swap.forwarding
15004                .register_candidate_module_connection(
15005                    CANDIDATE,
15006                    "aft".to_string(),
15007                    PROTOCOL_VERSION,
15008                    manifest_concurrency(&registration.manifest),
15009                    swap.candidate_ctx.egress.clone(),
15010                )
15011                .unwrap();
15012        }
15013
15014        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15015            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15016            swap.registry.promote_candidate("aft").unwrap().unwrap();
15017            cutover.incumbent.unwrap()
15018        }
15019
15020        /// A swap candidate's HELLO reserves its candidate registry entry
15021        /// before installing its candidate forwarding endpoint. The swap
15022        /// supervisor promotes a ready candidate as soon as it reads it as
15023        /// registered, and promotion cuts forwarding over to the candidate
15024        /// endpoint, so the lookup the supervisor reads
15025        /// (`Registry::published_registration`) must not return the candidate
15026        /// until that endpoint exists. Otherwise the cutover finds no candidate
15027        /// endpoint and the swap fails as if the candidate had disconnected.
15028        #[tokio::test]
15029        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15030            let mut swap = swap_with_incumbent().await;
15031            let hold = Arc::new(std::sync::Barrier::new(2));
15032            let handler = swap
15033                .handler
15034                .clone()
15035                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15036            handler
15037                .supervisor
15038                .open_swap("aft", "candidate-nonce".to_string());
15039            let hello = {
15040                let handler = handler.clone();
15041                let candidate_ctx = swap.candidate_ctx.clone();
15042                std::thread::spawn(move || {
15043                    handler
15044                        .handle_hello(
15045                            candidate_ctx.connection_id,
15046                            Some(candidate_ctx.egress.clone()),
15047                            hello_frame_with_nonce(
15048                                "aft",
15049                                PROTOCOL_VERSION,
15050                                8,
15051                                Some("candidate-nonce"),
15052                            ),
15053                        )
15054                        .unwrap()
15055                })
15056            };
15057
15058            hold.wait();
15059            // Parked: the candidate slot is claimed but not published, and
15060            // forwarding has no candidate endpoint.
15061            assert!(swap
15062                .registry
15063                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
15064                .unwrap()
15065                .is_some());
15066            assert!(swap
15067                .registry
15068                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15069                .unwrap()
15070                .is_none());
15071            assert!(swap
15072                .forwarding
15073                .module_endpoint_for_connection(CANDIDATE)
15074                .unwrap()
15075                .is_none());
15076
15077            hold.wait();
15078            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
15079            assert_eq!(
15080                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
15081                FrameType::HelloAck
15082            );
15083            let published = swap
15084                .registry
15085                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
15086                .unwrap()
15087                .expect("candidate is published once its HELLO completes");
15088            assert_eq!(published.connection_id, CANDIDATE);
15089            assert!(swap
15090                .forwarding
15091                .module_endpoint_for_connection(CANDIDATE)
15092                .unwrap()
15093                .is_some());
15094        }
15095
15096        fn keyed_total(counters: &Value, key: &str) -> u64 {
15097            counters[key]
15098                .as_object()
15099                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
15100                .unwrap_or(0)
15101        }
15102
15103        #[tokio::test]
15104        async fn replacement_logs_old_end_and_new_admission_once() {
15105            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15106            let mut swap = swap_with_incumbent().await;
15107            swap.handler
15108                .supervisor
15109                .open_swap("aft", "candidate-nonce".to_string());
15110            hello_via_sink(
15111                &swap.handler,
15112                &swap.candidate_ctx,
15113                &mut swap.candidate_rx,
15114                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
15115            )
15116            .await;
15117            cutover(&swap);
15118            swap.handler.cleanup_connection(INCUMBENT).unwrap();
15119
15120            let captured = crate::router::test_log::captured_logs(&logs);
15121            println!("captured replacement registry logs:\n{captured}");
15122            let new_admissions: Vec<_> = captured
15123                .lines()
15124                .filter(|line| {
15125                    line.contains("connection_id=40")
15126                        && line.contains("swap candidate registered; not routable until cutover")
15127                })
15128                .collect();
15129            assert_eq!(
15130                new_admissions.len(),
15131                1,
15132                "unexpected admission logs: {captured}"
15133            );
15134            assert!(new_admissions[0].contains("module_id=aft"));
15135            assert!(new_admissions[0].contains("connection_id=40"));
15136            assert!(
15137                new_admissions[0].contains("swap candidate registered; not routable until cutover")
15138            );
15139            assert!(new_admissions[0].contains("ready=true"));
15140
15141            let old_admissions: Vec<_> = captured
15142                .lines()
15143                .filter(|line| {
15144                    line.contains("module registered module_id=aft ")
15145                        && line.contains("connection_id=30")
15146                })
15147                .collect();
15148            assert_eq!(
15149                old_admissions.len(),
15150                1,
15151                "unexpected incumbent admission: {captured}"
15152            );
15153
15154            let old_ends: Vec<_> = captured
15155                .lines()
15156                .filter(|line| {
15157                    line.contains("connection_id=30") && line.contains("module registration ended")
15158                })
15159                .collect();
15160            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
15161            assert!(old_ends[0].contains("module_id=aft"));
15162            assert!(old_ends[0].contains("connection_id=30"));
15163            assert!(old_ends[0].contains("reason=replaced"));
15164            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
15165            assert_eq!(
15166                captured
15167                    .lines()
15168                    .filter(|line| line.contains("module registration promoted"))
15169                    .count(),
15170                1,
15171                "unexpected promotion logs: {captured}"
15172            );
15173        }
15174
15175        /// An ack from the incumbent for a bind it was sent before cutover,
15176        /// arriving before the incumbent is drained. The incumbent is the live
15177        /// connection carrying every other client's routes, so the ack must
15178        /// not end it: the waiting client is told to retry, the reservation is
15179        /// given back, and the incumbent is told to drop just that binding.
15180        #[tokio::test]
15181        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
15182            let mut swap = swap_with_incumbent().await;
15183            let handler = swap.handler.clone();
15184
15185            // A co-tenant route, bound on the incumbent before the swap.
15186            let cotenant = ConnectionId::new(31);
15187            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
15188            let (cotenant_task, cotenant_bind) = relay_route_open(
15189                &handler,
15190                cotenant,
15191                &cotenant_ctx.egress,
15192                &mut swap.incumbent_rx,
15193                100,
15194                "aft",
15195                "swap-cotenant",
15196            )
15197            .await;
15198            handler
15199                .handle_control_frame(
15200                    &swap.incumbent_ctx,
15201                    route_bind_ack(cotenant_bind.header.corr),
15202                )
15203                .await
15204                .unwrap();
15205            assert!(cotenant_task.await.unwrap().is_empty());
15206            let (cotenant_channel, cotenant_epoch) =
15207                published_route(&cotenant_rx.recv().await.unwrap());
15208
15209            // A second route.open, relayed to the incumbent and not yet acked.
15210            let caller = ConnectionId::new(32);
15211            let (caller_ctx, mut caller_rx) = route_ctx(caller);
15212            let (caller_task, caller_bind) = relay_route_open(
15213                &handler,
15214                caller,
15215                &caller_ctx.egress,
15216                &mut swap.incumbent_rx,
15217                101,
15218                "aft",
15219                "swap-caller",
15220            )
15221            .await;
15222            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
15223
15224            register_candidate(&swap, None);
15225            cutover(&swap);
15226
15227            // The incumbent acks after promotion and before any drain.
15228            let ack = handler
15229                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
15230                .await;
15231            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
15232            if module_loop_error.is_some() {
15233                // What the connection loop does with an untranslated router
15234                // error: end the connection, releasing every route on it.
15235                handler.cleanup_connection(INCUMBENT).unwrap();
15236            }
15237
15238            // 1. The incumbent's other routes survive.
15239            assert!(
15240                cotenant_rx.try_recv().is_err(),
15241                "the co-tenant route on the incumbent was torn down by one late ack: \
15242                 {module_loop_error:?}"
15243            );
15244            assert!(matches!(
15245                swap.forwarding
15246                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
15247                    .unwrap(),
15248                DataRoute::Client(DataRouteState::Bound(_))
15249            ));
15250            assert_eq!(module_loop_error, None);
15251            assert!(swap
15252                .registry
15253                .get_module_by_connection(INCUMBENT)
15254                .unwrap()
15255                .is_some());
15256
15257            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
15258            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
15259                .await
15260                .expect("the incumbent is told to drop the abandoned binding")
15261                .unwrap()
15262                .frame;
15263            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15264            assert_eq!(goodbye.header.channel, abandoned_channel);
15265            assert_eq!(goodbye.header.epoch, abandoned_epoch);
15266            assert!(swap.incumbent_rx.try_recv().is_err());
15267
15268            // 3. The waiting client gets a retryable refusal and no route.
15269            let response = caller_task.await.unwrap();
15270            assert_eq!(response.len(), 1);
15271            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15272            assert!(caller_rx.try_recv().is_err());
15273
15274            // 4. The reservation pair is given back, and the pending bind
15275            //    settled exactly once: one accepted open (the co-tenant) and one
15276            //    refused open (the caller), nothing counted twice.
15277            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
15278            let counters = handler.counters().snapshot();
15279            assert_eq!(
15280                keyed_total(&counters, "route_open_accepted_by_principal"),
15281                1
15282            );
15283            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
15284            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
15285        }
15286
15287        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
15288        /// id would resolve to the promoted candidate and every new route.open
15289        /// would be refused as reloading, leaving neither process routable.
15290        #[tokio::test]
15291        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
15292            let mut swap = swap_with_incumbent().await;
15293            register_candidate(&swap, None);
15294            let incumbent = cutover(&swap);
15295            swap.forwarding
15296                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
15297                .unwrap()
15298                .expect("the incumbent is still registered");
15299
15300            let client = ConnectionId::new(33);
15301            let (client_ctx, mut client_rx) = route_ctx(client);
15302            let route_handler = swap.handler.clone();
15303            let open_ctx = RouteCtx {
15304                connection_id: client,
15305                egress: client_ctx.egress.clone(),
15306            };
15307            let mut route_task = tokio::spawn(async move {
15308                route_handler
15309                    .handle_control_frame(
15310                        &open_ctx,
15311                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
15312                    )
15313                    .await
15314                    .unwrap()
15315            });
15316            let bind = tokio::select! {
15317                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
15318                response = &mut route_task => {
15319                    let response = response.unwrap();
15320                    panic!(
15321                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
15322                        parse_error(&response[0])["code"]
15323                    );
15324                }
15325            };
15326            swap.handler
15327                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
15328                .await
15329                .unwrap();
15330            assert!(route_task.await.unwrap().is_empty());
15331            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
15332            match swap
15333                .forwarding
15334                .lookup_data_route(client, channel, epoch)
15335                .unwrap()
15336            {
15337                DataRoute::Client(DataRouteState::Bound(route)) => {
15338                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
15339                }
15340                other => panic!("expected a bound route on the candidate, got {other:?}"),
15341            }
15342            assert!(swap.incumbent_rx.try_recv().is_err());
15343        }
15344
15345        /// A candidate declares itself ready with `catalog.update` on its own
15346        /// connection. If the connection-keyed registry lookups searched only the
15347        /// active slot, this would answer `not_registered` and the candidate
15348        /// would never become ready.
15349        #[tokio::test]
15350        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
15351            let swap = swap_with_incumbent().await;
15352            register_candidate(&swap, Some(false));
15353            let update = Frame::build(
15354                FrameType::Request,
15355                control_flags(),
15356                0,
15357                0,
15358                55,
15359                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15360                    provides: manifest("aft", PROTOCOL_VERSION).provides,
15361                    capabilities: None,
15362                    ready: Some(true),
15363                })
15364                .unwrap(),
15365            )
15366            .unwrap();
15367
15368            let replies = swap
15369                .handler
15370                .handle_control_frame(&swap.candidate_ctx, update)
15371                .await
15372                .unwrap();
15373
15374            assert_eq!(replies.len(), 1);
15375            assert_eq!(
15376                replies[0].header.ty,
15377                FrameType::Response,
15378                "candidate catalog.update was refused: {:?}",
15379                serde_json::from_slice::<Value>(&replies[0].body).ok()
15380            );
15381            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
15382            assert_eq!(
15383                swap.registry
15384                    .get_module("aft")
15385                    .unwrap()
15386                    .unwrap()
15387                    .connection_id,
15388                INCUMBENT
15389            );
15390        }
15391    }
15392
15393    /// The HELLO gate while the supervisor has a swap open: only the nonce it
15394    /// minted for the candidate admits a second process, into the candidate
15395    /// slot, and that check runs ahead of the reserved-module gate.
15396    mod swap_admission {
15397        use super::*;
15398
15399        const INCUMBENT_NONCE: &str = "incumbent-nonce";
15400        const CANDIDATE_NONCE: &str = "candidate-nonce";
15401
15402        fn handler_with_incumbent(
15403            module_id: &str,
15404            reserved: bool,
15405        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
15406            let registry = Arc::new(Registry::default());
15407            let supervisor = SupervisorHandle::new();
15408            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
15409            if reserved {
15410                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
15411            }
15412            let handler =
15413                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
15414            let incumbent = handler
15415                .handle_control(
15416                    ConnectionId::new(1),
15417                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
15418                )
15419                .unwrap();
15420            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
15421            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
15422            (registry, supervisor, handler)
15423        }
15424
15425        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
15426        /// admits every nonce, so while a swap is open the swap gate is the only
15427        /// thing between a key-holder and the candidate slot. A nonce the
15428        /// supervisor did not mint, or none at all, is refused, and neither the
15429        /// incumbent's registration nor the candidate slot moves.
15430        #[test]
15431        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15432            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15433
15434            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15435                let replies = handler
15436                    .handle_control(
15437                        ConnectionId::new(connection),
15438                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15439                    )
15440                    .unwrap();
15441                assert_eq!(replies[0].header.ty, FrameType::Error);
15442                assert_eq!(
15443                    parse_error(&replies[0])["code"],
15444                    "swap_token_invalid",
15445                    "nonce {nonce:?}"
15446                );
15447            }
15448            assert!(registry.get_candidate("aft").unwrap().is_none());
15449            assert_eq!(
15450                registry.get_module("aft").unwrap().unwrap().connection_id,
15451                ConnectionId::new(1)
15452            );
15453
15454            // Control: the minted token is admitted, into the candidate slot,
15455            // and only once.
15456            let admitted = handler
15457                .handle_control(
15458                    ConnectionId::new(4),
15459                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15460                )
15461                .unwrap();
15462            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15463            assert_eq!(
15464                registry
15465                    .get_candidate("aft")
15466                    .unwrap()
15467                    .unwrap()
15468                    .connection_id,
15469                ConnectionId::new(4)
15470            );
15471            assert_eq!(
15472                registry.get_module("aft").unwrap().unwrap().connection_id,
15473                ConnectionId::new(1),
15474                "the candidate must not take the active slot"
15475            );
15476            let replayed = handler
15477                .handle_control(
15478                    ConnectionId::new(5),
15479                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15480                )
15481                .unwrap();
15482            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15483
15484            // The case only this gate covers: the incumbent has died mid-swap,
15485            // so its duplicate refusal is gone too, and without the gate a
15486            // key-holder would take the id's ACTIVE slot.
15487            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15488            let squatter = handler
15489                .handle_control(
15490                    ConnectionId::new(6),
15491                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15492                )
15493                .unwrap();
15494            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15495            assert!(
15496                registry.get_module("aft").unwrap().is_none(),
15497                "a squatter took the active slot of an id being swapped"
15498            );
15499        }
15500
15501        /// Design mutation arm (iii). A reserved module's candidate presents a
15502        /// nonce the reserved gate has never seen (that gate holds the
15503        /// incumbent's), so the swap gate must run first or the candidate is
15504        /// refused `reserved_module` and a reserved module can never be swapped.
15505        #[test]
15506        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15507            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15508
15509            let replies = handler
15510                .handle_control(
15511                    ConnectionId::new(2),
15512                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15513                )
15514                .unwrap();
15515
15516            assert_eq!(
15517                replies[0].header.ty,
15518                FrameType::HelloAck,
15519                "reserved candidate refused: {:?}",
15520                serde_json::from_slice::<Value>(&replies[0].body).ok()
15521            );
15522            assert_eq!(
15523                registry
15524                    .get_candidate("vault")
15525                    .unwrap()
15526                    .unwrap()
15527                    .connection_id,
15528                ConnectionId::new(2)
15529            );
15530        }
15531
15532        /// With no swap open the gate is inert: the incumbent's reserved gate
15533        /// and duplicate refusal behave exactly as before.
15534        #[test]
15535        fn without_an_open_swap_the_ordinary_gates_decide() {
15536            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
15537            supervisor.close_swap("vault");
15538
15539            let candidate = handler
15540                .handle_control(
15541                    ConnectionId::new(2),
15542                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15543                )
15544                .unwrap();
15545            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
15546            let duplicate = handler
15547                .handle_control(
15548                    ConnectionId::new(3),
15549                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15550                )
15551                .unwrap();
15552            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15553            assert!(registry.get_candidate("vault").unwrap().is_none());
15554        }
15555    }
15556
15557    /// `scope.sync` and `scope.describe` through the real control handler: who
15558    /// may sync is decided by the registration and launch nonce of the module
15559    /// connection, never by the request body.
15560    mod scopes {
15561        mod idle_tests {
15562            use super::*;
15563            include!("scope_idle_tests.rs");
15564        }
15565        mod route_open_timings {
15566            use super::*;
15567            include!("route_open_timing_tests.rs");
15568        }
15569        mod activity {
15570            include!("activity_scope_tests.rs");
15571        }
15572        mod delta {
15573            include!("scope_handler_delta_tests.rs");
15574        }
15575        mod call_modes {
15576            include!("scope_call_mode_tests.rs");
15577        }
15578        #[tokio::test]
15579        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
15580            let rig = rig().await;
15581            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
15582                "s".into(),
15583                1,
15584                "tool".into(),
15585            ))
15586            .unwrap();
15587            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
15588            let (direct, _rx) = wide_ctx(999);
15589            for (ctx, expected) in [
15590                (&rig.owner, FrameType::Response),
15591                (&direct, FrameType::Error),
15592            ] {
15593                let frame =
15594                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
15595                        .unwrap();
15596                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
15597                assert_eq!(replies[0].header.ty, expected);
15598                if expected == FrameType::Response {
15599                    let reply: subc_control::ScopeServedByReply =
15600                        serde_json::from_slice(&replies[0].body).unwrap();
15601                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
15602                } else {
15603                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
15604                }
15605            }
15606            let reply = call(
15607                &rig.handler,
15608                &rig.owner,
15609                &ModuleControlRequestFromModule::ScopeDescribe {
15610                    owner: Principal::Reserved {
15611                        module_id: OWNER.into(),
15612                    },
15613                    scope_ref: "s".into(),
15614                },
15615            )
15616            .await;
15617            assert!(matches!(
15618                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
15619                ModuleControlResponseToModule::ScopeDescribe { .. }
15620            ));
15621        }
15622
15623        use subc_protocol::scope::{
15624            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
15625            ScopeStatus,
15626        };
15627
15628        use super::*;
15629
15630        const OWNER: &str = "prefrontal-core";
15631
15632        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
15633            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
15634        }
15635
15636        async fn call(
15637            handler: &ControlHandler,
15638            ctx: &RouteCtx,
15639            request: &ModuleControlRequestFromModule,
15640        ) -> Frame {
15641            let body = serde_json::to_vec(request).unwrap();
15642            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
15643            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
15644            assert_eq!(replies.len(), 1, "{replies:?}");
15645            replies.pop().unwrap()
15646        }
15647
15648        async fn sync(
15649            handler: &ControlHandler,
15650            ctx: &RouteCtx,
15651            generation: u64,
15652            scopes: Vec<ScopeRecord>,
15653        ) -> Result<ModuleControlResponseToModule, String> {
15654            let reply = call(
15655                handler,
15656                ctx,
15657                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
15658            )
15659            .await;
15660            match reply.header.ty {
15661                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
15662                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
15663            }
15664        }
15665
15666        async fn describe(
15667            handler: &ControlHandler,
15668            ctx: &RouteCtx,
15669            owner: &str,
15670            scope_ref: &str,
15671        ) -> ModuleControlResponseToModule {
15672            let reply = call(
15673                handler,
15674                ctx,
15675                &ModuleControlRequestFromModule::ScopeDescribe {
15676                    owner: Principal::Reserved {
15677                        module_id: owner.to_string(),
15678                    },
15679                    scope_ref: scope_ref.to_string(),
15680                },
15681            )
15682            .await;
15683            assert_eq!(
15684                reply.header.ty,
15685                FrameType::Response,
15686                "{:?}",
15687                parse_error(&reply)
15688            );
15689            serde_json::from_slice(&reply.body).unwrap()
15690        }
15691
15692        #[tokio::test]
15693        async fn scope_describe_never_echoes_bind_proof_or_hash() {
15694            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15695            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15696            let mut rig = rig().await;
15697            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15698                .await;
15699            let _bound = rig
15700                .bound(
15701                    Some(AFT),
15702                    PLEXUS,
15703                    Some(
15704                        rig.selector("s", Some(1))
15705                            .with_bind_proof(Some(PROOF.into())),
15706                    ),
15707                )
15708                .await;
15709            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
15710            let output = serde_json::to_string(&reply).unwrap();
15711            assert!(output.contains("\"status\":\"live\""));
15712            assert!(!output.contains(PROOF));
15713            assert!(!output.contains(HASH));
15714        }
15715
15716        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
15717        async fn module(
15718            handler: &ControlHandler,
15719            connection: u64,
15720            module_id: &str,
15721            nonce: Option<&str>,
15722        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15723            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
15724            hello_via_sink(
15725                handler,
15726                &ctx,
15727                &mut rx,
15728                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
15729            )
15730            .await;
15731            (ctx, rx)
15732        }
15733
15734        /// `direct` and every other client connection has no registration, so
15735        /// it can neither sync nor own a scope.
15736        #[tokio::test]
15737        async fn a_client_connection_cannot_sync_or_describe() {
15738            let handler = ControlHandler::new(Arc::new(Registry::default()));
15739            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
15740            for request in [
15741                ModuleControlRequestFromModule::ScopeSync {
15742                    generation: 1,
15743                    scopes: vec![head("s", 1)],
15744                },
15745                ModuleControlRequestFromModule::ScopeDescribe {
15746                    owner: Principal::Direct,
15747                    scope_ref: "s".to_string(),
15748                },
15749            ] {
15750                let reply = call(&handler, &ctx, &request).await;
15751                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
15752            }
15753            assert!(
15754                !handler
15755                    .scopes
15756                    .read()
15757                    .unwrap()
15758                    .describe(
15759                        &Principal::Reserved {
15760                            module_id: OWNER.to_string()
15761                        },
15762                        "s"
15763                    )
15764                    .owner_synced
15765            );
15766        }
15767
15768        /// A module the supervisor did not spawn registers without a launch
15769        /// nonce, so it is never an owner's current launch.
15770        #[tokio::test]
15771        async fn a_module_without_a_supervised_launch_cannot_sync() {
15772            let handler = ControlHandler::new(Arc::new(Registry::default()));
15773            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
15774            assert_eq!(
15775                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
15776                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15777            );
15778        }
15779
15780        #[tokio::test]
15781        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
15782            let supervisor = SupervisorHandle::new();
15783            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15784            let handler = ControlHandler::new(Arc::new(Registry::default()))
15785                .with_supervisor(supervisor.clone());
15786            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15787            sync(&handler, &incumbent, 1, vec![head("s", 1)])
15788                .await
15789                .expect("the current launch syncs");
15790
15791            // A swap candidate registers with the swap token and is refused
15792            // while the incumbent keeps syncing.
15793            supervisor.open_swap(OWNER, "n2".to_string());
15794            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
15795            assert_eq!(
15796                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15797                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15798            );
15799            sync(&handler, &incumbent, 2, vec![head("s", 1)])
15800                .await
15801                .expect("the serving owner syncs during the swap");
15802
15803            // The swap fails and is rolled back. The candidate never held sync
15804            // authority, and still cannot sync.
15805            supervisor.close_swap(OWNER);
15806            assert_eq!(
15807                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15808                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15809            );
15810            sync(&handler, &incumbent, 3, vec![head("s", 1)])
15811                .await
15812                .expect("the serving owner syncs after the rollback");
15813            handler.cleanup_connection(candidate.connection_id).unwrap();
15814
15815            // A swap that cuts over. Promotion records the candidate's nonce as
15816            // the module's spawn nonce, which is what `set_spawn_nonce` does
15817            // here; the promoted connection then takes authority at any
15818            // generation and the superseded incumbent is refused.
15819            supervisor.open_swap(OWNER, "n3".to_string());
15820            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
15821            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
15822            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
15823                .await
15824                .expect("the promoted launch takes authority");
15825            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15826                panic!("unexpected reply {reply:?}");
15827            };
15828            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
15829            assert_eq!(
15830                sync(&handler, &incumbent, 4, Vec::new()).await,
15831                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15832            );
15833        }
15834
15835        /// Authority dies with its connection: the cleanup path releases it,
15836        /// so the owner's next connection takes it at any generation.
15837        #[tokio::test]
15838        async fn closing_the_authority_connection_frees_sync_authority() {
15839            let supervisor = SupervisorHandle::new();
15840            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15841            let handler = ControlHandler::new(Arc::new(Registry::default()))
15842                .with_supervisor(supervisor.clone());
15843            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15844            sync(&handler, &first, 10, vec![head("s", 1)])
15845                .await
15846                .unwrap();
15847            handler.cleanup_connection(first.connection_id).unwrap();
15848
15849            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
15850            sync(&handler, &second, 1, vec![head("s", 1)])
15851                .await
15852                .expect("the next connection takes the released authority");
15853        }
15854
15855        #[tokio::test]
15856        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
15857            let supervisor = SupervisorHandle::new();
15858            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15859            let handler =
15860                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
15861            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
15862            sync(&handler, &first, 10, vec![head("s", 1)])
15863                .await
15864                .unwrap();
15865            handler
15866                .handle_control_frame(
15867                    &first,
15868                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
15869                )
15870                .await
15871                .unwrap();
15872            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
15873            sync(&handler, &second, 1, vec![head("s", 1)])
15874                .await
15875                .expect("GOODBYE releases authority even if the old socket remains open");
15876        }
15877
15878        #[tokio::test]
15879        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
15880            let registry = Arc::new(Registry::default());
15881            let supervisor_handle = SupervisorHandle::new();
15882            let supervisor =
15883                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
15884                    .with_handle(supervisor_handle.clone())
15885                    .with_daemon_incarnation("incarnation-7".to_string());
15886            // Configured with enabled: false, so the supervisor lists the
15887            // module without spawning a process for it.
15888            supervisor
15889                .supervise_configured(
15890                    ModuleSpec {
15891                        module_id: OWNER.to_string(),
15892                        program: PathBuf::from("/nonexistent/prefrontal-core"),
15893                        args: Vec::new(),
15894                        env: Vec::new(),
15895                        reserved: false,
15896                        reserved_prefixes: Vec::new(),
15897                        protocol: ModuleProtocol::Subc,
15898                        overlap: Default::default(),
15899                    },
15900                    false,
15901                )
15902                .unwrap();
15903            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
15904            let handler =
15905                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
15906            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
15907
15908            // Configured but not yet synced: a reader waits for the owner.
15909            let ModuleControlResponseToModule::ScopeDescribe {
15910                status,
15911                daemon_incarnation,
15912                owner_synced,
15913                owner_configured,
15914                scope,
15915                ..
15916            } = describe(&handler, &reader, OWNER, "s").await
15917            else {
15918                panic!("not a describe reply");
15919            };
15920            assert_eq!(status, ScopeStatus::NotLive);
15921            assert_eq!(daemon_incarnation, "incarnation-7");
15922            assert!(!owner_synced);
15923            assert!(owner_configured);
15924            assert!(scope.is_none());
15925
15926            // Not a supervised module: the owner will never sync, and a reader
15927            // refuses rather than waits.
15928            let ModuleControlResponseToModule::ScopeDescribe {
15929                status,
15930                owner_configured,
15931                ..
15932            } = describe(&handler, &reader, "ghost", "s").await
15933            else {
15934                panic!("not a describe reply");
15935            };
15936            assert_eq!(status, ScopeStatus::NotLive);
15937            assert!(!owner_configured);
15938
15939            // Live, with the stamp fields and the computed owner_authorized.
15940            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
15941            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
15942            let ModuleControlResponseToModule::ScopeDescribe {
15943                status,
15944                scope_epoch,
15945                owner_synced,
15946                scope,
15947                ..
15948            } = describe(&handler, &reader, OWNER, "s").await
15949            else {
15950                panic!("not a describe reply");
15951            };
15952            assert_eq!(status, ScopeStatus::Live);
15953            assert_eq!(scope_epoch, Some(4));
15954            assert!(owner_synced);
15955            let stamp = scope.expect("a live scope carries its stamp");
15956            assert!(
15957                stamp.owner_authorized,
15958                "prefrontal-core is the default authority"
15959            );
15960            assert_eq!(stamp.kind, ScopeKind::Head);
15961        }
15962
15963        #[tokio::test]
15964        async fn scope_authority_owners_decides_owner_authorized() {
15965            let supervisor = SupervisorHandle::new();
15966            supervisor.set_spawn_nonce("broca", "b1".to_string());
15967            let handler = ControlHandler::new(Arc::new(Registry::default()))
15968                .with_supervisor(supervisor)
15969                .with_scope_authority_owners(vec!["broca".to_string()]);
15970            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
15971            let mut gated = head("s", 1);
15972            gated.attributes.agent_id = Some("agent".to_string());
15973            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
15974            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
15975                describe(&handler, &broca, "broca", "s").await
15976            else {
15977                panic!("not a describe reply");
15978            };
15979            assert!(scope.unwrap().owner_authorized);
15980        }
15981
15982        /// With route admission, the stamp, the commit re-check and drains in
15983        /// place, the feature is advertised: the module ops in HELLO_ACK, and
15984        /// `scopes/v1` in HELLO_ACK and `server.describe`.
15985        #[tokio::test]
15986        async fn scope_ops_and_the_scopes_capability_are_advertised() {
15987            let handler = ControlHandler::new(Arc::new(Registry::default()));
15988            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
15989            let ack = hello_via_sink(
15990                &handler,
15991                &ctx,
15992                &mut rx,
15993                hello_frame("m", PROTOCOL_VERSION, 1),
15994            )
15995            .await;
15996            let ack = parse_ack(&ack);
15997            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
15998                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
15999            }
16000            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16001
16002            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16003            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16004            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16005            let reply = handler
16006                .handle_control_frame(&client, frame)
16007                .await
16008                .unwrap()
16009                .pop()
16010                .unwrap();
16011            let ClientControlResponse::ServerDescribe { capabilities, .. } =
16012                serde_json::from_slice(&reply.body).unwrap()
16013            else {
16014                panic!("not a server.describe reply");
16015            };
16016            assert!(
16017                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16018                "{capabilities:?}"
16019            );
16020        }
16021
16022        // ---- route admission, stamps, commit re-check and drains ----------
16023
16024        const PLEXUS: &str = "plexus";
16025        const OTHER: &str = "other";
16026        const AFT: &str = "aft";
16027        const BROCA: &str = "broca";
16028        const MAGIC: &str = "magic-context";
16029
16030        fn nonce(module_id: &str) -> String {
16031            format!("nonce-{module_id}")
16032        }
16033
16034        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16035            let (tx, rx) = mpsc::channel(64);
16036            (
16037                RouteCtx {
16038                    connection_id: ConnectionId::new(connection),
16039                    egress: FrameSink::new(tx),
16040                },
16041                rx,
16042            )
16043        }
16044
16045        /// A daemon with a configured owner (prefrontal-core) registered on its
16046        /// own module connection, two routable targets (plexus, other), and
16047        /// launch nonces minted for the modules that open routes as carriers.
16048        struct Rig {
16049            handler: ControlHandler,
16050            forwarding: Arc<ForwardingTable>,
16051            owner: RouteCtx,
16052            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16053            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
16054            generation: u64,
16055            next_connection: u64,
16056            _supervisor: Supervisor,
16057        }
16058
16059        async fn rig() -> Rig {
16060            rig_with_flow_support(true).await
16061        }
16062
16063        async fn rig_with_flow_support(flow_support: bool) -> Rig {
16064            rig_with_scope_capabilities(flow_support, false).await
16065        }
16066
16067        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
16068            let mut provides = Vec::new();
16069            if flow_support {
16070                provides.push("flow-scopes/v1");
16071            }
16072            if run_support {
16073                provides.push("agent-run-scopes/v1");
16074            }
16075            rig_with_target_capabilities(&provides).await
16076        }
16077
16078        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
16079            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
16080            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16081            // Decoder versions alone cannot admit scoped calls: only a capability
16082            // declaration promises the required run or flow behaviour.
16083            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16084            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
16085            Frame::build(
16086                FrameType::Hello,
16087                control_flags(),
16088                0,
16089                0,
16090                connection,
16091                serde_json::to_vec(&body).unwrap(),
16092            )
16093            .unwrap()
16094        }
16095
16096        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
16097            let registry = Arc::new(Registry::default());
16098            let forwarding = Arc::new(ForwardingTable::default());
16099            let supervisor_handle = SupervisorHandle::new();
16100            let supervisor =
16101                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16102                    .with_handle(supervisor_handle.clone());
16103            supervisor
16104                .supervise_configured(
16105                    ModuleSpec {
16106                        module_id: OWNER.to_string(),
16107                        program: PathBuf::from("/nonexistent/prefrontal-core"),
16108                        args: Vec::new(),
16109                        env: Vec::new(),
16110                        reserved: false,
16111                        reserved_prefixes: Vec::new(),
16112                        protocol: ModuleProtocol::Subc,
16113                        overlap: Default::default(),
16114                    },
16115                    false,
16116                )
16117                .unwrap();
16118            for module_id in [OWNER, AFT, BROCA, MAGIC] {
16119                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
16120            }
16121            let handler =
16122                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16123                    .with_supervisor(supervisor_handle);
16124            let (owner, mut owner_rx) = wide_ctx(1);
16125            hello_via_sink(
16126                &handler,
16127                &owner,
16128                &mut owner_rx,
16129                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
16130            )
16131            .await;
16132            let mut modules = BTreeMap::new();
16133            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
16134                let (ctx, mut rx) = wide_ctx(connection);
16135                let hello = hello_with_capabilities(module_id, connection, provides);
16136                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
16137                modules.insert(module_id.to_string(), (ctx, rx));
16138            }
16139            Rig {
16140                handler,
16141                forwarding,
16142                owner,
16143                _owner_rx: owner_rx,
16144                modules,
16145                generation: 0,
16146                next_connection: 100,
16147                _supervisor: supervisor,
16148            }
16149        }
16150
16151        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
16152            ScopeCarrier::new(Principal::Reserved {
16153                module_id: module_id.to_string(),
16154            })
16155            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
16156        }
16157
16158        /// The scope most tests open under: aft carries to any module, broca
16159        /// only to plexus and other, and the owner delegates as agent-1.
16160        fn session(scope_epoch: u64) -> ScopeRecord {
16161            let mut record = head("s", scope_epoch);
16162            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
16163            record.attributes.agent_id = Some("agent-1".to_string());
16164            record.attributes.delegates = true;
16165            record
16166        }
16167
16168        impl Rig {
16169            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
16170                self.generation += 1;
16171                sync(&self.handler, &self.owner, self.generation, scopes)
16172                    .await
16173                    .expect("the owner's sync is accepted");
16174            }
16175
16176            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16177                ScopeSelector::new(
16178                    Principal::Reserved {
16179                        module_id: OWNER.to_string(),
16180                    },
16181                    scope_ref.to_string(),
16182                    scope_epoch,
16183                )
16184            }
16185
16186            fn open_frame(
16187                &mut self,
16188                opener: Option<&str>,
16189                target: &str,
16190                scope: Option<ScopeSelector>,
16191            ) -> (
16192                RouteCtx,
16193                mpsc::Receiver<crate::router::OutboundFrame>,
16194                Frame,
16195            ) {
16196                self.next_connection += 1;
16197                let (ctx, rx) = wide_ctx(self.next_connection);
16198                let root = unique_project_root("scoped-open");
16199                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
16200                    target: RouteTarget::ToolProvider {
16201                        module_id: target.to_string(),
16202                    },
16203                    identity: BindIdentity::new(
16204                        root.path().to_path_buf(),
16205                        "unit".to_string(),
16206                        "session".to_string(),
16207                    ),
16208                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
16209                        module_id: module_id.to_string(),
16210                        launch_nonce: nonce(module_id),
16211                    }),
16212                    consumer_capabilities: None,
16213                    role_versions: None,
16214                    admission_facts: None,
16215                    scope,
16216                })
16217                .unwrap();
16218                let frame = Frame::build(
16219                    FrameType::Request,
16220                    control_flags(),
16221                    0,
16222                    0,
16223                    self.next_connection,
16224                    body,
16225                )
16226                .unwrap();
16227                (ctx, rx, frame)
16228            }
16229
16230            /// Open and expect a refusal before anything is relayed.
16231            async fn refused(
16232                &mut self,
16233                opener: Option<&str>,
16234                target: &str,
16235                scope: Option<ScopeSelector>,
16236            ) -> String {
16237                self.refusal_body(opener, target, scope).await["code"]
16238                    .as_str()
16239                    .unwrap()
16240                    .to_string()
16241            }
16242
16243            async fn refusal_body(
16244                &mut self,
16245                opener: Option<&str>,
16246                target: &str,
16247                scope: Option<ScopeSelector>,
16248            ) -> Value {
16249                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
16250                let replies = tokio::time::timeout(
16251                    Duration::from_secs(2),
16252                    self.handler.handle_control_frame(&ctx, frame),
16253                )
16254                .await
16255                .expect("the open must be refused before waiting for a bind ack")
16256                .unwrap();
16257                assert_eq!(replies.len(), 1, "{replies:?}");
16258                assert_eq!(replies[0].header.ty, FrameType::Error);
16259                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16260                assert!(
16261                    module_rx.try_recv().is_err(),
16262                    "a refused open relays nothing"
16263                );
16264                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
16265                parse_error(&replies[0])
16266            }
16267
16268            /// Start an open and return its task and the bind the target got.
16269            async fn relayed(
16270                &mut self,
16271                opener: Option<&str>,
16272                target: &str,
16273                scope: Option<ScopeSelector>,
16274            ) -> Relayed {
16275                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
16276                let handler = self.handler.clone();
16277                let task_ctx = ctx.clone();
16278                let task = tokio::spawn(async move {
16279                    handler
16280                        .handle_control_frame(&task_ctx, frame)
16281                        .await
16282                        .unwrap()
16283                });
16284                let (_, module_rx) = self.modules.get_mut(target).unwrap();
16285                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
16286                    .await
16287                    .expect("the target receives the relayed route.bind")
16288                    .unwrap()
16289                    .frame;
16290                Relayed {
16291                    target: target.to_string(),
16292                    client: ctx,
16293                    client_rx: rx,
16294                    task,
16295                    bind,
16296                }
16297            }
16298
16299            async fn ack(&self, relayed: &Relayed) {
16300                let (module, _) = &self.modules[&relayed.target];
16301                self.handler
16302                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
16303                    .await
16304                    .unwrap();
16305            }
16306
16307            /// Open, ack and return the bound route.
16308            async fn bound(
16309                &mut self,
16310                opener: Option<&str>,
16311                target: &str,
16312                scope: Option<ScopeSelector>,
16313            ) -> Bound {
16314                let relayed = self.relayed(opener, target, scope).await;
16315                self.ack(&relayed).await;
16316                let Relayed {
16317                    target,
16318                    client,
16319                    mut client_rx,
16320                    task,
16321                    bind,
16322                } = relayed;
16323                assert!(
16324                    task.await.unwrap().is_empty(),
16325                    "the open is answered by commit"
16326                );
16327                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
16328                Bound {
16329                    target,
16330                    client,
16331                    client_rx,
16332                    channel,
16333                    epoch,
16334                    bind,
16335                }
16336            }
16337
16338            fn live(&self, route: &Bound) -> bool {
16339                matches!(
16340                    self.forwarding
16341                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16342                        .unwrap(),
16343                    DataRoute::Client(DataRouteState::Bound(_))
16344                )
16345            }
16346        }
16347
16348        struct Relayed {
16349            target: String,
16350            client: RouteCtx,
16351            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16352            task: tokio::task::JoinHandle<Vec<Frame>>,
16353            bind: Frame,
16354        }
16355
16356        struct Bound {
16357            target: String,
16358            client: RouteCtx,
16359            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16360            channel: u16,
16361            epoch: u32,
16362            bind: Frame,
16363        }
16364
16365        impl Bound {
16366            /// The reason of the `route.closed` this client was sent, after
16367            /// checking it also got a GOODBYE on exactly this route.
16368            fn closed_reason(&mut self) -> RouteCloseReason {
16369                let mut reason = None;
16370                let mut goodbye = false;
16371                while let Ok(outbound) = self.client_rx.try_recv() {
16372                    let frame = outbound.frame;
16373                    match frame.header.ty {
16374                        FrameType::Goodbye => {
16375                            assert_eq!(
16376                                (frame.header.channel, frame.header.epoch),
16377                                (self.channel, self.epoch)
16378                            );
16379                            goodbye = true;
16380                        }
16381                        FrameType::Push => {
16382                            let ClientControlPush::RouteClosed {
16383                                reason: r,
16384                                module_id,
16385                                ..
16386                            } = serde_json::from_slice(&frame.body).unwrap()
16387                            else {
16388                                panic!("unexpected push");
16389                            };
16390                            assert_eq!(module_id, self.target);
16391                            reason = Some(r);
16392                        }
16393                        other => panic!("unexpected frame {other:?}"),
16394                    }
16395                }
16396                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
16397                reason.expect("the client is told why the route closed")
16398            }
16399
16400            fn untouched(&mut self) -> bool {
16401                self.client_rx.try_recv().is_err()
16402            }
16403
16404            fn stamp(&self) -> Option<ScopeStamp> {
16405                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
16406                    ModuleControlRequest::RouteBind { scope, .. } => scope,
16407                    other => panic!("expected a route.bind, got {other:?}"),
16408                }
16409            }
16410        }
16411
16412        #[tokio::test]
16413        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
16414        ) {
16415            let mut rig = rig().await;
16416            let mut record = session(1);
16417            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
16418            record.child_owners = vec![Principal::Reserved {
16419                module_id: MAGIC.to_string(),
16420            }];
16421            rig.sync(vec![record]).await;
16422            let scope = || Some(rig_selector("s", Some(1)));
16423
16424            // Admitted: the owner, a bare carrier to any module, a targeted
16425            // carrier to its listed module.
16426            rig.bound(Some(OWNER), PLEXUS, scope()).await;
16427            rig.bound(Some(AFT), OTHER, scope()).await;
16428            rig.bound(Some(BROCA), PLEXUS, scope()).await;
16429
16430            // Refused scope_not_carrier: a targeted carrier to an unlisted
16431            // module, a module that is not listed at all (a child owner is not
16432            // a carrier), and a direct key-holder.
16433            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16434                assert_eq!(
16435                    rig.refused(opener, target, scope()).await,
16436                    error_codes::SCOPE_NOT_CARRIER,
16437                    "{opener:?} -> {target}"
16438                );
16439            }
16440        }
16441
16442        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16443            ScopeSelector::new(
16444                Principal::Reserved {
16445                    module_id: OWNER.to_string(),
16446                },
16447                scope_ref.to_string(),
16448                scope_epoch,
16449            )
16450        }
16451
16452        #[tokio::test]
16453        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16454            let mut rig = rig().await;
16455            rig.sync(vec![session(1)]).await;
16456            for opener in [OWNER, AFT] {
16457                assert_eq!(
16458                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16459                        .await,
16460                    error_codes::SCOPE_EPOCH_REQUIRED,
16461                    "{opener}"
16462                );
16463            }
16464        }
16465
16466        #[tokio::test]
16467        async fn admission_separates_not_synced_not_live_and_ended() {
16468            let mut rig = rig().await;
16469            // Before the configured owner's first sync: retryable.
16470            let code = rig
16471                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16472                .await;
16473            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16474            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16475
16476            // An owner that is not configured will never sync: terminal.
16477            let ghost = ScopeSelector::new(
16478                Principal::Reserved {
16479                    module_id: "ghost".to_string(),
16480                },
16481                "s".to_string(),
16482                Some(1),
16483            );
16484            assert_eq!(
16485                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16486                error_codes::SCOPE_NOT_LIVE
16487            );
16488
16489            rig.sync(vec![session(2)]).await;
16490            assert_eq!(
16491                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16492                    .await,
16493                error_codes::SCOPE_NOT_LIVE
16494            );
16495            for epoch in [1, 3] {
16496                assert_eq!(
16497                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16498                        .await,
16499                    error_codes::SCOPE_ENDED,
16500                    "epoch {epoch}"
16501                );
16502            }
16503            // Control: the live epoch is admitted.
16504            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16505                .await;
16506        }
16507
16508        #[tokio::test]
16509        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16510            let mut rig = rig().await;
16511            rig.sync(vec![session(1)]).await;
16512            let route = rig
16513                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16514                .await;
16515            let stamp = route.stamp().expect("a scoped bind carries the stamp");
16516            assert_eq!(stamp.scope_ref, "s");
16517            assert_eq!(stamp.scope_epoch, 1);
16518            assert_eq!(stamp.kind, ScopeKind::Head);
16519            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16520            assert!(stamp.attributes.delegates);
16521            assert!(stamp.owner_authorized);
16522            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16523            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16524
16525            // broca owns a scope of its own on its own module connection; it is
16526            // not in scope_authority_owners, so its stamp is not authorized.
16527            let (broca, mut broca_rx) = wide_ctx(50);
16528            hello_via_sink(
16529                &rig.handler,
16530                &broca,
16531                &mut broca_rx,
16532                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
16533            )
16534            .await;
16535            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
16536                .await
16537                .unwrap();
16538            let own = ScopeSelector::new(
16539                Principal::Reserved {
16540                    module_id: BROCA.to_string(),
16541                },
16542                "b".to_string(),
16543                Some(1),
16544            );
16545            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
16546            assert!(!route.stamp().unwrap().owner_authorized);
16547        }
16548
16549        #[tokio::test]
16550        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16551            let mut rig = rig().await;
16552            let mut record = head("s", 1);
16553            record.carriers = vec![carrier(AFT, None)];
16554            let flow_id = "Flow:run-7/step_2!~";
16555            record.attributes.flow_id = Some(flow_id.to_string());
16556            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16557                .await
16558                .unwrap();
16559            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16560                panic!("not a sync reply");
16561            };
16562            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
16563            let route = rig
16564                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16565                .await;
16566            assert!(rig.live(&route), "the stamped bind committed");
16567            let stamp = route.stamp().expect("a flow scope carries a stamp");
16568            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
16569            assert_eq!(stamp.attributes.agent_id, None);
16570            assert!(!stamp.attributes.delegates);
16571            assert!(stamp.owner_authorized);
16572            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16573            assert_eq!(unscoped.stamp(), None);
16574        }
16575
16576        #[tokio::test]
16577        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
16578            let mut rig = rig_with_flow_support(false).await;
16579            let mut record = session(1);
16580            record.attributes.flow_id = Some("flow:7".to_string());
16581            rig.sync(vec![record]).await;
16582            for opener in [OWNER, AFT] {
16583                let body = rig
16584                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16585                    .await;
16586                assert_eq!(body["code"], "target_flow_unsupported");
16587                let message = body["message"].as_str().unwrap();
16588                for required in [PLEXUS, "flow-scopes/v1"] {
16589                    assert!(message.contains(required), "{message}");
16590                }
16591            }
16592        }
16593
16594        #[tokio::test]
16595        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
16596            let mut rig = rig_with_flow_support(true).await;
16597            let mut record = session(1);
16598            record.attributes.flow_id = Some("flow:7".to_string());
16599            rig.sync(vec![record]).await;
16600            for opener in [OWNER, AFT] {
16601                let route = rig
16602                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16603                    .await;
16604                assert!(rig.live(&route));
16605                assert_eq!(
16606                    route.stamp().unwrap().attributes.flow_id.as_deref(),
16607                    Some("flow:7")
16608                );
16609            }
16610        }
16611
16612        #[tokio::test]
16613        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
16614            use std::future::Future;
16615
16616            let mut rig = rig_with_flow_support(true).await;
16617            let mut record = session(1);
16618            record.attributes.flow_id = Some("flow:7".to_string());
16619            rig.sync(vec![record]).await;
16620            let (client, mut client_rx, frame) =
16621                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
16622            // Hold the route-open response permit so admission sees the first
16623            // target but relay reservation cannot capture an endpoint yet.
16624            for _ in 0..64 {
16625                client.egress.try_send(route_bind_ack(1)).unwrap();
16626            }
16627            let handler = rig.handler.clone();
16628            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16629            std::future::poll_fn(|cx| {
16630                assert!(open.as_mut().poll(cx).is_pending());
16631                std::task::Poll::Ready(())
16632            })
16633            .await;
16634            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16635
16636            let old_connection = rig.modules[PLEXUS].0.connection_id;
16637            rig.handler.cleanup_connection(old_connection).unwrap();
16638            let (replacement, mut replacement_rx) = wide_ctx(200);
16639            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
16640            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16641            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16642            let hello = Frame::build(
16643                FrameType::Hello,
16644                control_flags(),
16645                0,
16646                0,
16647                200,
16648                serde_json::to_vec(&body).unwrap(),
16649            )
16650            .unwrap();
16651            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
16652
16653            client_rx.try_recv().unwrap();
16654            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16655                .await
16656                .expect("the replacement is refused without waiting for a bind ack")
16657                .unwrap();
16658            assert_eq!(replies.len(), 1);
16659            let body = parse_error(&replies[0]);
16660            assert_eq!(body["code"], "target_flow_unsupported");
16661            for required in [PLEXUS, "flow-scopes/v1"] {
16662                assert!(body["message"].as_str().unwrap().contains(required));
16663            }
16664            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
16665            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16666            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16667            assert!(rig
16668                .handler
16669                .registry
16670                .get_module_by_connection(replacement.connection_id)
16671                .unwrap()
16672                .is_some());
16673        }
16674
16675        #[tokio::test]
16676        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
16677            use std::future::Future;
16678            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16679            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16680            const NEXT_HASH: &str =
16681                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16682            let mut rig = rig().await;
16683            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
16684            record.carriers.push(ScopeCarrier::new(Principal::Direct));
16685            rig.sync(vec![record.clone()]).await;
16686            let (client, mut rx, frame) = rig.open_frame(
16687                None,
16688                PLEXUS,
16689                Some(
16690                    rig.selector("s", Some(1))
16691                        .with_bind_proof(Some(PROOF.into())),
16692                ),
16693            );
16694            // Hold egress reservation while the scope owner changes its bind-proof
16695            // hash, so a proof valid at admission must be checked again at relay.
16696            for _ in 0..64 {
16697                client.egress.try_send(route_bind_ack(1)).unwrap();
16698            }
16699            let handler = rig.handler.clone();
16700            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16701            std::future::poll_fn(|cx| {
16702                assert!(open.as_mut().poll(cx).is_pending());
16703                std::task::Poll::Ready(())
16704            })
16705            .await;
16706            record.bind_proof_sha256 = Some(NEXT_HASH.into());
16707            rig.sync(vec![record]).await;
16708            rx.try_recv().unwrap();
16709            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16710                .await
16711                .unwrap()
16712                .unwrap();
16713            assert_eq!(
16714                parse_error(&replies[0])["code"],
16715                error_codes::SCOPE_BIND_PROOF_MISMATCH
16716            );
16717            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
16718            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16719            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16720        }
16721
16722        #[tokio::test]
16723        async fn scope_bind_proof_reserved_openers_ignore_proof() {
16724            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16725            let mut rig = rig().await;
16726            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16727                .await;
16728            for opener in [OWNER, AFT] {
16729                for proof in [None, Some("malformed+proof=")] {
16730                    let route = rig
16731                        .bound(
16732                            Some(opener),
16733                            PLEXUS,
16734                            Some(
16735                                rig.selector("s", Some(1))
16736                                    .with_bind_proof(proof.map(str::to_string)),
16737                            ),
16738                        )
16739                        .await;
16740                    assert!(rig.live(&route));
16741                }
16742            }
16743        }
16744
16745        #[tokio::test]
16746        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
16747            use std::future::Future;
16748            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16749            const NEXT_HASH: &str =
16750                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16751            let mut rig = rig().await;
16752            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16753                .await;
16754            let (client, mut rx, frame) =
16755                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
16756            for _ in 0..64 {
16757                client.egress.try_send(route_bind_ack(1)).unwrap();
16758            }
16759            let handler = rig.handler.clone();
16760            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16761            std::future::poll_fn(|cx| {
16762                assert!(open.as_mut().poll(cx).is_pending());
16763                std::task::Poll::Ready(())
16764            })
16765            .await;
16766            rig.sync(vec![
16767                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
16768            ])
16769            .await;
16770            rx.try_recv().unwrap();
16771            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16772            let bind = std::future::poll_fn(|cx| {
16773                assert!(open.as_mut().poll(cx).is_pending());
16774                module_rx.poll_recv(cx)
16775            })
16776            .await
16777            .unwrap()
16778            .frame;
16779            handler
16780                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
16781                .await
16782                .unwrap();
16783            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16784                .await
16785                .unwrap()
16786                .unwrap();
16787            assert_eq!(replies.len(), 1);
16788            // Proof rechecking allowed the relay. Independent tag settlement
16789            // still asks the opener to retry after any same-epoch content change.
16790            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
16791            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16792            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
16793            let route = rig
16794                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
16795                .await;
16796            assert!(rig.live(&route));
16797        }
16798
16799        #[tokio::test]
16800        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
16801        {
16802            let mut rig = rig_with_flow_support(false).await;
16803            rig.sync(vec![session(1)]).await;
16804            let route = rig
16805                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16806                .await;
16807            assert!(rig.live(&route));
16808            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
16809            let mut record = session(1);
16810            record.attributes.flow_id = Some("flow:7".to_string());
16811            rig.sync(vec![record]).await;
16812            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
16813            assert!(rig.live(&unscoped));
16814            assert_eq!(unscoped.stamp(), None);
16815        }
16816
16817        #[tokio::test]
16818        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
16819            let mut rig = rig().await;
16820            let mut record = session(1);
16821            record.attributes.flow_id = Some("flow:7".to_string());
16822            rig.sync(vec![record.clone()]).await;
16823            let before = rig.forwarding.published_scope_tag(OWNER, "s");
16824            let mut owner_route = rig
16825                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16826                .await;
16827            let mut carrier_route = rig
16828                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
16829                .await;
16830            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16831            rig.sync(vec![record.clone()]).await;
16832            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16833            assert!(rig.live(&owner_route) && owner_route.untouched());
16834            assert!(rig.live(&carrier_route) && carrier_route.untouched());
16835
16836            record.attributes.flow_id = Some("flow:8".to_string());
16837            rig.sync(vec![record]).await;
16838            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
16839            let before = before.unwrap();
16840            assert_eq!(after.scope_epoch, before.scope_epoch);
16841            assert!(after.version > before.version);
16842            for route in [&mut owner_route, &mut carrier_route] {
16843                assert!(!rig.live(route));
16844                assert_eq!(
16845                    route.closed_reason(),
16846                    RouteCloseReason::ScopeDelegationChanged
16847                );
16848            }
16849            assert!(rig.live(&unscoped) && unscoped.untouched());
16850            // Each provider also receives a GOODBYE for its drained route;
16851            // consume it before expecting the next route.bind on that sink.
16852            for target in [PLEXUS, OTHER] {
16853                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
16854                let goodbye = module_rx
16855                    .try_recv()
16856                    .expect("the provider sees the drain")
16857                    .frame;
16858                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16859                assert!(module_rx.try_recv().is_err());
16860            }
16861            let rebound = rig
16862                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16863                .await;
16864            assert_eq!(
16865                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
16866                Some("flow:8")
16867            );
16868        }
16869
16870        /// The owner's sync lands between admission and the module's ack. The
16871        /// open is refused by name, the module's other routes stay up, and the
16872        /// reserved pair is released. Changed content is retryable; an ended
16873        /// scope is not.
16874        #[tokio::test]
16875        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
16876            let mut rig = rig().await;
16877            rig.sync(vec![session(1)]).await;
16878            let mut cotenant = rig
16879                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16880                .await;
16881
16882            let mut changed = session(1);
16883            changed.child_owners.push(Principal::Reserved {
16884                module_id: MAGIC.to_string(),
16885            });
16886            let mut ended = None;
16887            for (code, next) in [
16888                (error_codes::SCOPE_CHANGED, vec![changed]),
16889                (error_codes::SCOPE_ENDED, Vec::new()),
16890            ] {
16891                let relayed = rig
16892                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16893                    .await;
16894                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
16895                ended = Some(next.is_empty());
16896                rig.sync(next).await;
16897                rig.ack(&relayed).await;
16898                let replies = relayed.task.await.unwrap();
16899                assert_eq!(replies.len(), 1, "{replies:?}");
16900                assert_eq!(parse_error(&replies[0])["code"], code);
16901                assert_eq!(
16902                    subc_protocol::error_codes::is_retryable_route_open(code),
16903                    code == error_codes::SCOPE_CHANGED
16904                );
16905                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16906                // The module is told to drop just the binding it created.
16907                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16908                // Collected, because ending the scope also closes the co-tenant
16909                // route, whose GOODBYE comes first.
16910                let mut goodbyes = Vec::new();
16911                while let Ok(outbound) = plexus_rx.try_recv() {
16912                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
16913                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
16914                }
16915                assert!(
16916                    goodbyes.contains(&(bind_channel, bind_epoch)),
16917                    "{goodbyes:?}"
16918                );
16919                assert!(rig
16920                    .handler
16921                    .registry
16922                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
16923                    .unwrap()
16924                    .is_some());
16925            }
16926            assert_eq!(ended, Some(true));
16927            // The co-tenant stayed up through the change, and closed only when
16928            // the scope ended, by the drain rule rather than by the commit.
16929            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
16930        }
16931
16932        /// Each row of the drain table on one set of routes: the owner's, a
16933        /// bare carrier's, and a targeted carrier's to each of its targets.
16934        #[tokio::test]
16935        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
16936            struct Case {
16937                name: &'static str,
16938                change: fn(&mut ScopeRecord),
16939                /// Closed routes by index: owner->plexus, aft->plexus,
16940                /// broca->plexus, broca->other.
16941                closed: [Option<RouteCloseReason>; 4],
16942            }
16943            use RouteCloseReason::*;
16944            let cases = [
16945                Case {
16946                    name: "a carrier entry removed",
16947                    change: |r| {
16948                        r.carriers.retain(|c| {
16949                            c.principal
16950                                != Principal::Reserved {
16951                                    module_id: AFT.to_string(),
16952                                }
16953                        })
16954                    },
16955                    closed: [None, Some(ScopeCarrierRemoved), None, None],
16956                },
16957                Case {
16958                    name: "a target removed from a carrier",
16959                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
16960                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
16961                },
16962                Case {
16963                    name: "a bare carrier narrowed to targets",
16964                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
16965                    closed: [None, Some(ScopeCarrierRemoved), None, None],
16966                },
16967                Case {
16968                    name: "delegates turned off",
16969                    change: |r| r.attributes.delegates = false,
16970                    closed: [Some(ScopeDelegationChanged); 4],
16971                },
16972                Case {
16973                    name: "agent_id changed",
16974                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
16975                    closed: [Some(ScopeDelegationChanged); 4],
16976                },
16977                Case {
16978                    name: "a carrier added, child owners changed, the record re-sent",
16979                    change: |r| {
16980                        r.carriers.push(carrier(MAGIC, None));
16981                        r.child_owners.push(Principal::Reserved {
16982                            module_id: MAGIC.to_string(),
16983                        });
16984                    },
16985                    closed: [None; 4],
16986                },
16987                Case {
16988                    name: "a target added",
16989                    change: |r| {
16990                        r.carriers[1]
16991                            .targets
16992                            .as_mut()
16993                            .unwrap()
16994                            .push("third".to_string())
16995                    },
16996                    closed: [None; 4],
16997                },
16998                Case {
16999                    name: "delegates turned on",
17000                    change: |r| r.attributes.delegates = true,
17001                    closed: [None; 4],
17002                },
17003            ];
17004            for case in cases {
17005                let mut rig = rig().await;
17006                rig.sync(vec![session(1)]).await;
17007                let scope = || Some(rig_selector("s", Some(1)));
17008                let mut routes = [
17009                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
17010                    rig.bound(Some(AFT), PLEXUS, scope()).await,
17011                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
17012                    rig.bound(Some(BROCA), OTHER, scope()).await,
17013                ];
17014                let mut record = session(1);
17015                (case.change)(&mut record);
17016                rig.sync(vec![record]).await;
17017                for (index, expected) in case.closed.iter().enumerate() {
17018                    let route = &mut routes[index];
17019                    match expected {
17020                        Some(reason) => {
17021                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
17022                            assert_eq!(
17023                                route.closed_reason(),
17024                                *reason,
17025                                "{}: route {index}",
17026                                case.name
17027                            );
17028                        }
17029                        None => {
17030                            assert!(rig.live(route), "{}: route {index} closed", case.name);
17031                            assert!(
17032                                route.untouched(),
17033                                "{}: route {index} was told something",
17034                                case.name
17035                            );
17036                        }
17037                    }
17038                }
17039            }
17040        }
17041
17042        #[tokio::test]
17043        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
17044            // Removed, and replaced by a higher epoch.
17045            for next in [Vec::new(), vec![session(2)]] {
17046                let mut rig = rig().await;
17047                rig.sync(vec![session(1)]).await;
17048                let mut route = rig
17049                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17050                    .await;
17051                rig.sync(next).await;
17052                assert!(!rig.live(&route));
17053                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
17054            }
17055
17056            // A child whose parent ends: its routes close as parent-ended, the
17057            // child stays live, and routes under the parent close as ended.
17058            let mut rig = rig().await;
17059            let mut child = session(1);
17060            child.scope_ref = "child".to_string();
17061            child.kind = ScopeKind::Worker;
17062            child.parent = Some(ScopeParent::new(
17063                Principal::Reserved {
17064                    module_id: OWNER.to_string(),
17065                },
17066                "s".to_string(),
17067                1,
17068            ));
17069            rig.sync(vec![session(1), child.clone()]).await;
17070            let mut child_route = rig
17071                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
17072                .await;
17073            assert_eq!(
17074                child_route.stamp().unwrap().parent_state,
17075                Some(ParentState::Linked)
17076            );
17077            rig.sync(vec![child]).await;
17078            assert!(!rig.live(&child_route));
17079            assert_eq!(
17080                child_route.closed_reason(),
17081                RouteCloseReason::ScopeParentEnded
17082            );
17083        }
17084
17085        #[tokio::test]
17086        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
17087        ) {
17088            let mut rig = rig().await;
17089            rig.sync(vec![session(1)]).await;
17090            let mut route = rig
17091                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17092                .await;
17093            let before = rig.forwarding.published_scope_tag(OWNER, "s");
17094            rig.sync(vec![session(1)]).await;
17095            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17096            assert!(rig.live(&route) && route.untouched());
17097
17098            // A call in flight on the route when another carrier is added. A
17099            // forwarded REQUEST holds one credit on the route's flow until the
17100            // module answers; the router takes it exactly like this.
17101            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
17102                .forwarding
17103                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17104                .unwrap()
17105            else {
17106                panic!("the route is bound");
17107            };
17108            binding.flow.acquire_tagged(9, false).await.unwrap();
17109            let mut widened = session(1);
17110            widened.carriers.push(carrier(MAGIC, None));
17111            rig.sync(vec![widened]).await;
17112            assert!(rig.live(&route) && route.untouched());
17113            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17114            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
17115            // The call's credit is still held on an open flow, so its answer
17116            // will be delivered: closing the route would have closed the flow.
17117            assert_eq!(binding.flow.in_flight(), 1);
17118            binding
17119                .flow
17120                .acquire_tagged(10, false)
17121                .await
17122                .expect("the flow is still open");
17123        }
17124
17125        /// A swap's superseded endpoint keeps its routes until drained; ending
17126        /// the scope closes them there too.
17127        #[tokio::test]
17128        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
17129            let mut rig = rig().await;
17130            rig.sync(vec![session(1)]).await;
17131            let mut on_incumbent = rig
17132                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17133                .await;
17134
17135            // Swap plexus: register a candidate and cut over, leaving the
17136            // incumbent superseded with the route still on it.
17137            let (candidate, _candidate_rx) = wide_ctx(9);
17138            let registration = rig
17139                .handler
17140                .registry
17141                .register_candidate_with_control_ops(
17142                    manifest(PLEXUS, PROTOCOL_VERSION),
17143                    PROTOCOL_VERSION,
17144                    candidate.connection_id,
17145                    module_baseline_control_ops(),
17146                )
17147                .unwrap();
17148            rig.forwarding
17149                .register_candidate_module_connection(
17150                    candidate.connection_id,
17151                    PLEXUS.to_string(),
17152                    PROTOCOL_VERSION,
17153                    manifest_concurrency(&registration.manifest),
17154                    candidate.egress.clone(),
17155                )
17156                .unwrap();
17157            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
17158            rig.handler
17159                .registry
17160                .promote_candidate(PLEXUS)
17161                .unwrap()
17162                .unwrap();
17163            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
17164
17165            rig.sync(Vec::new()).await;
17166            assert!(!rig.live(&on_incumbent));
17167            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
17168            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17169            let goodbye = incumbent_rx
17170                .try_recv()
17171                .expect("the superseded endpoint is told")
17172                .frame;
17173            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17174        }
17175    }
17176}
17177
17178#[cfg(test)]
17179mod concurrency_default_exposure_tests {
17180    use super::*;
17181
17182    fn hello_body(role_json: &str) -> Vec<u8> {
17183        format!(
17184            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
17185        )
17186        .into_bytes()
17187    }
17188
17189    fn manifest_from(body: &[u8]) -> ModuleManifest {
17190        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
17191        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
17192            .expect("manifest parses")
17193    }
17194
17195    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
17196
17197    #[test]
17198    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
17199        let body = hello_body(&format!(
17200            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
17201        ));
17202        let manifest = manifest_from(&body);
17203        // Precondition: serde really resolved it to the default, so the typed
17204        // manifest alone cannot answer the question this probe exists for.
17205        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17206        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
17207    }
17208
17209    #[test]
17210    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
17211        let body = hello_body(&format!(
17212            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
17213        ));
17214        let manifest = manifest_from(&body);
17215        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
17216        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17217    }
17218
17219    #[test]
17220    fn non_management_roles_are_never_reported() {
17221        let body = hello_body(
17222            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
17223        );
17224        let manifest = manifest_from(&body);
17225        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
17226    }
17227}