Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_event_declarations,
25        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27    },
28    scope::{
29        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31        SCOPE_SYNC_OP,
32    },
33    session::{
34        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37        ROLE_VERSIONS_FIELD,
38    },
39    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40    Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46    caller_identity::CallerIdentity,
47    capability_requirements::{
48        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50        RuntimeModule,
51    },
52    child_roster::DaemonShutdownFlag,
53    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
54    forwarding::{
55        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
56        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
57        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
58    },
59    observability::{
60        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
61    },
62    privileged_prompt::{prompt_text, PrivilegedOperation},
63    provenance::{
64        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
65    },
66    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
67    router::{RouteCtx, RouterError},
68    scopes::{HelloLaunchNonces, ScopeTable},
69    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
70    stderr_tail::{CaptureState, TailEntry},
71    supervise::{
72        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
73        SupervisorHandle, SwapHelloAdmission,
74    },
75    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
76};
77
78/// Lowest envelope version this subc build will negotiate.
79///
80/// Module HELLO negotiation is exact: peers must use the daemon's locked
81/// protocol version. Older and newer peers receive `version_unsupported` and
82/// are not registered.
83pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
84
85const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
86const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
87const CAP_PING_PONG: &str = "ping_pong_v1";
88const CAP_SESSION_ATTACH: &str = "session_attach_v1";
89const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
90
91const SUBC_CONTROL_OPS: &[&str] = &[
92    ops::ACTIVITY_RECORD,
93    ops::ACTIVITY_QUERY,
94    ops::ACTIVITY_SUBSCRIBE,
95    ops::SERVER_DESCRIBE,
96    ops::CATALOG_LIST,
97    ops::ROUTE_OPEN,
98    ops::ROUTE_POLL,
99    ops::ROUTE_CLOSING,
100    ops::ROUTE_CLOSED,
101    ops::SUPERVISOR_LIST,
102    ops::SUPERVISOR_RESTART,
103    ops::SUPERVISOR_SWAP,
104    ops::SUPERVISOR_RELOAD,
105    ops::SUPERVISOR_RESCAN,
106    ops::SUPERVISOR_RELEASE_RESERVED,
107    ops::SUPERVISOR_SET_ENABLED,
108    ops::SUPERVISOR_HEALTH_PROBE,
109    ops::SUPERVISOR_HEALTH,
110    ops::SUPERVISOR_STDERR_TAIL,
111    ops::SUPERVISOR_TERMINALS,
112    ops::SUPERVISOR_ROUTES,
113    ops::SUPERVISOR_PROVENANCE,
114    ops::SUPERVISOR_SPAWN_SNAPSHOT,
115    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
116];
117
118const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
119    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
120    "supervisor.live_roots",
121    SCOPE_SYNC_OP,
122    SCOPE_APPLY_OP,
123    SCOPE_DESCRIBE_OP,
124    "operator.confirm",
125];
126
127/// Module-originated ops the daemon answers but does not advertise in
128/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
129/// belongs to is incomplete, so no module is told it works before it does.
130const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
131
132const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
133
134/// How long subc waits for a module to ack a relayed route.bind before returning
135/// `module_timeout`. The ack waits on the module's own configure, which for AFT
136/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
137/// and DB-open work — on a cold page cache or a large repo that legitimately
138/// exceeds a couple of seconds. The default is generous because rejecting a VALID
139/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
140/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
141pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
142
143/// How many CONSECUTIVE full-budget relay timeouts against one target module
144/// open that module's bind-relay breaker.
145///
146/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
147/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
148/// so three consecutive full-budget timeouts take ~36s to observe: every client
149/// whose open contributed to opening the breaker had already given up on its
150/// own. That is what makes opening the breaker unable to turn a call that would
151/// have succeeded into a refusal — it can only make an already-failing module
152/// fail faster.
153///
154/// Two would be reachable inside one default deadline. One would convict a
155/// module on a single cold-cache bind, which is exactly the valid-but-slow case
156/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
157pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
158
159/// How long a module's bind-relay breaker stays open before exactly one
160/// `route.open` is let through as a probe.
161///
162/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
163/// re-pays a full-budget stall almost continuously, and the breaker stops being
164/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
165/// deadline: a client that starts retrying after the module recovers has to get
166/// a probe opportunity inside its own deadline, or the breaker converts a
167/// recovered module into a failed call — the failure it exists to prevent,
168/// pointed the other way.
169///
170/// 20s sits between those with room on both sides, and it caps what a wedged
171/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
172/// rather than one per `route.open` per connection. The stall that motivated
173/// this, with its measurements, is written up in
174/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
175/// waited the whole budget out.
176pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
177
178const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
179const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
180
181fn reload_verdict(
182    configured: &Path,
183    spawned_from: Option<&Path>,
184    image: subc_control::RunningImageAgreement,
185) -> PendingReloadVerdict {
186    let path = match spawned_from {
187        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
188            ReloadPathAgreement::Match
189        }
190        Some(spawned_from) => ReloadPathAgreement::Mismatch {
191            configured: configured.to_path_buf(),
192            spawned_from: spawned_from.to_path_buf(),
193        },
194        None => ReloadPathAgreement::Unavailable {
195            reason: if matches!(
196                image,
197                subc_control::RunningImageAgreement::Unavailable {
198                    reason: subc_control::RunningImageUnavailableReason::NotRunning
199                }
200            ) {
201                ReloadPathUnavailableReason::NotRunning
202            } else {
203                ReloadPathUnavailableReason::SpawnedPathUnavailable
204            },
205        },
206    };
207    PendingReloadVerdict {
208        path,
209        image,
210        spec: None,
211    }
212}
213
214fn launch_spec_agreement(
215    configured: &crate::supervise::ModuleSpec,
216    running: bool,
217    spawned: Option<&crate::supervise::LaunchSpecDigest>,
218) -> subc_control::LaunchSpecAgreement {
219    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
220    if !running {
221        return Agreement::Unavailable {
222            reason: Reason::NotRunning,
223        };
224    }
225    match spawned {
226        None => Agreement::Unavailable {
227            reason: Reason::SpawnedSpecUnavailable,
228        },
229        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
230            Agreement::Match
231        }
232        Some(_) => Agreement::Mismatch,
233    }
234}
235
236fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
237    if configured == spawned_from {
238        return true;
239    }
240    #[cfg(windows)]
241    {
242        // Windows spawn records the resolved verbatim path (\\?\...), while
243        // configuration keeps its original spelling. Resolve both with the OS
244        // instead of stripping prefixes or folding case, which could conflate
245        // distinct names in case-sensitive directories. Different resolved
246        // paths still differ even if their files have identical content.
247        match (
248            std::fs::canonicalize(configured),
249            std::fs::canonicalize(spawned_from),
250        ) {
251            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
252            _ => false,
253        }
254    }
255    #[cfg(not(windows))]
256    {
257        false
258    }
259}
260
261#[derive(Clone)]
262struct DaemonProvenanceFacts {
263    build: DaemonBuildProvenance,
264    pid: Option<u32>,
265    started_at_ms: Option<u64>,
266    start_clock: Option<crate::clock::StartClock>,
267    executable_path: Option<PathBuf>,
268    executable_identity: Option<SpawnedFileIdentity>,
269    process_start_time: Option<u64>,
270    probe: ExecutableIdentityProbe,
271}
272
273impl Default for DaemonProvenanceFacts {
274    fn default() -> Self {
275        Self {
276            build: DaemonBuildProvenance {
277                build_git_sha: None,
278                build_lock_digest: None,
279            },
280            pid: None,
281            started_at_ms: None,
282            start_clock: None,
283            executable_path: None,
284            executable_identity: None,
285            process_start_time: None,
286            probe: ExecutableIdentityProbe::default(),
287        }
288    }
289}
290
291#[derive(Debug, Clone)]
292struct SupervisorRescanContext {
293    supervisor: Supervisor,
294    config_path: PathBuf,
295    configured_port: Option<u16>,
296    storage_config: Option<crate::daemon_config::StorageConfig>,
297    admission_facts_carrier_module_id: Option<String>,
298    admission_facts_targets: Option<Vec<String>>,
299    scope_authority_owners: Vec<String>,
300    privileged_operator_prompt: PrivilegedOperatorPrompt,
301}
302
303/// Refusal labels passed to `observe_route_open_refusal` that mean the target
304/// module is not serving right now, and so open or extend an outage in the
305/// route outage tracker. Every one of them is only reachable after the target
306/// was found in the registry, which is what keeps an arbitrary client-chosen
307/// id from ever creating tracker state.
308///
309/// Deliberately absent: `not_registered` and `removed` (the id may be
310/// anything a client sent, and a removed module is gone on purpose),
311/// `protocol_none` (such a module never serves routes, so nothing is out),
312/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
313/// capability and admission-facts refusals (they refuse the caller, not a
314/// module outage), and `relay_reservation_failed` (its code ranges over
315/// capacity limits as well as a vanished connection). Capacity, breaker,
316/// relay-timeout and module-rejection refusals do not pass through that
317/// function at all; the breaker logs its own transitions.
318///
319/// The two not-serving refusals that bypass that function record themselves
320/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
321/// `required_capability_unprovided` is not tracked: the module itself is up,
322/// and the outage belongs to the missing provider.
323const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
324    "reloading",
325    "supervisor_not_live",
326    "registration_not_active",
327    "no_forwarding_connection",
328    "relay_send_failed",
329];
330
331/// Real channel-0 control handler for subc itself.
332#[derive(Clone)]
333pub struct ControlHandler {
334    registry: Arc<Registry>,
335    activity: Option<crate::activity::ActivityLedger>,
336    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
337    forwarding: Arc<ForwardingTable>,
338    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
339    supervisor: SupervisorHandle,
340    subc_capabilities: Arc<[String]>,
341    /// Daemon-wide route.bind relay budget. Used as the fallback when the
342    /// target module has no per-module override in
343    /// `route_bind_relay_timeouts`.
344    route_bind_relay_timeout: Duration,
345    /// Per-module route.bind relay budget overrides, keyed by module id. When
346    /// `handle_route_open` resolves the deadline for a target module, a
347    /// per-module entry wins over the daemon-wide value above.
348    route_bind_relay_timeouts: BTreeMap<String, Duration>,
349    /// Per-target-module bind-relay breaker state. Shared with the forwarding
350    /// table, which is where a new module connection resets it.
351    route_bind_breakers: RouteBindBreakers,
352    /// Live relay admissions keyed by target module. Shared through the
353    /// forwarding table so cloned or separately built handlers enforce one cap.
354    route_bind_concurrency: RouteBindConcurrency,
355    /// Start and end of each module's not-serving period as seen by
356    /// `route.open`, so an outage gets one line at each edge instead of only
357    /// the per-refusal INFO lines. Taken from the forwarding table, so every
358    /// handler built over one table shares it.
359    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
360    /// Consecutive relay timeouts that open a module's breaker.
361    route_bind_breaker_threshold: u32,
362    /// How long a breaker stays open before one probe is admitted.
363    route_bind_breaker_cooldown: Duration,
364    health_probe_timeout: Duration,
365    /// Central storage policy. When set, each registering module receives its
366    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
367    storage_config: Option<crate::daemon_config::StorageConfig>,
368    /// The machine id established at boot, served on every HELLO_ACK and on
369    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
370    /// changes the file, never this value. `None` serves no id.
371    machine_id: Option<crate::machine_id::MachineId>,
372    admission_facts_carrier_module_id: Option<String>,
373    admission_facts_targets: Option<Vec<String>>,
374    /// Scope records with their sync authorities and tombstones; see
375    /// `crate::scopes`. Shared by clones of this handler, so every connection
376    /// reads and writes one table.
377    scopes: Arc<RwLock<ScopeTable>>,
378    /// Wakes the expiry loop when an accepted scope change may have added a
379    /// deadline, so the loop can wait without ticking while no live scope has
380    /// one. A permit is stored when nothing waits, so no wake is lost.
381    scope_deadline_added: Arc<tokio::sync::Notify>,
382    /// The configured `scope_authority_owners`, kept so a rescan can report a
383    /// changed value as needing a daemon restart; rescan never applies it.
384    scope_authority_owners: Vec<String>,
385    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
386    privileged_operator_prompt: PrivilegedOperatorPrompt,
387    /// The local and peer address of each accepted TCP client connection, read
388    /// from the daemon's own socket, so a privileged request can name its caller.
389    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
390    #[cfg(test)]
391    caller_lookup: Option<TestCallerLookup>,
392    /// The launch nonce each module connection presented at HELLO, which is how
393    /// a `scope.sync` is matched to the owner's current launch.
394    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
395    rescan: Option<SupervisorRescanContext>,
396    connected_clients: ConnectedClients,
397    counters: DaemonCounters,
398    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
399    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
400    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
401    /// their usual severity (see [`log_requirement_events`]). A handler built
402    /// without one has a flag that is never set.
403    daemon_shutdown: DaemonShutdownFlag,
404    daemon_provenance: DaemonProvenanceFacts,
405    #[cfg(test)]
406    control_dispatch_delay: Option<Duration>,
407    #[cfg(test)]
408    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
409    /// Holds `route.open` after the route.bind is queued to the module and
410    /// before it is marked enqueued, so a test can make the module's reply (or
411    /// a client teardown) land exactly inside that window.
412    #[cfg(test)]
413    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
414    /// Holds a HELLO after its registry entry is reserved and before its
415    /// forwarding endpoint is installed, so a test can read the catalog and
416    /// open routes while the module is registered but not yet routable. A
417    /// blocking barrier because the HELLO handler is synchronous; the test
418    /// runs the HELLO on its own thread.
419    #[cfg(test)]
420    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
421}
422
423impl fmt::Debug for ControlHandler {
424    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
425        f.debug_struct("ControlHandler")
426            .field("registry", &self.registry)
427            .field("forwarding", &self.forwarding)
428            .field("process_liveness", &self.process_liveness.is_some())
429            .field("supervisor", &self.supervisor)
430            .field("subc_capabilities", &self.subc_capabilities)
431            .finish()
432    }
433}
434
435struct RouteOpenRequest {
436    target: RouteTarget,
437    identity: BindIdentity,
438    consumer_identity: Option<ConsumerIdentity>,
439    consumer_capabilities: Option<Vec<String>>,
440    role_versions: Option<BTreeMap<String, String>>,
441    admission_facts: Option<serde_json::Value>,
442    scope: Option<ScopeSelector>,
443}
444
445#[derive(Clone, Copy)]
446enum RouteOpenPhase {
447    Admission,
448    ProjectRoot,
449    EgressReservation,
450    BindSend,
451    BindAckWait,
452}
453
454/// Store elapsed time for each route-open phase, including attempts that end
455/// in an early refusal or backend error. Admission also includes rechecks and
456/// commit bookkeeping performed between the other timed phases.
457struct RouteOpenTimings {
458    module_id: String,
459    principal: String,
460    admission: Duration,
461    project_root: Duration,
462    egress_reservation: Duration,
463    bind_send: Duration,
464    bind_ack_wait: Duration,
465    phase: Option<RouteOpenPhase>,
466    phase_started_at: Instant,
467}
468
469impl RouteOpenTimings {
470    fn new() -> Self {
471        Self {
472            module_id: String::new(),
473            // Refusals before attestation must not claim a verified principal.
474            principal: "unresolved".to_string(),
475            admission: Duration::ZERO,
476            project_root: Duration::ZERO,
477            egress_reservation: Duration::ZERO,
478            bind_send: Duration::ZERO,
479            bind_ack_wait: Duration::ZERO,
480            phase: Some(RouteOpenPhase::Admission),
481            phase_started_at: Instant::now(),
482        }
483    }
484
485    fn finish(&mut self) {
486        let elapsed = self.phase_started_at.elapsed();
487        match self.phase.take() {
488            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
489            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
490            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
491            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
492            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
493            None => {}
494        }
495    }
496
497    fn enter(&mut self, phase: RouteOpenPhase) {
498        self.finish();
499        self.phase = Some(phase);
500        self.phase_started_at = Instant::now();
501    }
502}
503
504struct RouteBindReservationGuard {
505    forwarding: Arc<ForwardingTable>,
506    endpoint: ModuleEndpointId,
507    relay_corr: u64,
508    armed: bool,
509}
510
511struct ModuleControlRpcGuard {
512    forwarding: Arc<ForwardingTable>,
513    endpoint: ModuleEndpointId,
514    corr: u64,
515    armed: bool,
516}
517
518impl ModuleControlRpcGuard {
519    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
520        Self {
521            forwarding,
522            endpoint,
523            corr,
524            armed: true,
525        }
526    }
527
528    fn disarm(&mut self) {
529        self.armed = false;
530    }
531}
532
533impl Drop for ModuleControlRpcGuard {
534    fn drop(&mut self) {
535        if self.armed {
536            let _ = self
537                .forwarding
538                .cancel_module_control_rpc(self.endpoint, self.corr);
539        }
540    }
541}
542
543impl RouteBindReservationGuard {
544    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
545        Self {
546            forwarding,
547            endpoint,
548            relay_corr,
549            armed: true,
550        }
551    }
552
553    fn release_and_disarm(&mut self) {
554        if !self.armed {
555            return;
556        }
557        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
558            self.endpoint,
559            self.relay_corr,
560            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
561        ) {
562            send_goodbye_target_best_effort(
563                &self.forwarding.counters(),
564                &target,
565                "canceled route.bind",
566            );
567        }
568        self.armed = false;
569    }
570
571    fn disarm(&mut self) {
572        self.armed = false;
573    }
574}
575
576impl Drop for RouteBindReservationGuard {
577    fn drop(&mut self) {
578        self.release_and_disarm();
579    }
580}
581
582/// Per-target-module circuit breaker around the `route.bind` relay.
583///
584/// The connection reader is serial per connection, so a module whose `on_bind`
585/// sits on the ack blocks every LATER frame on the connections that call it,
586/// including calls to unrelated modules. This does not make any module's bind
587/// fast; it stops the daemon paying the full budget again and again for a
588/// condition it has already observed.
589///
590/// State is keyed by TARGET MODULE and shared by every connection: a wedged
591/// module wedges everyone, so what one connection learned should protect the
592/// rest.
593///
594/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
595/// relay to that module has actually timed out, and is removed again when a
596/// relay is accepted or the module reconnects, so it cannot grow with traffic
597/// or with modules that behave.
598///
599/// # Why a `std` mutex here is not the head-of-line defect again
600///
601/// Acquisition never awaits. The critical section is a hash lookup plus a few
602/// integer updates, with no I/O and no `.await` inside it, so a reader task
603/// cannot be descheduled behind it the way it can behind
604/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
605/// primitive, held for the same kind of work, as the refusal counter this very
606/// path already increments.
607///
608/// It is also NOT on the data-plane splice path: only `route.open` and module
609/// registration touch it, so bound-route frames gain no state check and no
610/// contention.
611#[derive(Debug, Clone, Default)]
612pub(crate) struct RouteBindBreakers {
613    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
614}
615
616#[derive(Debug, Clone, Default)]
617pub(crate) struct RouteBindConcurrency {
618    modules: Arc<Mutex<HashMap<String, usize>>>,
619}
620
621struct RouteBindConcurrencyGuard {
622    concurrency: RouteBindConcurrency,
623    module_id: String,
624}
625
626impl RouteBindConcurrency {
627    /// Admit without waiting. Waiting here would move the bind stall from the
628    /// module reply to a semaphore and restore reader head-of-line blocking.
629    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
630        let mut modules = self
631            .modules
632            .lock()
633            .expect("route.bind concurrency mutex poisoned");
634        let in_flight = modules.entry(module_id.to_string()).or_default();
635        if *in_flight >= limit {
636            return Err(*in_flight);
637        }
638        *in_flight += 1;
639        Ok(RouteBindConcurrencyGuard {
640            concurrency: self.clone(),
641            module_id: module_id.to_string(),
642        })
643    }
644}
645
646impl Drop for RouteBindConcurrencyGuard {
647    fn drop(&mut self) {
648        let mut modules = self
649            .concurrency
650            .modules
651            .lock()
652            .expect("route.bind concurrency mutex poisoned");
653        let remove = {
654            let in_flight = modules
655                .get_mut(&self.module_id)
656                .expect("admitted route.bind has a concurrency entry");
657            *in_flight -= 1;
658            *in_flight == 0
659        };
660        if remove {
661            modules.remove(&self.module_id);
662        }
663    }
664}
665
666#[derive(Debug, Default)]
667struct ModuleBreakerState {
668    /// Relay timeouts observed with no accepted relay in between.
669    consecutive_timeouts: u32,
670    /// `Some` while the breaker is open: the instant the cooldown expires and
671    /// the next arrival may probe. `None` means closed.
672    cooldown_until: Option<Instant>,
673    /// A half-open probe has been admitted and has not settled yet. This is
674    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
675    /// that read the cooldown, so concurrent opens arriving at the moment the
676    /// cooldown expires cannot all decide that they are the probe.
677    probe_in_flight: Option<Arc<()>>,
678}
679
680/// What the breaker decided for one `route.open`, before any relay work.
681enum RouteBindAdmission<'a> {
682    Admitted {
683        guard: RouteBindBreakerGuard<'a>,
684        /// This open is the single half-open probe, so the transition is worth
685        /// one log line.
686        probe: bool,
687    },
688    Refused {
689        consecutive_timeouts: u32,
690        /// What is left of the cooldown. Zero when the refusal is because the
691        /// one probe is already in flight rather than because the cooldown has
692        /// not elapsed.
693        retry_in: Duration,
694        probe_in_flight: bool,
695    },
696}
697
698/// An outstanding admission, which must be told how its relay settled.
699///
700/// `Drop` settles it as inconclusive, so an early return between admission and
701/// the relay -- or the whole handler being cancelled when the client
702/// disconnects -- releases a half-open probe slot instead of leaving the
703/// breaker wedged half-open with no further probes.
704struct RouteBindBreakerGuard<'a> {
705    breakers: RouteBindBreakers,
706    module_id: &'a str,
707    probe_token: Option<Arc<()>>,
708    settled: bool,
709}
710
711impl RouteBindBreakerGuard<'_> {
712    /// The module answered within the budget and took the bind. THE ONLY
713    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
714    /// breaker, which is a transition worth logging.
715    fn record_accepted(&mut self) -> bool {
716        self.settled = true;
717        self.breakers.record_accepted(self.module_id)
718    }
719
720    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
721    /// COUNTS TOWARD OPENING.
722    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
723        self.settled = true;
724        self.breakers.record_timeout(
725            self.module_id,
726            self.probe_token.as_ref(),
727            threshold,
728            cooldown,
729        )
730    }
731
732    /// Everything else: the module REJECTED the bind, its connection went away
733    /// mid-relay, or the waiter was cancelled.
734    ///
735    /// None of these is evidence that a module is slow, and each already has
736    /// its own refusal with its own code. A module that rejects a bind in
737    /// microseconds is healthy and must never be convicted for it; a module
738    /// that died has said nothing about the module that replaces it. So these
739    /// neither increment nor reset the count -- they only release a probe slot.
740    fn record_inconclusive(&mut self) {
741        self.settled = true;
742        self.breakers
743            .record_inconclusive(self.module_id, self.probe_token.as_ref());
744    }
745}
746
747impl Drop for RouteBindBreakerGuard<'_> {
748    fn drop(&mut self) {
749        if !self.settled {
750            self.breakers
751                .record_inconclusive(self.module_id, self.probe_token.as_ref());
752        }
753    }
754}
755
756/// The breaker moved to open, reported so the caller can log it outside the
757/// lock. Opening is rare and load-bearing; the refusals that follow are
758/// frequent and are counted rather than logged.
759struct BreakerOpened {
760    consecutive_timeouts: u32,
761    /// True when a failed probe re-opened an already-open breaker, which reads
762    /// very differently in a log from a first opening.
763    reopened_after_probe: bool,
764}
765
766impl RouteBindBreakers {
767    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
768        self.modules
769            .lock()
770            .expect("route.bind breaker mutex poisoned")
771    }
772
773    /// Decide whether this `route.open` may attempt its relay. Takes the map
774    /// lock and nothing else, and never awaits.
775    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
776        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
777            probe: probe_token.is_some(),
778            guard: RouteBindBreakerGuard {
779                breakers: self.clone(),
780                module_id,
781                probe_token,
782                settled: false,
783            },
784        };
785
786        let mut modules = self.lock();
787        let Some(state) = modules.get_mut(module_id) else {
788            return admitted(None);
789        };
790        let Some(cooldown_until) = state.cooldown_until else {
791            return admitted(None);
792        };
793        if state.probe_in_flight.is_some() {
794            return RouteBindAdmission::Refused {
795                consecutive_timeouts: state.consecutive_timeouts,
796                retry_in: Duration::ZERO,
797                probe_in_flight: true,
798            };
799        }
800        let now = Instant::now();
801        if now < cooldown_until {
802            return RouteBindAdmission::Refused {
803                consecutive_timeouts: state.consecutive_timeouts,
804                retry_in: cooldown_until - now,
805                probe_in_flight: false,
806            };
807        }
808        let token = Arc::new(());
809        state.probe_in_flight = Some(Arc::clone(&token));
810        admitted(Some(token))
811    }
812
813    fn record_accepted(&self, module_id: &str) -> bool {
814        self.lock()
815            .remove(module_id)
816            .is_some_and(|state| state.cooldown_until.is_some())
817    }
818
819    fn record_timeout(
820        &self,
821        module_id: &str,
822        probe_token: Option<&Arc<()>>,
823        threshold: u32,
824        cooldown: Duration,
825    ) -> Option<BreakerOpened> {
826        let mut modules = self.lock();
827        let state = modules.entry(module_id.to_string()).or_default();
828        let was_open = state.cooldown_until.is_some();
829        let was_probe = Self::owns_probe(state, probe_token);
830        if was_probe {
831            state.probe_in_flight = None;
832        }
833        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
834        if state.consecutive_timeouts < threshold {
835            return None;
836        }
837        state.cooldown_until = Some(Instant::now() + cooldown);
838        Some(BreakerOpened {
839            consecutive_timeouts: state.consecutive_timeouts,
840            reopened_after_probe: was_open && was_probe,
841        })
842    }
843
844    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
845        // A relay can finish after the breaker was reset or after it opened
846        // again and started a new probe. Only the guard whose token matches the
847        // active probe may release it, so a late relay never frees a newer probe.
848        state
849            .probe_in_flight
850            .as_ref()
851            .zip(token)
852            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
853    }
854
855    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
856        if let Some(state) = self.lock().get_mut(module_id) {
857            if Self::owns_probe(state, probe_token) {
858                state.probe_in_flight = None;
859            }
860        }
861    }
862
863    /// Discard what was learned about a module, because the process it was
864    /// learned about is gone. Returns the discarded count when it was non-zero.
865    ///
866    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
867    /// `module_id` is a configuration identity that outlives any particular
868    /// child; what the breaker observed was the process behind the module
869    /// connection of the moment. When a new connection registers under that id
870    /// the verdict's subject no longer exists, so the verdict is stale by
871    /// construction rather than merely likely to be wrong. Keeping it would
872    /// apply a dead process's record to a live one, which is the same defect
873    /// class this breaker exists to stop the daemon committing.
874    ///
875    /// A half-open probe in flight is discarded with the rest: it was a
876    /// question about the old process.
877    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
878        self.lock()
879            .remove(module_id)
880            .map(|state| state.consecutive_timeouts)
881            .filter(|discarded| *discarded > 0)
882    }
883
884    /// Open breakers, for the `server.describe` counters object. `None` when
885    /// none is open, so the key stays absent rather than present-and-empty.
886    ///
887    /// This is the operator's answer to "is this module refusing instantly or
888    /// is it fine?", which look identical from a client that retries and then
889    /// succeeds.
890    fn open_snapshot(&self) -> Option<serde_json::Value> {
891        let now = Instant::now();
892        let modules = self.lock();
893        let open = modules
894            .iter()
895            .filter_map(|(module_id, state)| {
896                let cooldown_until = state.cooldown_until?;
897                Some((
898                    module_id.clone(),
899                    serde_json::json!({
900                        "consecutive_timeouts": state.consecutive_timeouts,
901                        "cooldown_remaining_ms":
902                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
903                        "probe_in_flight": state.probe_in_flight.is_some(),
904                    }),
905                ))
906            })
907            .collect::<serde_json::Map<String, serde_json::Value>>();
908        (!open.is_empty()).then_some(serde_json::Value::Object(open))
909    }
910}
911
912impl ControlHandler {
913    pub fn new(registry: Arc<Registry>) -> Self {
914        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
915    }
916
917    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
918        let counters = forwarding.counters();
919        // Taken from the forwarding table rather than created here, so that the
920        // breaker a `route.open` consults is the same one a module's
921        // registration resets, however many handlers are built over one table.
922        let route_bind_breakers = forwarding.route_bind_breakers();
923        let route_bind_concurrency = forwarding.route_bind_concurrency();
924        let route_outages = forwarding.route_outages();
925        Self {
926            registry,
927            activity: None,
928            activity_streams: Arc::new(Mutex::new(HashMap::new())),
929            forwarding,
930            process_liveness: None,
931            supervisor: SupervisorHandle::new(),
932            subc_capabilities: Arc::from([
933                CAP_MANIFEST_REGISTRATION.to_string(),
934                CAP_CHANNEL_LIFECYCLE.to_string(),
935                CAP_PING_PONG.to_string(),
936                CAP_SESSION_ATTACH.to_string(),
937                CAP_ADMISSION_FACTS_RELAY.to_string(),
938                CAP_SCOPES_V1.to_string(),
939                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
940                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
941                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
942                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
943                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
944                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
945                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
946                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
947            ]),
948            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
949            route_bind_relay_timeouts: BTreeMap::new(),
950            route_bind_breakers,
951            route_bind_concurrency,
952            route_outages,
953            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
954            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
955            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
956            storage_config: None,
957            machine_id: None,
958            admission_facts_carrier_module_id: None,
959            admission_facts_targets: None,
960            scopes: Arc::new(RwLock::new(ScopeTable::new(
961                crate::daemon_config::default_scope_authority_owners(),
962            ))),
963            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
964            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
965            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
966            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
967            #[cfg(test)]
968            caller_lookup: None,
969            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
970            rescan: None,
971            connected_clients: ConnectedClients::new(),
972            counters,
973            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
974            daemon_shutdown: DaemonShutdownFlag::default(),
975            daemon_provenance: DaemonProvenanceFacts::default(),
976            #[cfg(test)]
977            control_dispatch_delay: None,
978            #[cfg(test)]
979            provenance_probe_override: None,
980            #[cfg(test)]
981            route_bind_relay_sent_hold: None,
982            #[cfg(test)]
983            hello_registry_to_forwarding_hold: None,
984        }
985    }
986
987    /// Set the central storage policy: registering modules then receive their
988    /// resolved storage descriptor in HELLO_ACK.
989    pub fn with_storage_config(
990        mut self,
991        storage_config: Option<crate::daemon_config::StorageConfig>,
992    ) -> Self {
993        self.storage_config = storage_config;
994        self
995    }
996
997    /// Open the ledger on its own writer thread; an unavailable sink does not
998    /// prevent the daemon from serving routing or supervisor requests.
999    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1000        self.activity = Some(crate::activity::ActivityLedger::open(
1001            run_dir.join("activity.jsonl"),
1002            incarnation,
1003        ));
1004        self.supervisor.set_activity(self.activity.clone());
1005        self
1006    }
1007
1008    /// Set the machine id served to every registering module (HELLO_ACK) and on
1009    /// `server.describe`.
1010    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1011        self.machine_id = machine_id;
1012        self
1013    }
1014
1015    /// Configure the exact reserved module and target ids permitted to relay
1016    /// opaque admission facts. Config-file loading validates this authority;
1017    /// this builder keeps the same policy available to embedded test daemons.
1018    pub fn with_admission_facts_config(
1019        mut self,
1020        carrier_module_id: Option<String>,
1021        targets: Option<Vec<String>>,
1022    ) -> Self {
1023        self.admission_facts_carrier_module_id = carrier_module_id;
1024        self.admission_facts_targets = targets;
1025        self
1026    }
1027
1028    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1029    /// Replaces the scope table with an empty one under the new list, so call it
1030    /// while building the handler, before any module can sync.
1031    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1032        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1033        self.scope_authority_owners = owners;
1034        self
1035    }
1036
1037    /// Set the startup policy before installing the rescan context or serving requests.
1038    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1039        self.privileged_operator_prompt = mode;
1040        self
1041    }
1042
1043    /// Override the route.bind relay timeout. Used by tests that assert the
1044    /// timeout path so they don't block on the production-safe default.
1045    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1046        self.route_bind_relay_timeout = timeout;
1047        self
1048    }
1049
1050    /// Install per-module route.bind relay budget overrides. A module id
1051    /// listed here wins over the daemon-wide default set via
1052    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1053    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1054    /// the same `Duration` the bind path will use.
1055    pub fn with_route_bind_relay_timeouts(
1056        mut self,
1057        timeouts: impl IntoIterator<Item = (String, Duration)>,
1058    ) -> Self {
1059        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1060        self
1061    }
1062
1063    /// Resolve the route.bind relay budget for a specific target module id.
1064    /// Per-module overrides win; the daemon-wide value (set via
1065    /// `with_route_bind_relay_timeout` or the built-in default) is the
1066    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1067    /// the same resolution `handle_route_open` will use.
1068    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1069        self.route_bind_relay_timeouts
1070            .get(module_id)
1071            .copied()
1072            .unwrap_or(self.route_bind_relay_timeout)
1073    }
1074
1075    /// Override the per-module bind-relay breaker policy.
1076    ///
1077    /// Used by tests, which cannot spend three production budgets opening a
1078    /// breaker or twenty seconds waiting for its cooldown. The production
1079    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1080    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1081    /// reasoning for the numbers.
1082    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1083        self.route_bind_breaker_threshold = threshold.max(1);
1084        self.route_bind_breaker_cooldown = cooldown;
1085        self
1086    }
1087
1088    #[cfg(test)]
1089    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1090        self.health_probe_timeout = timeout;
1091        self
1092    }
1093
1094    #[cfg(test)]
1095    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1096        self.control_dispatch_delay = Some(delay);
1097        self
1098    }
1099
1100    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1101    /// announce it is parked, once to be released. Build the barrier for two
1102    /// parties, the test being the other one.
1103    #[cfg(test)]
1104    pub(crate) fn with_route_bind_relay_sent_hold(
1105        mut self,
1106        hold: Arc<tokio::sync::Barrier>,
1107    ) -> Self {
1108        self.route_bind_relay_sent_hold = Some(hold);
1109        self
1110    }
1111
1112    /// A HELLO waits on `hold` twice between reserving its registry entry and
1113    /// installing its forwarding endpoint: once to announce it is parked, once
1114    /// to be released. Build the barrier for two parties, the test being the
1115    /// other one, and run the HELLO on a thread of its own.
1116    #[cfg(test)]
1117    pub(crate) fn with_hello_registry_to_forwarding_hold(
1118        mut self,
1119        hold: Arc<std::sync::Barrier>,
1120    ) -> Self {
1121        self.hello_registry_to_forwarding_hold = Some(hold);
1122        self
1123    }
1124
1125    #[cfg(test)]
1126    fn hold_between_registry_and_forwarding(&self) {
1127        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1128            hold.wait();
1129            hold.wait();
1130        }
1131    }
1132
1133    pub fn with_process_liveness(
1134        mut self,
1135        process_liveness: Arc<dyn ModuleProcessLiveness>,
1136    ) -> Self {
1137        self.process_liveness = Some(process_liveness);
1138        self
1139    }
1140
1141    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1142        self.supervisor = supervisor;
1143        self.supervisor.set_activity(self.activity.clone());
1144        self
1145    }
1146
1147    pub fn with_daemon_provenance(
1148        mut self,
1149        pid: u32,
1150        started_at_ms: u64,
1151        executable_path: Option<PathBuf>,
1152        build_git_sha: Option<String>,
1153        build_lock_digest: Option<String>,
1154    ) -> Self {
1155        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1156        let process_start_time = process_start_time(pid);
1157        self.daemon_provenance = DaemonProvenanceFacts {
1158            build: DaemonBuildProvenance {
1159                build_git_sha,
1160                build_lock_digest,
1161            },
1162            pid: Some(pid),
1163            started_at_ms: Some(started_at_ms),
1164            start_clock: None,
1165            executable_path,
1166            executable_identity,
1167            process_start_time,
1168            probe: ExecutableIdentityProbe::default(),
1169        };
1170        self
1171    }
1172
1173    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1174        self.daemon_provenance.start_clock = Some(clock);
1175        self
1176    }
1177
1178    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1179    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1180        self.scopes
1181            .write()
1182            .unwrap_or_else(|poisoned| poisoned.into_inner())
1183            .set_wall_clock(Arc::new(clock));
1184        self
1185    }
1186
1187    #[cfg(test)]
1188    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1189        self.provenance_probe_override = Some(result);
1190        self
1191    }
1192
1193    /// Install the configured module set and its reserved capability bindings.
1194    /// Bindings are configuration-scoped and may point at a provider that has not
1195    /// been installed yet, so this does not require the bound module to exist.
1196    pub fn with_capability_config(
1197        self,
1198        modules: impl IntoIterator<Item = (String, bool)>,
1199        reserved_capabilities: BTreeMap<String, String>,
1200    ) -> Self {
1201        self.capability_evaluator
1202            .configure(modules, reserved_capabilities);
1203        self
1204    }
1205
1206    /// Share the supervisor's daemon-shutdown flag with this handler, so
1207    /// requirement changes after the daemon has begun its announced shutdown
1208    /// are not reported as faults.
1209    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1210        self.daemon_shutdown = flag;
1211        self
1212    }
1213
1214    pub fn with_supervisor_rescan(
1215        mut self,
1216        supervisor: Supervisor,
1217        config_path: impl Into<PathBuf>,
1218        configured_port: Option<u16>,
1219    ) -> Self {
1220        self.rescan = Some(SupervisorRescanContext {
1221            supervisor,
1222            config_path: config_path.into(),
1223            configured_port,
1224            storage_config: self.storage_config.clone(),
1225            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1226            admission_facts_targets: self.admission_facts_targets.clone(),
1227            scope_authority_owners: self.scope_authority_owners.clone(),
1228            privileged_operator_prompt: self.privileged_operator_prompt,
1229        });
1230        self
1231    }
1232
1233    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1234        self.connected_clients = connected_clients;
1235        self
1236    }
1237
1238    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1239        Arc::clone(&self.forwarding)
1240    }
1241
1242    pub(crate) fn counters(&self) -> DaemonCounters {
1243        self.counters.clone()
1244    }
1245
1246    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1247    /// requirement event without depending on an operator polling a status command.
1248    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1249        tokio::spawn(async move {
1250            loop {
1251                self.capability_evaluator
1252                    .wait_for_change_or_deadline()
1253                    .await;
1254                self.refresh_capability_requirements();
1255            }
1256        });
1257    }
1258
1259    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1260    /// deadline passed during sleep is enforced on the first tick after wake.
1261    /// While no live scope carries a deadline the loop waits for an accepted
1262    /// scope change instead of ticking, so a daemon without deadlines never
1263    /// wakes for expiry.
1264    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1265        tokio::spawn(async move {
1266            loop {
1267                let has_deadlines = self
1268                    .scopes
1269                    .read()
1270                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1271                    .has_deadlines();
1272                if !has_deadlines {
1273                    self.scope_deadline_added.notified().await;
1274                    continue;
1275                }
1276                tokio::time::sleep(Duration::from_secs(1)).await;
1277                if let Err(error) = self.sweep_expired_scopes() {
1278                    warn!(%error, "scope expiry sweep failed");
1279                }
1280            }
1281        });
1282    }
1283
1284    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1285        let mut table = self
1286            .scopes
1287            .write()
1288            .unwrap_or_else(|poisoned| poisoned.into_inner());
1289        let swept = table.sweep_expired();
1290        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1291        drop(table);
1292        self.close_scope_drained_routes(drained);
1293        Ok(())
1294    }
1295
1296    fn publish_scope_changes(
1297        &self,
1298        changes: &[crate::scopes::ScopeTagChange],
1299        expired: &[crate::scopes::ScopeExpired],
1300    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1301        if changes.is_empty() {
1302            return Ok(Vec::new());
1303        }
1304        let drained = self
1305            .forwarding
1306            .publish_scope_changes(changes)
1307            .map_err(RouterError::Forwarding)?;
1308        let mut counts = HashMap::new();
1309        for route in &drained {
1310            *counts
1311                .entry((
1312                    route.scope.owner.as_str(),
1313                    route.scope.scope_ref.as_str(),
1314                    route.scope.tag.scope_epoch,
1315                ))
1316                .or_insert(0usize) += 1;
1317        }
1318        for scope in expired {
1319            let routes_closed = counts
1320                .get(&(
1321                    scope.owner.as_str(),
1322                    scope.scope_ref.as_str(),
1323                    scope.scope_epoch,
1324                ))
1325                .copied()
1326                .unwrap_or(0);
1327            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1328                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1329                routes_closed, "scope expired");
1330        }
1331        Ok(drained)
1332    }
1333
1334    fn runtime_capability_snapshot(
1335        &self,
1336    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1337        let runtime = self
1338            .supervisor
1339            .list()
1340            .into_iter()
1341            .map(|module| {
1342                let status = module.status().map_err(|err| {
1343                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1344                })?;
1345                Ok(RuntimeModule {
1346                    module_id: status.module_id,
1347                    state: status.state,
1348                    enabled: status.enabled,
1349                })
1350            })
1351            .collect::<Result<Vec<_>, RouterError>>()?;
1352        // Include every registration, published or not. The capability
1353        // evaluator counts a module as providing its capabilities as soon as it
1354        // has registered, even before `catalog.list` shows it, so a HELLO in
1355        // progress does not change other modules' required-capability
1356        // verdicts. See `Registry::list_registered_modules`.
1357        let registrations = self.registry.list_registered_modules().map_err(|err| {
1358            RouterError::backend(
1359                0,
1360                0,
1361                format!("failed to list capability registrations: {err}"),
1362            )
1363        })?;
1364        let registrations = registrations
1365            .into_iter()
1366            .map(|registration| RegisteredModule {
1367                module_id: registration.manifest.module_id,
1368                module_version: registration.manifest.module_version,
1369                capabilities: registration.manifest.capabilities,
1370            })
1371            .collect();
1372        Ok((runtime, registrations))
1373    }
1374
1375    /// The capability side effects of a module becoming the active registration
1376    /// for its id: cache its manifest (warning if its claims drifted), run the
1377    /// deny census when its declarations call for one, and recompute the
1378    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1379    /// candidate's does not, and the supervisor does it at promotion instead,
1380    /// through [`crate::supervise::SwapPromotionObserver`].
1381    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1382        let cached_registration = RegisteredModule {
1383            module_id: registration.manifest.module_id.clone(),
1384            module_version: registration.manifest.module_version.clone(),
1385            capabilities: registration.manifest.capabilities.clone(),
1386        };
1387        if self.capability_evaluator.record_hello(&cached_registration) {
1388            warn!(
1389                module_id = %cached_registration.module_id,
1390                "capability claims drifted from the cached manifest"
1391            );
1392        }
1393        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1394            self.enforce_capability_denies();
1395        }
1396        self.refresh_capability_requirements();
1397    }
1398
1399    /// Point the shared supervisor handle at this handler for swap promotions.
1400    /// Called wherever a handler is put behind the `Arc` the router serves, so
1401    /// it can be held weakly.
1402    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1403        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1404            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1405        self.supervisor.set_swap_promotion_observer(observer);
1406    }
1407
1408    pub fn refresh_capability_requirements(&self) {
1409        match self.runtime_capability_snapshot() {
1410            Ok((runtime, registrations)) => {
1411                log_requirement_events(
1412                    self.capability_evaluator
1413                        .evaluate_now(&runtime, &registrations),
1414                    &self.daemon_shutdown,
1415                );
1416            }
1417            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1418        }
1419    }
1420
1421    /// Reconcile only live, attested route bindings after a capability deny edge
1422    /// or target claim was added. This is deliberately a control-plane census:
1423    /// the opaque forwarding hot path must not grow a per-frame capability check.
1424    fn enforce_capability_denies(&self) {
1425        let registrations = match self.registry.list_registered_modules() {
1426            Ok(snapshot) => snapshot,
1427            Err(err) => {
1428                warn!(error = %err, "failed to read registrations for capability deny census");
1429                return;
1430            }
1431        };
1432        let manifests = registrations
1433            .into_iter()
1434            .map(|registration| {
1435                (
1436                    registration.manifest.module_id.clone(),
1437                    registration.manifest,
1438                )
1439            })
1440            .collect::<BTreeMap<_, _>>();
1441        let census = match self.forwarding.route_census(None) {
1442            Ok(census) => census,
1443            Err(err) => {
1444                warn!(error = %err, "failed to read route census for capability deny enforcement");
1445                return;
1446            }
1447        };
1448
1449        for (target_module_id, routes) in census {
1450            let Some(target_manifest) = manifests.get(&target_module_id) else {
1451                continue;
1452            };
1453            let mut closed_routes = Vec::new();
1454            let mut module_goodbyes = Vec::new();
1455            for route in routes {
1456                let Principal::Reserved {
1457                    module_id: opening_module_id,
1458                } = &route.principal
1459                else {
1460                    continue;
1461                };
1462                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1463                    continue;
1464                };
1465                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1466                    continue;
1467                };
1468
1469                match self.forwarding.release_client_route(
1470                    route.goodbye_target.connection_id,
1471                    route.goodbye_target.channel,
1472                    route.goodbye_target.epoch,
1473                ) {
1474                    Ok(RouteRelease::Removed(module_goodbye)) => {
1475                        warn!(
1476                            opening_module_id,
1477                            target_module_id,
1478                            capability,
1479                            "force-closing route because an attested capability deny edge now matches"
1480                        );
1481                        closed_routes.push(route);
1482                        module_goodbyes.push(module_goodbye);
1483                    }
1484                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1485                    Err(err) => warn!(
1486                        opening_module_id,
1487                        target_module_id,
1488                        capability,
1489                        error = %err,
1490                        "failed to force-close capability-denied route"
1491                    ),
1492                }
1493            }
1494
1495            if closed_routes.is_empty() {
1496                continue;
1497            }
1498            send_route_control_pushes(
1499                &self.forwarding,
1500                closed_routes,
1501                ClientControlPush::RouteClosed {
1502                    module_id: target_module_id,
1503                    channels: Vec::new(),
1504                    reason: RouteCloseReason::CapabilityDenied,
1505                    drained: false,
1506                    abandoned: 0,
1507                    excluded_subscriptions: 0,
1508                    terminal: Some(false),
1509                },
1510            );
1511            self.emit_route_goodbyes(module_goodbyes);
1512        }
1513    }
1514
1515    /// Why a registered module is not accepting new route binds, or `None` when
1516    /// it is. This is the module's effective readiness: its declared readiness
1517    /// first, then every `need: required` capability it declares evaluating to
1518    /// `provided`. `route.open` and `catalog.list` both read it here so the
1519    /// catalog never reports a module routable that `route.open` would refuse.
1520    fn not_ready_reason(
1521        &self,
1522        registration: &crate::registry::ModuleRegistration,
1523    ) -> Option<NotReadyReason> {
1524        if !registration.ready {
1525            return Some(NotReadyReason {
1526                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1527                capability: None,
1528            });
1529        }
1530        self.first_unprovided_required_capability(registration)
1531            .map(|capability| NotReadyReason {
1532                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1533                capability: Some(capability),
1534            })
1535    }
1536
1537    /// The lexicographically first capability this registration declares
1538    /// `need: required` whose evaluator verdict is not `provided`.
1539    ///
1540    /// The verdicts are the capability evaluator's own; nothing here decides
1541    /// what "provided" means. The evaluator counts a capability provided as
1542    /// soon as a module claiming it has REGISTERED, not once that module is
1543    /// ready. That distinction is what keeps two modules that require each
1544    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1545    /// is ready", each would wait for the other to become ready first and
1546    /// neither ever would. Do not tighten it to readiness.
1547    ///
1548    /// A required capability with no verdict at all means this registration's
1549    /// HELLO or catalog.update landed after the last recompute; recompute once
1550    /// rather than let a missing verdict read as either answer. If it is still
1551    /// missing (the recompute itself failed) the capability counts as
1552    /// unprovided: the refusal is retryable, and routing a module whose
1553    /// required provider is unknown is the outcome this check exists to stop.
1554    fn first_unprovided_required_capability(
1555        &self,
1556        registration: &crate::registry::ModuleRegistration,
1557    ) -> Option<String> {
1558        let required = registration
1559            .manifest
1560            .capabilities
1561            .iter()
1562            .flat_map(|declarations| declarations.requires.iter())
1563            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1564            .map(|requirement| requirement.capability.as_str())
1565            .collect::<BTreeSet<_>>();
1566        if required.is_empty() {
1567            return None;
1568        }
1569        let module_id = registration.manifest.module_id.as_str();
1570        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1571        if required
1572            .iter()
1573            .any(|capability| verdict(capability).is_none())
1574        {
1575            self.refresh_capability_requirements();
1576        }
1577        required
1578            .into_iter()
1579            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1580            .map(str::to_string)
1581    }
1582
1583    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1584        self.capability_evaluator
1585            .statuses()
1586            .into_iter()
1587            .map(capability_requirement_status)
1588            .collect()
1589    }
1590
1591    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1592    /// registration-release watch. The signal is what the supervisor waits on
1593    /// before spawning a replacement, so it must only fire once forwarding
1594    /// teardown is also done (see [`Self::cleanup_connection`] /
1595    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1596    /// admitted but its forwarding endpoint could not be installed.
1597    fn deregister_connection(
1598        &self,
1599        connection_id: ConnectionId,
1600        reason: RegistrationEndReason,
1601    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1602        self.registry
1603            .deregister_connection_with_reason(connection_id, reason)
1604    }
1605
1606    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1607        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1608            return None;
1609        }
1610        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1611            parse_client_control_request(&frame.body)
1612        else {
1613            return None;
1614        };
1615        Some(target_module_id(&target).to_string())
1616    }
1617
1618    pub(crate) fn route_open_capacity_refusal(
1619        &self,
1620        ctx: &RouteCtx,
1621        frame: &Frame,
1622        target_module_id: &str,
1623        in_flight: usize,
1624        limit: usize,
1625    ) -> Result<Frame, RouterError> {
1626        self.route_open_admission_refusal_frame(
1627            ctx,
1628            frame,
1629            target_module_id,
1630            "open_admission_full",
1631            (in_flight, limit),
1632            format!(
1633                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1634            ),
1635        )
1636    }
1637
1638    fn route_open_target_capacity_refusal(
1639        &self,
1640        ctx: &RouteCtx,
1641        frame: &Frame,
1642        target_module_id: &str,
1643        in_flight: usize,
1644    ) -> Result<Frame, RouterError> {
1645        self.route_open_admission_refusal_frame(
1646            ctx,
1647            frame,
1648            target_module_id,
1649            "target_binds_full",
1650            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1651            format!(
1652                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1653            ),
1654        )
1655    }
1656
1657    /// Admission pressure clears as existing binds settle, so its refusal must
1658    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1659    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1660    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1661    /// cannot currently reach its target; `module_timeout` would falsely claim
1662    /// that a wait expired. A new, cleaner code would be terminal to deployed
1663    /// clients, so it requires a client-tolerance rollout before daemon emission.
1664    fn route_open_admission_refusal_frame(
1665        &self,
1666        ctx: &RouteCtx,
1667        frame: &Frame,
1668        target_module_id: &str,
1669        reason: &'static str,
1670        (in_flight, limit): (usize, usize),
1671        message: impl Into<String>,
1672    ) -> Result<Frame, RouterError> {
1673        let code = error_codes::TARGET_UNAVAILABLE;
1674        self.counters.increment_route_open_refused(code);
1675        info!(
1676            target: "control",
1677            code,
1678            reason,
1679            module_id = ?target_module_id,
1680            connection_id = ctx.connection_id.get(),
1681            in_flight,
1682            limit,
1683            "route.open refused"
1684        );
1685        control_error_frame(frame, code, message.into())
1686    }
1687
1688    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1689    ///
1690    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1691    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1692    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1693    #[cfg(test)]
1694    pub fn handle_control(
1695        &self,
1696        connection_id: ConnectionId,
1697        frame: Frame,
1698    ) -> Result<Vec<Frame>, RouterError> {
1699        match frame.header.ty {
1700            FrameType::Ping => Ok(vec![pong(&frame)?]),
1701            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1702            FrameType::Goodbye => self.handle_goodbye(connection_id),
1703            ty => Ok(vec![control_error_frame(
1704                &frame,
1705                "unsupported_control_frame",
1706                format!("unsupported channel-0 frame {ty:?}"),
1707            )?]),
1708        }
1709    }
1710
1711    pub async fn handle_control_frame(
1712        &self,
1713        ctx: &RouteCtx,
1714        frame: Frame,
1715    ) -> Result<Vec<Frame>, RouterError> {
1716        self.handle_control_frame_timed(ctx, frame, None).await
1717    }
1718
1719    pub(crate) fn record_caller_endpoints(
1720        &self,
1721        connection: ConnectionId,
1722        endpoints: Option<(SocketAddr, SocketAddr)>,
1723    ) {
1724        let mut callers = self
1725            .caller_endpoints
1726            .lock()
1727            .unwrap_or_else(|p| p.into_inner());
1728        callers.remove(&connection);
1729        if let Some(endpoints) = endpoints {
1730            callers.insert(connection, endpoints);
1731        }
1732    }
1733
1734    #[cfg(test)]
1735    pub(crate) fn with_caller_lookup_for_test(
1736        mut self,
1737        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1738    ) -> Self {
1739        self.caller_lookup = Some(Arc::new(lookup));
1740        self
1741    }
1742
1743    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1744        let endpoints = self
1745            .caller_endpoints
1746            .lock()
1747            .unwrap_or_else(|p| p.into_inner())
1748            .get(&connection)
1749            .copied();
1750        #[cfg(test)]
1751        let lookup = self.caller_lookup.clone();
1752        tokio::task::spawn_blocking(move || {
1753            #[cfg(test)]
1754            if let Some(lookup) = lookup {
1755                return lookup(endpoints);
1756            }
1757            endpoints
1758                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1759                .unwrap_or_default()
1760        })
1761        .await
1762        .unwrap_or_default()
1763    }
1764
1765    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1766        frame.header.channel == 0
1767            && frame.header.ty == FrameType::Request
1768            && self
1769                .forwarding
1770                .module_endpoint_for_connection(ctx.connection_id)
1771                .is_ok_and(|endpoint| endpoint.is_none())
1772            && parse_client_control_request(&frame.body)
1773                .ok()
1774                .is_some_and(|request| privileged_operation(&request).is_some())
1775    }
1776
1777    pub(crate) async fn handle_control_frame_timed(
1778        &self,
1779        ctx: &RouteCtx,
1780        frame: Frame,
1781        dispatch_started_at: Option<StdInstant>,
1782    ) -> Result<Vec<Frame>, RouterError> {
1783        match frame.header.ty {
1784            FrameType::Ping => Ok(vec![pong(&frame)?]),
1785            FrameType::Hello => {
1786                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1787            }
1788            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1789            FrameType::Cancel => {
1790                // A Cancel on channel 0 names a waiting operator.confirm or a
1791                // subscription. Successful cancellation has no ordinary reply.
1792                if self
1793                    .forwarding
1794                    .operator_confirms()
1795                    .cancel(ctx.connection_id, frame.header.corr)
1796                    || self
1797                        .supervisor
1798                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1799                    || self.cancel_activity_subscription(ctx, &frame)?
1800                {
1801                    Ok(Vec::new())
1802                } else {
1803                    Ok(vec![control_error_frame(
1804                        &frame,
1805                        "unknown_subscription",
1806                        "no subscription has this correlation id",
1807                    )?])
1808                }
1809            }
1810            FrameType::Request => {
1811                // This additive operation is not part of the existing exhaustive
1812                // module-control enum. Probe the op before decoding that enum.
1813                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1814                if op
1815                    .as_ref()
1816                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1817                {
1818                    return self.handle_activity_record(ctx, frame).await;
1819                }
1820                if op.as_ref().is_some_and(|probe| {
1821                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1822                }) {
1823                    return self.handle_activity_read(ctx, frame).await;
1824                }
1825                if op
1826                    .as_ref()
1827                    .is_some_and(|probe| probe.op == "operator.confirm")
1828                {
1829                    return self.handle_operator_confirm(ctx, frame);
1830                }
1831                if op
1832                    .as_ref()
1833                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1834                {
1835                    return self.handle_scope_served_by(ctx.connection_id, frame);
1836                }
1837                if self
1838                    .forwarding
1839                    .module_endpoint_for_connection(ctx.connection_id)
1840                    .map_err(RouterError::Forwarding)?
1841                    .is_some()
1842                {
1843                    if !is_known_module_request_op(&frame.body) {
1844                        return Ok(vec![control_error_frame(
1845                            &frame,
1846                            "unsupported_control_frame",
1847                            "module-originated channel-0 REQUEST is not supported",
1848                        )?]);
1849                    }
1850                    let request = match parse_module_control_request_from_module(&frame.body) {
1851                        Ok(request) => request,
1852                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1853                            return Ok(vec![control_error_frame(
1854                                &frame,
1855                                "unsupported_control_frame",
1856                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1857                            )?])
1858                        }
1859                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1860                            return Ok(vec![control_error_frame(
1861                                &frame,
1862                                "invalid_control_body",
1863                                format!("malformed module control body: {err}"),
1864                            )?])
1865                        }
1866                    };
1867                    let op = module_control_request_op(&request);
1868                    let corr = frame.header.corr;
1869                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1870                    let result =
1871                        self.handle_module_control_request(ctx.connection_id, frame, request);
1872                    log_slow_control_dispatch(
1873                        dispatch_started_at,
1874                        op,
1875                        ctx.connection_id,
1876                        corr,
1877                        None,
1878                    );
1879                    return result;
1880                }
1881
1882                if is_known_module_request_op(&frame.body) {
1883                    return Ok(vec![control_error_frame(
1884                        &frame,
1885                        "not_registered",
1886                        "catalog.update requires an active module registration owned by this connection",
1887                    )?]);
1888                }
1889
1890                let request = match parse_client_control_request(&frame.body) {
1891                    Ok(request) => request,
1892                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1893                        return Ok(vec![control_error_frame(
1894                            &frame,
1895                            "unknown_control_op",
1896                            format!("unknown client control op: {err}"),
1897                        )?])
1898                    }
1899                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1900                        return Ok(vec![control_error_frame(
1901                            &frame,
1902                            "invalid_control_body",
1903                            format!("malformed client control body: {err}"),
1904                        )?])
1905                    }
1906                };
1907                let op = client_control_request_op(&request);
1908                let corr = frame.header.corr;
1909                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1910                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1911                #[cfg(test)]
1912                if let Some(delay) = self.control_dispatch_delay {
1913                    tokio::time::sleep(delay).await;
1914                }
1915                let result = self
1916                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1917                    .await;
1918                if let Some(timings) = route_open_timings.as_mut() {
1919                    timings.finish();
1920                }
1921                log_slow_control_dispatch(
1922                    dispatch_started_at,
1923                    op,
1924                    ctx.connection_id,
1925                    corr,
1926                    route_open_timings.as_ref(),
1927                );
1928                result
1929            }
1930            FrameType::Push => {
1931                let Some(endpoint) = self
1932                    .forwarding
1933                    .module_endpoint_for_connection(ctx.connection_id)
1934                    .map_err(RouterError::Forwarding)?
1935                else {
1936                    return Ok(vec![control_error_frame(
1937                        &frame,
1938                        "unsupported_control_frame",
1939                        "client-originated channel-0 PUSH is not supported",
1940                    )?]);
1941                };
1942                self.handle_status_update(endpoint, frame)
1943            }
1944            FrameType::Response | FrameType::Error
1945                if self
1946                    .forwarding
1947                    .module_endpoint_for_connection(ctx.connection_id)
1948                    .map_err(RouterError::Forwarding)?
1949                    .is_some() =>
1950            {
1951                self.handle_module_relay_response(ctx.connection_id, frame)
1952            }
1953            ty => Ok(vec![control_error_frame(
1954                &frame,
1955                "unsupported_control_frame",
1956                format!("unsupported channel-0 frame {ty:?}"),
1957            )?]),
1958        }
1959    }
1960
1961    pub fn cleanup_connection(
1962        &self,
1963        connection_id: ConnectionId,
1964    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1965        self.caller_endpoints
1966            .lock()
1967            .unwrap_or_else(|p| p.into_inner())
1968            .remove(&connection_id);
1969        self.forwarding
1970            .operator_confirms()
1971            .connection_closed(connection_id);
1972        self.cleanup_connection_with_end_reason(
1973            connection_id,
1974            RegistrationEndReason::ConnectionClosed,
1975        )
1976    }
1977
1978    fn cleanup_connection_with_end_reason(
1979        &self,
1980        connection_id: ConnectionId,
1981        requested_reason: RegistrationEndReason,
1982    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1983        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1984            self.registry
1985                .get_module_by_connection(connection_id)?
1986                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
1987                .and_then(|module| module.registration_end_reason().ok().flatten())
1988                .unwrap_or(requested_reason)
1989        } else {
1990            requested_reason
1991        };
1992        let crash_closed = self
1993            .registry
1994            .get_module_by_connection(connection_id)?
1995            .and_then(|registration| {
1996                self.forwarding
1997                    .module_endpoint_for_connection(connection_id)
1998                    .ok()
1999                    .flatten()
2000                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2001                    .map(|routes| (registration.manifest.module_id, routes))
2002            });
2003        let crash_closed = crash_closed.map(|(module_id, routes)| {
2004            let terminal = match self.supervisor.get(&module_id) {
2005                None => false,
2006                Some(module) => match module.will_recover_after_connection_loss() {
2007                    Ok(will_recover) => !will_recover,
2008                    Err(err) => {
2009                        warn!(
2010                            %module_id,
2011                            error = %err,
2012                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2013                        );
2014                        false
2015                    }
2016                },
2017            };
2018            // The forwarding table gates all providers at the start of daemon
2019            // shutdown, before their connections are closed. An ordinary
2020            // module disconnect still reports crash if that gate is not set.
2021            let (reason, drained) = match self
2022                .forwarding
2023                .daemon_drain_result_for_connection(connection_id)
2024            {
2025                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2026                Ok(None) => (RouteCloseReason::Crash, false),
2027                Err(err) => {
2028                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2029                    (RouteCloseReason::Crash, false)
2030                }
2031            };
2032            (module_id, routes, reason, drained, terminal)
2033        });
2034        let registrations = self.deregister_connection(connection_id, end_reason);
2035        let cleanup = if crash_closed.is_some() {
2036            self.forwarding.cleanup_connection_counted(connection_id)
2037        } else {
2038            // A module connection's teardown needs the count of abandoned
2039            // route.bind relays for its route.closed notice below. Any other
2040            // connection, such as a client's, sends no such notice and needs
2041            // only its routes released, so it uses the route-only wrapper and
2042            // reports zero.
2043            self.forwarding
2044                .cleanup_connection(connection_id)
2045                .map(|released| crate::forwarding::ConnectionCleanup {
2046                    released,
2047                    abandoned_relays: 0,
2048                })
2049        };
2050        // The route.closed push waits for forwarding teardown because only
2051        // teardown knows how many pending route.bind relays it aborted. It still
2052        // goes out before the GOODBYEs for the released routes, and its targets
2053        // were captured above, before teardown removed those routes.
2054        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2055            let abandoned = cleanup
2056                .as_ref()
2057                .map_or(0, |cleanup| cleanup.abandoned_relays);
2058            send_route_control_pushes(
2059                &self.forwarding,
2060                routes,
2061                ClientControlPush::RouteClosed {
2062                    module_id,
2063                    channels: Vec::new(),
2064                    reason,
2065                    drained,
2066                    abandoned,
2067                    excluded_subscriptions: 0,
2068                    terminal: Some(terminal),
2069                },
2070            );
2071        }
2072        if let Ok(cleanup) = cleanup {
2073            self.emit_route_goodbyes(cleanup.released);
2074        }
2075        // Signal the registration-release watch only now that BOTH registry and
2076        // forwarding teardown are done, so a supervisor waiting to spawn a
2077        // replacement never observes release while old routes still exist.
2078        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2079            crate::supervise::notify_registration_release();
2080            self.capability_evaluator.wake_deadline_loop();
2081            self.refresh_capability_requirements();
2082        }
2083        self.supervisor.remove_spawn_subscribers(connection_id);
2084        self.activity_streams
2085            .lock()
2086            .unwrap_or_else(|p| p.into_inner())
2087            .retain(|(id, _), task| {
2088                if *id == connection_id {
2089                    task.abort();
2090                    false
2091                } else {
2092                    true
2093                }
2094            });
2095        // Sync authority dies with its connection, so the owner's next
2096        // connection can take it; the owner's scopes stay as they are.
2097        self.hello_launch_nonces
2098            .lock()
2099            .unwrap_or_else(|poisoned| poisoned.into_inner())
2100            .forget(connection_id);
2101        self.scopes
2102            .write()
2103            .unwrap_or_else(|poisoned| poisoned.into_inner())
2104            .release_connection(connection_id);
2105        registrations
2106    }
2107
2108    pub(crate) fn handle_route_goodbye(
2109        &self,
2110        connection_id: ConnectionId,
2111        route_channel: u16,
2112        route_epoch: u32,
2113    ) -> Result<bool, RouterError> {
2114        debug!(
2115            connection_id = connection_id.get(),
2116            route_channel, route_epoch, "handling route GOODBYE"
2117        );
2118        let RouteRelease::Removed(released_route) = self
2119            .forwarding
2120            .release_client_route(connection_id, route_channel, route_epoch)
2121            .map_err(RouterError::Forwarding)?
2122        else {
2123            return Ok(false);
2124        };
2125        self.emit_route_goodbyes(vec![released_route]);
2126        Ok(true)
2127    }
2128
2129    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2130        for released in released_routes {
2131            let frame = match Frame::build_with_version(
2132                released.negotiated_ver,
2133                FrameType::Goodbye,
2134                control_flags(),
2135                released.channel,
2136                released.epoch,
2137                0,
2138                Vec::new(),
2139            ) {
2140                Ok(frame) => frame,
2141                Err(err) => {
2142                    warn!(
2143                        route_channel = released.channel,
2144                        error = %err,
2145                        "failed to build route GOODBYE frame"
2146                    );
2147                    continue;
2148                }
2149            };
2150            if !released.close_on_delivery_failure() {
2151                crate::forwarding::send_module_route_goodbye(
2152                    &self.counters,
2153                    &released.sink,
2154                    frame,
2155                    released.module_id.as_deref(),
2156                    "client route released",
2157                );
2158                continue;
2159            }
2160            if let Err(err) = released.sink.try_send(frame) {
2161                warn!(
2162                    target_connection_id = released.connection_id.get(),
2163                    route_channel = released.channel,
2164                    error = %err,
2165                    "route GOODBYE was not delivered to client; closing target connection"
2166                );
2167                if self
2168                    .forwarding
2169                    .escalate_client_delivery_failure(
2170                        released.connection_id,
2171                        released.channel,
2172                        released.epoch,
2173                        CloseReason::new(
2174                            "route_goodbye_delivery_failed",
2175                            format!(
2176                                "failed to enqueue route GOODBYE for channel {}: {err}",
2177                                released.channel
2178                            ),
2179                        ),
2180                        crate::forwarding::UndeliveredFrame {
2181                            module_id: released.module_id.as_deref(),
2182                            sink: &released.sink,
2183                        },
2184                    )
2185                    .unwrap_or(false)
2186                {
2187                    self.counters.increment_goodbye_relay_client_failed();
2188                }
2189            }
2190        }
2191    }
2192
2193    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2194    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2195    /// own commit failed after the module had already accepted). Without this, a
2196    /// module that accepts late keeps a binding subc has torn down, so a later
2197    /// frame on that module channel could misdeliver if the channel is reused.
2198    ///
2199    /// Never closes the shared module connection on failure: a dropped notification
2200    /// only wastes a bounded amount of warm module-side state, which the module's
2201    /// own idle reaper reclaims. Only call this once the route.bind relay was
2202    /// actually enqueued to the module — if the relay send itself failed, the
2203    /// module never created a binding and there is nothing to tear down.
2204    fn send_abandoned_route_bind_goodbye(
2205        &self,
2206        module_sink: &crate::FrameSink,
2207        negotiated_ver: u8,
2208        module_channel: u16,
2209        module_epoch: u32,
2210    ) {
2211        let frame = match Frame::build_with_version(
2212            negotiated_ver,
2213            FrameType::Goodbye,
2214            control_flags(),
2215            module_channel,
2216            module_epoch,
2217            0,
2218            Vec::new(),
2219        ) {
2220            Ok(frame) => frame,
2221            Err(err) => {
2222                warn!(
2223                    route_channel = module_channel,
2224                    error = %err,
2225                    "failed to build GOODBYE for abandoned route.bind"
2226                );
2227                return;
2228            }
2229        };
2230        crate::forwarding::send_module_route_goodbye(
2231            &self.counters,
2232            module_sink,
2233            frame,
2234            None,
2235            "abandoned route.bind",
2236        );
2237    }
2238
2239    fn handle_hello(
2240        &self,
2241        connection_id: ConnectionId,
2242        sink: Option<crate::FrameSink>,
2243        frame: Frame,
2244    ) -> Result<Vec<Frame>, RouterError> {
2245        debug!(
2246            connection_id = connection_id.get(),
2247            corr = frame.header.corr,
2248            "handling HELLO"
2249        );
2250        // A module connection has one identity for its entire lifetime. A second
2251        // registration would leave the old registry owner behind while replacing
2252        // its forwarding endpoint and launch nonce.
2253        if self
2254            .registry
2255            .get_module_by_connection(connection_id)
2256            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2257            .is_some()
2258        {
2259            return Ok(vec![control_error_frame(
2260                &frame,
2261                "invalid_hello",
2262                "connection is already registered as a module",
2263            )?]);
2264        }
2265        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2266            Ok(value) => value,
2267            Err(err) => {
2268                return Ok(vec![control_error_frame(
2269                    &frame,
2270                    "invalid_hello",
2271                    format!("malformed HELLO body: {err}"),
2272                )?])
2273            }
2274        };
2275        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2276            return Ok(vec![control_error_frame(
2277                &frame,
2278                "invalid_capability_grammar",
2279                err.to_string(),
2280            )?]);
2281        }
2282        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2283            return Ok(vec![control_error_frame(
2284                &frame,
2285                "invalid_manifest",
2286                err.to_string(),
2287            )?]);
2288        }
2289        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2290            return Ok(vec![control_error_frame(
2291                &frame,
2292                "invalid_event_declaration",
2293                err.to_string(),
2294            )?]);
2295        }
2296        if let Some(provenance) = hello_value
2297            .get("manifest")
2298            .and_then(|manifest| manifest.get("provenance"))
2299        {
2300            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2301                return Ok(vec![control_error_frame(
2302                    &frame,
2303                    "invalid_manifest",
2304                    format!("malformed manifest provenance: {err}"),
2305                )?]);
2306            }
2307        }
2308        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2309            Ok(hello) => hello,
2310            Err(err) => {
2311                return Ok(vec![control_error_frame(
2312                    &frame,
2313                    "invalid_hello",
2314                    format!("malformed HELLO body: {err}"),
2315                )?])
2316            }
2317        };
2318
2319        if hello.protocol_ver != hello.manifest.protocol_ver {
2320            return Ok(vec![control_error_frame(
2321                &frame,
2322                "invalid_manifest",
2323                format!(
2324                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2325                    hello.protocol_ver, hello.manifest.protocol_ver
2326                ),
2327            )?]);
2328        }
2329
2330        if hello.manifest.module_id.trim().is_empty() {
2331            return Ok(vec![control_error_frame(
2332                &frame,
2333                "invalid_manifest",
2334                "manifest module_id must not be empty",
2335            )?]);
2336        }
2337
2338        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2339            Ok(negotiated_ver) => negotiated_ver,
2340            Err(message) => {
2341                return Ok(vec![control_error_frame(
2342                    &frame,
2343                    "version_unsupported",
2344                    message,
2345                )?])
2346            }
2347        };
2348
2349        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2350        // swap is open for this id, the only HELLO admitted as a second process
2351        // is the one carrying the candidate's launch nonce (the swap token), and
2352        // it registers into the candidate slot rather than being refused as a
2353        // duplicate. Run after the reserved gate, a reserved module's candidate
2354        // would be refused `reserved_module` for presenting a nonce that gate
2355        // does not know. See `SupervisorHandle::swap_hello_admission`.
2356        let swap_admission = self
2357            .supervisor
2358            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2359        if swap_admission == SwapHelloAdmission::Refused {
2360            warn!(
2361                module_id = %hello.manifest.module_id,
2362                connection_id = connection_id.get(),
2363                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2364            );
2365            return Ok(vec![control_error_frame(
2366                &frame,
2367                "swap_token_invalid",
2368                format!(
2369                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2370                    hello.manifest.module_id
2371                ),
2372            )?]);
2373        }
2374        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2375
2376        // Reserved-module identity gate: a module_id configured `reserved` may be
2377        // registered ONLY by the process subc spawned for it, proven by echoing the
2378        // one-time launch nonce subc injected. A non-reserved id has no recorded
2379        // nonce and always passes. This blocks a key-holder from impersonating a
2380        // security-boundary module (e.g. the credential vault) while the real one is
2381        // down/restarting and its registration slot is momentarily free. A swap
2382        // candidate has already proven the same thing with its own nonce above.
2383        if let Some(rejection) = (!swap_candidate)
2384            .then(|| {
2385                self.supervisor.reserved_hello_rejection(
2386                    &hello.manifest.module_id,
2387                    hello.launch_nonce.as_deref(),
2388                )
2389            })
2390            .flatten()
2391        {
2392            let message = match rejection {
2393                ReservedHelloRejection::Exact { module_id } => format!(
2394                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2395                ),
2396                ReservedHelloRejection::Prefix {
2397                    prefix,
2398                    owner_module_id,
2399                } => format!(
2400                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2401                    hello.manifest.module_id
2402                ),
2403            };
2404            return Ok(vec![control_error_frame(
2405                &frame,
2406                "reserved_module",
2407                message,
2408            )?]);
2409        }
2410
2411        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2412            &hello.manifest.module_id,
2413            hello.manifest.capabilities.as_ref(),
2414        );
2415        if let Some(refusal) = reserved_capability_refusals.first() {
2416            let capability = refusal.capability.clone();
2417            let bound_module = refusal.claimants[0].clone();
2418            log_duplicate_claim_events(reserved_capability_refusals);
2419            return Ok(vec![control_error_frame(
2420                &frame,
2421                "reserved_capability",
2422                format!(
2423                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2424                    capability, bound_module, hello.manifest.module_id
2425                ),
2426            )?]);
2427        }
2428
2429        // A connection that already opened client routes must not also register as
2430        // a module: cleanup would then release only one side and leak the other.
2431        if self
2432            .forwarding
2433            .connection_has_client_routes(connection_id)
2434            .map_err(RouterError::Forwarding)?
2435        {
2436            return Ok(vec![control_error_frame(
2437                &frame,
2438                "invalid_hello",
2439                "connection has open client routes and cannot also register as a module",
2440            )?]);
2441        }
2442
2443        // Kept for scope sync authority, which goes only to the connection that
2444        // presented the module's current launch nonce. Recorded before the
2445        // registration is attempted: a connection whose registration then fails
2446        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2447        self.hello_launch_nonces
2448            .lock()
2449            .unwrap_or_else(|poisoned| poisoned.into_inner())
2450            .record(connection_id, hello.launch_nonce.as_deref());
2451        let control_ops = effective_module_control_ops(hello.control_ops);
2452        // Built before anything is registered so an encoding failure leaves no
2453        // registry or forwarding state behind.
2454        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2455        if swap_candidate {
2456            return self.register_swap_candidate(
2457                connection_id,
2458                sink,
2459                &frame,
2460                hello.manifest,
2461                negotiated_ver,
2462                control_ops,
2463                hello_ack,
2464            );
2465        }
2466        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2467        // is refused, but `catalog.list` does not show the module and the
2468        // catalog generation does not move until it is published below, once
2469        // its forwarding endpoint exists. Listing it now would advertise a
2470        // module that `route.open` still refuses for having no forwarding
2471        // connection.
2472        let registration = match self.registry.register_unpublished_with_control_ops(
2473            hello.manifest,
2474            negotiated_ver,
2475            connection_id,
2476            control_ops,
2477        ) {
2478            Ok(registration) => registration,
2479            Err(RegistryError::DuplicateModuleId { module_id }) => {
2480                return Ok(vec![control_error_frame(
2481                    &frame,
2482                    "duplicate_module_id",
2483                    format!(
2484                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2485                    ),
2486                )?])
2487            }
2488            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2489                return Ok(vec![control_error_frame(
2490                    &frame,
2491                    "invalid_module_id",
2492                    err.to_string(),
2493                )?])
2494            }
2495            Err(err) => {
2496                return Ok(vec![control_error_frame(
2497                    &frame,
2498                    "registry_error",
2499                    err.to_string(),
2500                )?])
2501            }
2502        };
2503        #[cfg(test)]
2504        self.hold_between_registry_and_forwarding();
2505
2506        let reply = if let Some(sink) = sink {
2507            // The forwarding table's module store is also the daemon-to-module
2508            // control-RPC lane, so every HELLO gets a live endpoint even when the
2509            // manifest has no routable provider role. Non-routable modules still
2510            // cannot receive route.bind in production: `handle_route_open` checks
2511            // the registry manifest with `target_has_required_role` before the
2512            // only production call to `begin_route_bind_relay_for` below that
2513            // route.open path. The remaining direct relay callers are unit tests
2514            // and benchmark harnesses that construct forwarding state explicitly.
2515            //
2516            // The HELLO_ACK is queued by the forwarding table itself, before the
2517            // endpoint becomes visible, and is NOT returned as a reply. A module
2518            // reads HELLO_ACK first and exits on anything else; a reply is only
2519            // written after this handler returns, by which time a route.open on
2520            // another connection could already have queued a route.bind request
2521            // for this module ahead of it.
2522            let concurrency = manifest_concurrency(&registration.manifest);
2523            if let Err(err) = self.forwarding.register_module_connection_acked(
2524                connection_id,
2525                registration.manifest.module_id.clone(),
2526                negotiated_ver,
2527                concurrency,
2528                sink,
2529                hello_ack,
2530            ) {
2531                // Forwarding registration failed, so there is no forwarding
2532                // state to tear down. Remove the registry entry, which was never
2533                // published and so never listed or counted in the catalog
2534                // generation, and signal the release watch directly.
2535                if matches!(
2536                    self.deregister_connection(
2537                        connection_id,
2538                        RegistrationEndReason::RegistrationFailed,
2539                    ),
2540                    Ok(r) if !r.is_empty()
2541                ) {
2542                    crate::supervise::notify_registration_release();
2543                }
2544                return Ok(vec![control_error_frame(
2545                    &frame,
2546                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2547                        "invalid_hello"
2548                    } else {
2549                        forwarding_error_code(&err)
2550                    },
2551                    err.to_string(),
2552                )?]);
2553            }
2554            Vec::new()
2555        } else {
2556            // No sink means no forwarding endpoint, so nothing can be routed
2557            // ahead of the ack; it goes out as the reply.
2558            vec![hello_ack]
2559        };
2560
2561        // The module is routable now (or, without a sink, as routable as it
2562        // will ever be), so the catalog may list it. Publishing bumps the
2563        // catalog generation in the same registry critical section that makes
2564        // the module listed.
2565        if !self
2566            .registry
2567            .publish_registration(connection_id)
2568            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2569        {
2570            debug!(
2571                module_id = %registration.manifest.module_id,
2572                connection_id = connection_id.get(),
2573                "registration was removed before it could be published"
2574            );
2575        }
2576
2577        // Exposure over assumption: Concurrency's serde default is pinned to the
2578        // pre-field behavior (ModuleManaged), so a management surface that is
2579        // genuinely Serial and just never declared it inherits concurrent
2580        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2581        // turns "no module has been bitten yet" into the checkable claim "no
2582        // module is exposed" -- one read of the boot log instead of a fleet
2583        // audit. Detected from the raw HELLO bytes because the serde default
2584        // deliberately erases the absent/declared distinction from the type.
2585        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2586            info!(
2587                module_id = %registration.manifest.module_id,
2588                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2589            );
2590        }
2591
2592        self.apply_registration_capabilities(&registration);
2593
2594        self.supervisor
2595            .record_activity_registration(&registration.manifest.module_id);
2596
2597        info!(
2598            module_id = %registration.manifest.module_id,
2599            module_version = %registration.manifest.module_version,
2600            negotiated_ver,
2601            routable_provider = manifest_provides_routable_role(&registration.manifest),
2602            connection_id = connection_id.get(),
2603            "module registered"
2604        );
2605
2606        Ok(reply)
2607    }
2608
2609    /// Register a HELLO the swap gate admitted into the candidate slot of the
2610    /// registry and of forwarding, where it is reachable over its own
2611    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2612    /// nothing routes to it until the supervisor cuts over.
2613    ///
2614    /// Registry first (unpublished), then forwarding, then publication, the
2615    /// same order as an ordinary HELLO; a forwarding failure removes the
2616    /// registry entry again. The swap supervisor treats a published, ready
2617    /// candidate as registered and promotes it, and promotion needs the
2618    /// candidate's forwarding endpoint, so the candidate is published only
2619    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2620    /// recompute capability verdicts or run the capability deny census
2621    /// (`enforce_capability_denies`): both are about the module in the active
2622    /// slot, which `route.open` can target. A candidate sits in the candidate
2623    /// slot, so it neither provides capabilities to other modules nor can be
2624    /// routed to until promotion moves it into the active slot.
2625    #[allow(clippy::too_many_arguments)]
2626    fn register_swap_candidate(
2627        &self,
2628        connection_id: ConnectionId,
2629        sink: Option<crate::FrameSink>,
2630        frame: &Frame,
2631        manifest: ModuleManifest,
2632        negotiated_ver: u8,
2633        control_ops: Vec<String>,
2634        hello_ack: Frame,
2635    ) -> Result<Vec<Frame>, RouterError> {
2636        let module_id = manifest.module_id.clone();
2637        let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2638            manifest,
2639            negotiated_ver,
2640            connection_id,
2641            control_ops,
2642        ) {
2643            Ok(registration) => registration,
2644            Err(RegistryError::DuplicateModuleId { module_id }) => {
2645                return Ok(vec![control_error_frame(
2646                    frame,
2647                    "duplicate_module_id",
2648                    format!(
2649                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2650                    ),
2651                )?])
2652            }
2653            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2654                return Ok(vec![control_error_frame(
2655                    frame,
2656                    "invalid_module_id",
2657                    err.to_string(),
2658                )?])
2659            }
2660            Err(err) => {
2661                return Ok(vec![control_error_frame(
2662                    frame,
2663                    "registry_error",
2664                    err.to_string(),
2665                )?])
2666            }
2667        };
2668        #[cfg(test)]
2669        self.hold_between_registry_and_forwarding();
2670        let reply = if let Some(sink) = sink {
2671            // Same ordering as an ordinary HELLO: the forwarding table queues
2672            // the HELLO_ACK before the candidate endpoint is inserted, because
2673            // a module exits if its first frame after HELLO is anything else.
2674            let concurrency = manifest_concurrency(&registration.manifest);
2675            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2676                connection_id,
2677                module_id.clone(),
2678                negotiated_ver,
2679                concurrency,
2680                sink,
2681                hello_ack,
2682            ) {
2683                if matches!(
2684                    self.deregister_connection(
2685                        connection_id,
2686                        RegistrationEndReason::RegistrationFailed,
2687                    ),
2688                    Ok(r) if !r.is_empty()
2689                ) {
2690                    crate::supervise::notify_registration_release();
2691                }
2692                return Ok(vec![control_error_frame(
2693                    frame,
2694                    forwarding_error_code(&err),
2695                    err.to_string(),
2696                )?]);
2697            }
2698            Vec::new()
2699        } else {
2700            vec![hello_ack]
2701        };
2702        if !self
2703            .registry
2704            .publish_registration(connection_id)
2705            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2706        {
2707            debug!(
2708                module_id = %module_id,
2709                connection_id = connection_id.get(),
2710                "swap candidate registration was removed before it could be published"
2711            );
2712        }
2713        self.supervisor.mark_swap_candidate_admitted(&module_id);
2714        info!(
2715            module_id = %module_id,
2716            module_version = %registration.manifest.module_version,
2717            negotiated_ver,
2718            ready = registration.ready,
2719            connection_id = connection_id.get(),
2720            "swap candidate registered; not routable until cutover"
2721        );
2722        Ok(reply)
2723    }
2724
2725    fn build_hello_ack(
2726        &self,
2727        frame: &Frame,
2728        negotiated_ver: u8,
2729        module_id: &str,
2730    ) -> Result<Frame, RouterError> {
2731        let ack = ModuleHelloAckBody {
2732            negotiated_ver,
2733            subc_ops: module_subc_ops(),
2734            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2735            storage: self
2736                .storage_config
2737                .as_ref()
2738                .map(|cfg| cfg.descriptor_for(module_id)),
2739            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2740        };
2741        let body = serde_json::to_vec(&ack).map_err(|err| {
2742            RouterError::backend(
2743                0,
2744                frame.header.corr,
2745                format!("failed to encode HELLO_ACK: {err}"),
2746            )
2747        })?;
2748
2749        Frame::build_with_version(
2750            negotiated_ver,
2751            FrameType::HelloAck,
2752            control_flags(),
2753            0,
2754            0,
2755            frame.header.corr,
2756            body,
2757        )
2758        .map_err(RouterError::FrameBuild)
2759    }
2760
2761    async fn handle_client_control_request(
2762        &self,
2763        ctx: &RouteCtx,
2764        frame: Frame,
2765        request: ClientControlRequest,
2766        route_open_timings: &mut Option<RouteOpenTimings>,
2767    ) -> Result<Vec<Frame>, RouterError> {
2768        if let Some(operation) = privileged_operation(&request) {
2769            let caller = self.resolve_caller(ctx.connection_id).await;
2770            let op = client_control_request_op(&request);
2771            let text = prompt_text(operation, &caller);
2772            match self.privileged_operator_prompt {
2773                PrivilegedOperatorPrompt::Log => {
2774                    info!(target: "subc_daemon::privileged_prompt", operation = op,
2775                        target = ?operation, caller = %caller.description(), summary = %text,
2776                        "privileged request would prompt the operator");
2777                }
2778                PrivilegedOperatorPrompt::Enforce => {
2779                    let key =
2780                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2781                    let outcome = self
2782                        .forwarding
2783                        .operator_confirms()
2784                        .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2785                        .await;
2786                    if outcome.code != "confirmed" {
2787                        return Ok(vec![outcome.frame(&frame)]);
2788                    }
2789                }
2790            }
2791        }
2792        match request {
2793            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2794            ClientControlRequest::CatalogList { module_id } => {
2795                self.handle_catalog_list(frame, module_id)
2796            }
2797            ClientControlRequest::RouteOpen {
2798                target,
2799                identity,
2800                consumer_identity,
2801                consumer_capabilities,
2802                role_versions,
2803                admission_facts,
2804                scope,
2805            } => {
2806                self.handle_route_open(
2807                    ctx,
2808                    frame,
2809                    RouteOpenRequest {
2810                        target,
2811                        identity,
2812                        consumer_identity,
2813                        consumer_capabilities,
2814                        role_versions,
2815                        admission_facts,
2816                        scope,
2817                    },
2818                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2819                )
2820                .await
2821            }
2822            ClientControlRequest::RoutePoll {
2823                route_channel,
2824                route_epoch,
2825                kind,
2826            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2827            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2828            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2829                self.handle_supervisor_spawn_snapshot(frame)
2830            }
2831            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2832                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2833            }
2834            ClientControlRequest::SupervisorRestart {
2835                module_id,
2836                drain_timeout_ms,
2837            } => {
2838                self.log_supervisor_request_received(
2839                    ctx,
2840                    frame.header.corr,
2841                    ops::SUPERVISOR_RESTART,
2842                    Some(&module_id),
2843                    None,
2844                )?;
2845                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2846                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2847                    .await
2848            }
2849            ClientControlRequest::SupervisorSwap {
2850                module_id,
2851                ready_timeout_ms,
2852            } => {
2853                self.log_supervisor_request_received(
2854                    ctx,
2855                    frame.header.corr,
2856                    ops::SUPERVISOR_SWAP,
2857                    Some(&module_id),
2858                    None,
2859                )?;
2860                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2861                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2862                    .await
2863            }
2864            ClientControlRequest::SupervisorReload { module_id } => {
2865                self.log_supervisor_request_received(
2866                    ctx,
2867                    frame.header.corr,
2868                    ops::SUPERVISOR_RELOAD,
2869                    Some(&module_id),
2870                    None,
2871                )?;
2872                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2873                self.handle_supervisor_reload(frame, module_id, requester)
2874                    .await
2875            }
2876            ClientControlRequest::SupervisorRescan { preview } => {
2877                if !preview {
2878                    self.log_supervisor_request_received(
2879                        ctx,
2880                        frame.header.corr,
2881                        ops::SUPERVISOR_RESCAN,
2882                        None,
2883                        None,
2884                    )?;
2885                }
2886                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2887                self.handle_supervisor_rescan(frame, preview, requester)
2888                    .await
2889            }
2890            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2891                self.log_supervisor_request_received(
2892                    ctx,
2893                    frame.header.corr,
2894                    ops::SUPERVISOR_RELEASE_RESERVED,
2895                    Some(&module_id),
2896                    None,
2897                )?;
2898                self.handle_supervisor_release_reserved(frame, module_id)
2899                    .await
2900            }
2901            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2902                self.log_supervisor_request_received(
2903                    ctx,
2904                    frame.header.corr,
2905                    ops::SUPERVISOR_SET_ENABLED,
2906                    Some(&module_id),
2907                    Some(enabled),
2908                )?;
2909                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2910                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2911                    .await
2912            }
2913            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2914                self.handle_supervisor_health_probe(frame, module_id).await
2915            }
2916            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2917            ClientControlRequest::SupervisorRoutes { module_id } => {
2918                self.handle_supervisor_routes(frame, module_id)
2919            }
2920            ClientControlRequest::SupervisorProvenance { module_id } => {
2921                self.handle_supervisor_provenance(frame, module_id).await
2922            }
2923            ClientControlRequest::SupervisorStderrTail {
2924                module_id,
2925                max_lines,
2926                max_bytes,
2927            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2928            ClientControlRequest::SupervisorTerminals { module_id } => {
2929                self.handle_supervisor_terminals(frame, module_id).await
2930            }
2931        }
2932    }
2933
2934    fn log_supervisor_request_received(
2935        &self,
2936        ctx: &RouteCtx,
2937        corr: u64,
2938        op: &'static str,
2939        module_id: Option<&str>,
2940        enabled: Option<bool>,
2941    ) -> Result<(), RouterError> {
2942        let caller = self.supervisor_requester(ctx, corr)?;
2943        let caller = principal_label(&caller);
2944        // `Option` fields are recorded only when present, so a request with no
2945        // module id (rescan) or no enabled flag simply omits that field.
2946        info!(
2947            target: "control",
2948            op,
2949            module_id,
2950            enabled,
2951            connection_id = ctx.connection_id.get(),
2952            caller = %caller,
2953            "supervisor request received"
2954        );
2955        Ok(())
2956    }
2957
2958    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2959        Ok(self
2960            .registry
2961            .get_module_by_connection(ctx.connection_id)
2962            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2963            .map(|registration| Principal::Reserved {
2964                module_id: registration.manifest.module_id,
2965            })
2966            .unwrap_or(Principal::Direct))
2967    }
2968
2969    fn handle_module_control_request(
2970        &self,
2971        connection_id: ConnectionId,
2972        frame: Frame,
2973        request: ModuleControlRequestFromModule,
2974    ) -> Result<Vec<Frame>, RouterError> {
2975        match request {
2976            ModuleControlRequestFromModule::CatalogUpdate {
2977                provides,
2978                capabilities,
2979                ready,
2980            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2981            ModuleControlRequestFromModule::LiveRoots {} => {
2982                let registered = self
2983                    .registry
2984                    .get_module_by_connection(connection_id)
2985                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2986                let Some(registration) = registered else {
2987                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2988                };
2989                let response = self
2990                    .forwarding
2991                    .live_roots(&registration.manifest.module_id)
2992                    .map_err(RouterError::Forwarding)?;
2993                Ok(vec![control_response_body_frame(
2994                    &frame,
2995                    &response,
2996                    "ModuleControlResponseToModule::LiveRoots",
2997                )?])
2998            }
2999            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3000                self.handle_scope_sync(connection_id, frame, generation, scopes)
3001            }
3002            ModuleControlRequestFromModule::ScopeApply {
3003                generation,
3004                upsert,
3005                end,
3006            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3007            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3008                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3009            }
3010        }
3011    }
3012
3013    fn handle_scope_served_by(
3014        &self,
3015        connection_id: ConnectionId,
3016        frame: Frame,
3017    ) -> Result<Vec<Frame>, RouterError> {
3018        let registered = self
3019            .registry
3020            .get_module_by_connection(connection_id)
3021            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3022        let Some(registered) = registered else {
3023            return Ok(vec![control_error_frame(
3024                &frame,
3025                "not_registered",
3026                "scope.served_by requires an active module registration owned by this connection",
3027            )?]);
3028        };
3029        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3030        {
3031            Ok(request) => request,
3032            Err(err) => {
3033                return Ok(vec![control_error_frame(
3034                    &frame,
3035                    "invalid_control_body",
3036                    format!("malformed scope.served_by body: {err}"),
3037                )?])
3038            }
3039        };
3040        let answer = self
3041            .scopes
3042            .read()
3043            .unwrap_or_else(|poisoned| poisoned.into_inner())
3044            .served_by(
3045                &registered.manifest.module_id,
3046                &request.scope_ref,
3047                request.scope_epoch,
3048                &request.module_id,
3049            );
3050        let reply = subc_control::ScopeServedByReply::new(
3051            answer,
3052            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3053        );
3054        Ok(vec![control_response_body_frame(
3055            &frame,
3056            &reply,
3057            "ScopeServedByReply",
3058        )?])
3059    }
3060
3061    fn handle_operator_confirm(
3062        &self,
3063        ctx: &RouteCtx,
3064        frame: Frame,
3065    ) -> Result<Vec<Frame>, RouterError> {
3066        use crate::operator_confirm::{audit, Outcome};
3067        let registration = self
3068            .registry
3069            .get_module_by_connection(ctx.connection_id)
3070            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3071        let Some(registration) = registration else {
3072            let outcome = Outcome::refusal("not_registered");
3073            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3074            return Ok(vec![outcome.frame(&frame)]);
3075        };
3076        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3077            Ok(request) => request,
3078            Err(_) => {
3079                let outcome = Outcome::refusal("invalid_control_body");
3080                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3081                return Ok(vec![outcome.frame(&frame)]);
3082            }
3083        };
3084        let module_id = registration.manifest.module_id;
3085        // Read the launch nonce (under its own lock) before taking the forwarding
3086        // table's lock below: holding forwarding while waiting on another daemon
3087        // lock risks a lock-order deadlock with paths that take them the other way.
3088        let nonce = self
3089            .hello_launch_nonces
3090            .lock()
3091            .unwrap_or_else(|p| p.into_inner())
3092            .nonce(ctx.connection_id)
3093            .map(str::to_owned);
3094        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3095            self.supervisor
3096                .spawned_consumer_authorized(&module_id, nonce)
3097        });
3098        let confirms = self.forwarding.operator_confirms();
3099        self.forwarding
3100            .with_operator_route(
3101                ctx.connection_id,
3102                request.route_channel,
3103                request.route_epoch,
3104                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3105            )
3106            .map_err(RouterError::Forwarding)
3107    }
3108
3109    /// `scope.sync`: the owner is the module registered on this connection.
3110    /// A connection with no registration (every client connection, `direct`
3111    /// included) is refused `not_registered` before the table is consulted.
3112    fn handle_scope_sync(
3113        &self,
3114        connection_id: ConnectionId,
3115        frame: Frame,
3116        generation: u64,
3117        scopes: Vec<ScopeRecord>,
3118    ) -> Result<Vec<Frame>, RouterError> {
3119        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3120    }
3121
3122    fn handle_scope_change(
3123        &self,
3124        connection_id: ConnectionId,
3125        frame: Frame,
3126        generation: u64,
3127        scopes: Vec<ScopeRecord>,
3128        end: Option<Vec<ScopeEnd>>,
3129    ) -> Result<Vec<Frame>, RouterError> {
3130        let is_apply = end.is_some();
3131        let Some(registration) = self
3132            .registry
3133            .get_module_by_connection(connection_id)
3134            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3135        else {
3136            return Ok(vec![control_error_frame(
3137                &frame,
3138                "not_registered",
3139                "scope.sync and scope.apply require an active module registration owned by this connection",
3140            )?]);
3141        };
3142        let owner = registration.manifest.module_id;
3143        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3144        let is_current_launch = |connection: ConnectionId| {
3145            self.hello_launch_nonces
3146                .lock()
3147                .unwrap_or_else(|poisoned| poisoned.into_inner())
3148                .presented(connection, current_nonce.as_deref())
3149        };
3150        // Lock order is the scope table, then the forwarding table: the new
3151        // tags are published, and the routes the change closes are selected,
3152        // while the scope table is still write-locked, so no admission can read
3153        // a record whose tag is not yet published.
3154        let mut table = self
3155            .scopes
3156            .write()
3157            .unwrap_or_else(|poisoned| poisoned.into_inner());
3158        let outcome = match end {
3159            Some(end) => table.apply(
3160                &owner,
3161                connection_id,
3162                is_current_launch,
3163                generation,
3164                scopes,
3165                end,
3166            ),
3167            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3168        };
3169        let drained = match &outcome {
3170            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3171            Err(_) => Vec::new(),
3172        };
3173        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3174        drop(table);
3175        if has_deadlines {
3176            self.scope_deadline_added.notify_one();
3177        }
3178        match outcome {
3179            Ok(applied) => {
3180                let counts = ScopeOutcomeCounts::of(&applied.results);
3181                info!(
3182                    owner = %owner,
3183                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3184                    generation,
3185                    records = applied.results.len(),
3186                    created = counts.created,
3187                    replaced = counts.replaced,
3188                    updated = counts.updated,
3189                    unchanged = counts.unchanged,
3190                    refused = counts.refused,
3191                    ended = applied.ended.len(),
3192                    tag_changes = applied.tag_changes.len(),
3193                    routes_closed = drained.len(),
3194                    "scope change accepted"
3195                );
3196                // An accepted scope change can still refuse individual records, and the
3197                // owner is the only party that sees the reply. Name them here so
3198                // an operator can tell a refused session from a missing one
3199                // without the owner's logs. Capped so a call that refuses
3200                // thousands cannot flood the log; the count above is complete.
3201                for refused in applied
3202                    .results
3203                    .iter()
3204                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3205                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3206                {
3207                    warn!(
3208                        owner = %owner,
3209                        generation,
3210                        scope_ref = %refused.scope_ref,
3211                        scope_epoch = refused.scope_epoch,
3212                        code = refused.code.as_deref().unwrap_or(""),
3213                        "scope record refused"
3214                    );
3215                }
3216                self.close_scope_drained_routes(drained);
3217                let response = if is_apply {
3218                    ModuleControlResponseToModule::ScopeApply {
3219                        generation,
3220                        results: applied.results,
3221                        end_results: applied.end_results,
3222                        ended: applied.ended,
3223                    }
3224                } else {
3225                    ModuleControlResponseToModule::ScopeSync {
3226                        generation,
3227                        results: applied.results,
3228                        ended: applied.ended,
3229                    }
3230                };
3231                Ok(vec![control_response_body_frame(
3232                    &frame,
3233                    &response,
3234                    "scope change response",
3235                )?])
3236            }
3237            Err(refusal) => {
3238                info!(
3239                    owner = %owner,
3240                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3241                    generation,
3242                    code = refusal.code,
3243                    "scope change refused"
3244                );
3245                Ok(vec![control_error_frame(
3246                    &frame,
3247                    refusal.code,
3248                    refusal.message,
3249                )?])
3250            }
3251        }
3252    }
3253
3254    /// Tell both ends of each route a scope change closed. The module gets a
3255    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3256    /// the client's route handle. The client also gets `route.closed` with the
3257    /// scope reason, one push per module and reason, so it can tell a revoked
3258    /// route from an ordinary close and not reopen it.
3259    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3260        if drained.is_empty() {
3261            return;
3262        }
3263        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3264            BTreeMap::new();
3265        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3266        for route in drained {
3267            warn!(
3268                module_id = %route.module_id,
3269                reason = ?route.reason,
3270                client_connection_id = route.client.connection_id.get(),
3271                route_channel = route.client.channel,
3272                "closing route because its scope changed"
3273            );
3274            pushes
3275                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3276                .or_insert_with(|| (route.reason, Vec::new()))
3277                .1
3278                .push(EndpointRoute {
3279                    goodbye_target: route.client.clone(),
3280                    principal: Principal::Unverified,
3281                    bound_at: Instant::now(),
3282                    draining: false,
3283                    drain_reason: None,
3284                });
3285            goodbyes.push(route.module);
3286            goodbyes.push(route.client);
3287        }
3288        for ((module_id, _), (reason, routes)) in pushes {
3289            send_route_control_pushes(
3290                &self.forwarding,
3291                routes,
3292                ClientControlPush::RouteClosed {
3293                    module_id,
3294                    channels: Vec::new(),
3295                    reason,
3296                    drained: false,
3297                    abandoned: 0,
3298                    excluded_subscriptions: 0,
3299                    terminal: Some(false),
3300                },
3301            );
3302        }
3303        self.emit_route_goodbyes(goodbyes);
3304    }
3305
3306    /// `scope.describe`: any registered module may read any scope, because a
3307    /// provider must read the scope a route it serves is stamped with.
3308    fn handle_scope_describe(
3309        &self,
3310        connection_id: ConnectionId,
3311        frame: Frame,
3312        owner: Principal,
3313        scope_ref: String,
3314    ) -> Result<Vec<Frame>, RouterError> {
3315        let registered = self
3316            .registry
3317            .get_module_by_connection(connection_id)
3318            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3319        let Some(registered) = registered else {
3320            return Ok(vec![control_error_frame(
3321                &frame,
3322                "not_registered",
3323                "scope.describe requires an active module registration owned by this connection",
3324            )?]);
3325        };
3326        let mut description = self
3327            .scopes
3328            .read()
3329            .unwrap_or_else(|poisoned| poisoned.into_inner())
3330            .describe(&owner, &scope_ref);
3331        // Describe readers also decode strict ScopeAttributes. Attribution is
3332        // safe to omit unless the requesting module declares it understands it.
3333        if !registered
3334            .manifest
3335            .capabilities
3336            .as_ref()
3337            .is_some_and(|caps| {
3338                caps.provides
3339                    .iter()
3340                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3341            })
3342        {
3343            if let Some(stamp) = description.stamp.as_mut() {
3344                stamp.attributes.on_behalf_of_agent = None;
3345            }
3346        }
3347        let owner_configured = match &owner {
3348            // Ask whether the owner is configured (`is_configured`), not
3349            // whether it is on the roster (`get(..).is_some()`): a supervised
3350            // module's process can register and describe a scope before the
3351            // supervisor has put it on the roster.
3352            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3353            _ => false,
3354        };
3355        let response = ModuleControlResponseToModule::ScopeDescribe {
3356            status: description.status,
3357            scope_epoch: description.scope_epoch,
3358            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3359            owner_synced: description.owner_synced,
3360            owner_configured,
3361            scope: description.stamp,
3362        };
3363        Ok(vec![control_response_body_frame(
3364            &frame,
3365            &response,
3366            "ModuleControlResponseToModule::ScopeDescribe",
3367        )?])
3368    }
3369
3370    fn handle_catalog_update(
3371        &self,
3372        connection_id: ConnectionId,
3373        frame: Frame,
3374        provides: Vec<ProviderRole>,
3375        capabilities: Option<CapabilityDeclarations>,
3376        ready: Option<bool>,
3377    ) -> Result<Vec<Frame>, RouterError> {
3378        self.refresh_capability_requirements();
3379        let Some(registration) = self
3380            .registry
3381            .get_module_by_connection(connection_id)
3382            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3383        else {
3384            return Ok(vec![control_error_frame(
3385                &frame,
3386                "not_registered",
3387                "catalog.update requires an active module registration owned by this connection",
3388            )?]);
3389        };
3390
3391        if let Some(message) =
3392            catalog_update_frozen_field_message(&registration.manifest, &provides)
3393        {
3394            return Ok(vec![control_error_frame(
3395                &frame,
3396                "catalog_update_frozen_field",
3397                message,
3398            )?]);
3399        }
3400
3401        let mut candidate = registration.manifest.clone();
3402        candidate.provides = provides.clone();
3403        candidate.capabilities = capabilities
3404            .clone()
3405            .or_else(|| registration.manifest.capabilities.clone());
3406        if let Err(err) = candidate.validate_capability_grammar() {
3407            return Ok(vec![control_error_frame(
3408                &frame,
3409                "invalid_capability_grammar",
3410                err.to_string(),
3411            )?]);
3412        }
3413
3414        // Updates must honor the same reserved owner as initial registration;
3415        // otherwise an empty HELLO could acquire the claim after admission.
3416        let mut conflicts = self
3417            .capability_evaluator
3418            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3419        if let Some(conflict) = conflicts.first() {
3420            let message = format!(
3421                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3422                conflict.capability, conflict.claimants[0], candidate.module_id
3423            );
3424            for conflict in &mut conflicts {
3425                conflict.source = DuplicateClaimSource::CatalogUpdate;
3426            }
3427            log_duplicate_claim_events(conflicts);
3428            return Ok(vec![control_error_frame(
3429                &frame,
3430                "reserved_capability",
3431                message,
3432            )?]);
3433        }
3434
3435        let updated = self
3436            .registry
3437            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3438            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3439        if updated.is_none() {
3440            return Ok(vec![control_error_frame(
3441                &frame,
3442                "not_registered",
3443                "catalog.update requires an active module registration owned by this connection",
3444            )?]);
3445        }
3446        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3447            log_duplicate_claim_events(
3448                self.capability_evaluator
3449                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3450            );
3451        }
3452        if capability_census_trigger(
3453            registration.manifest.capabilities.as_ref(),
3454            updated
3455                .as_ref()
3456                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3457        ) {
3458            self.enforce_capability_denies();
3459        }
3460        self.refresh_capability_requirements();
3461
3462        let response = ModuleControlResponseToModule::CatalogUpdate {};
3463        control_response_body_frame(
3464            &frame,
3465            &response,
3466            "ModuleControlResponseToModule::CatalogUpdate",
3467        )
3468        .map(|frame| vec![frame])
3469    }
3470
3471    pub(crate) fn with_activity_ledger_config(
3472        mut self,
3473        run_dir: &Path,
3474        incarnation: String,
3475        config: crate::activity::ActivityConfig,
3476    ) -> Self {
3477        self.activity = Some(crate::activity::ActivityLedger::open_config(
3478            run_dir.join("activity.jsonl"),
3479            incarnation,
3480            config,
3481        ));
3482        self.supervisor.set_activity(self.activity.clone());
3483        self
3484    }
3485
3486    pub(crate) async fn configure_activity(
3487        &self,
3488        config: crate::activity::ActivityConfig,
3489    ) -> Result<(), &'static str> {
3490        match &self.activity {
3491            Some(ledger) => ledger.configure(config).await,
3492            None => Ok(()),
3493        }
3494    }
3495
3496    fn cancel_activity_subscription(
3497        &self,
3498        ctx: &RouteCtx,
3499        frame: &Frame,
3500    ) -> Result<bool, RouterError> {
3501        let task = self
3502            .activity_streams
3503            .lock()
3504            .unwrap_or_else(|p| p.into_inner())
3505            .remove(&(ctx.connection_id, frame.header.corr));
3506        let Some(task) = task else {
3507            return Ok(false);
3508        };
3509        task.abort();
3510        let end = Frame::build_with_version(
3511            response_version(frame),
3512            FrameType::StreamEnd,
3513            control_flags(),
3514            0,
3515            0,
3516            frame.header.corr,
3517            Vec::new(),
3518        )
3519        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3520        let sink = ctx.egress.clone();
3521        tokio::spawn(async move {
3522            let _ = sink.send(end).await;
3523        });
3524        Ok(true)
3525    }
3526
3527    async fn handle_activity_read(
3528        &self,
3529        ctx: &RouteCtx,
3530        frame: Frame,
3531    ) -> Result<Vec<Frame>, RouterError> {
3532        use subc_control::{
3533            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3534            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3535        };
3536        #[derive(Deserialize)]
3537        struct Identity {
3538            #[serde(default)]
3539            consumer_identity: Option<ConsumerIdentity>,
3540        }
3541        // A registered module connection already identifies its caller. Ignore
3542        // consumer_identity there so a module cannot borrow a mirror's identity.
3543        // Only separate consumer connections use the Direct/nonce trust boundary.
3544        let principal = match self
3545            .forwarding
3546            .module_id_for_connection(ctx.connection_id)
3547            .map_err(RouterError::Forwarding)?
3548        {
3549            Some(module_id) => Ok(Principal::Reserved { module_id }),
3550            None => {
3551                let identity: Identity = match serde_json::from_slice(&frame.body) {
3552                    Ok(identity) => identity,
3553                    Err(_) => {
3554                        return Ok(vec![control_error_frame(
3555                            &frame,
3556                            ACTIVITY_MALFORMED,
3557                            "malformed activity read",
3558                        )?])
3559                    }
3560                };
3561                self.route_open_principal(&frame, identity.consumer_identity)?
3562            }
3563        };
3564        let admitted = matches!(principal, Ok(Principal::Direct))
3565            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3566        if !admitted {
3567            return Ok(vec![control_error_frame(
3568                &frame,
3569                ACTIVITY_READ_FORBIDDEN,
3570                "activity reads require an operator or trusted mirror principal",
3571            )?]);
3572        }
3573        let Some(ledger) = &self.activity else {
3574            return Ok(vec![control_error_frame(
3575                &frame,
3576                ACTIVITY_UNAVAILABLE,
3577                "activity ledger unavailable",
3578            )?]);
3579        };
3580        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3581            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3582        {
3583            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3584                Ok(request) => ledger.query(request).await,
3585                Err(_) => Err(ACTIVITY_MALFORMED),
3586            };
3587            return match result {
3588                Ok(reply) => Ok(vec![control_response_body_frame(
3589                    &frame,
3590                    &reply,
3591                    ops::ACTIVITY_QUERY,
3592                )?]),
3593                Err(code) => Ok(vec![control_error_frame(
3594                    &frame,
3595                    code,
3596                    "activity query refused",
3597                )?]),
3598            };
3599        }
3600        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3601            Ok(request) => request,
3602            Err(_) => {
3603                return Ok(vec![control_error_frame(
3604                    &frame,
3605                    ACTIVITY_MALFORMED,
3606                    "malformed activity subscription",
3607                )?])
3608            }
3609        };
3610        let mut subscription = match ledger.subscribe(request.since).await {
3611            Ok(subscription) => subscription,
3612            Err(code) => {
3613                return Ok(vec![control_error_frame(
3614                    &frame,
3615                    code,
3616                    "activity subscription refused",
3617                )?])
3618            }
3619        };
3620        let key = (ctx.connection_id, frame.header.corr);
3621        let sink = ctx.egress.clone();
3622        let mut streams = self
3623            .activity_streams
3624            .lock()
3625            .unwrap_or_else(|p| p.into_inner());
3626        streams.retain(|_, task| !task.is_finished());
3627        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3628            return Ok(vec![control_error_frame(
3629                &frame,
3630                "duplicate_subscription",
3631                "activity subscription correlation id already in use",
3632            )?]);
3633        }
3634        let task = tokio::spawn(async move {
3635            loop {
3636                let entry = if let Some(entry) = subscription.replay.pop_front() {
3637                    Some(entry)
3638                } else {
3639                    subscription.receiver.recv().await
3640                };
3641                let Some(entry) = entry else {
3642                    break;
3643                };
3644                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3645                    return;
3646                };
3647                let Ok(next) = Frame::build_with_version(
3648                    response_version(&frame),
3649                    FrameType::StreamData,
3650                    control_flags(),
3651                    0,
3652                    0,
3653                    frame.header.corr,
3654                    body,
3655                ) else {
3656                    return;
3657                };
3658                if sink.send(next).await.is_err() {
3659                    return;
3660                }
3661            }
3662            // Dropping a full subscriber's sender drains its accepted entries
3663            // before the terminal error, preserving their order on the wire.
3664            if let Some(code) = subscription.terminal_code() {
3665                if let Ok(end) =
3666                    control_error_frame(&frame, code, "activity subscriber fell behind")
3667                {
3668                    let _ = sink.send(end).await;
3669                }
3670            }
3671        });
3672        streams.insert(key, task.abort_handle());
3673        Ok(Vec::new())
3674    }
3675
3676    async fn handle_activity_record(
3677        &self,
3678        ctx: &RouteCtx,
3679        frame: Frame,
3680    ) -> Result<Vec<Frame>, RouterError> {
3681        use subc_control::{
3682            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3683            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3684        };
3685        let result = async {
3686            let module = self
3687                .forwarding
3688                .module_id_for_connection(ctx.connection_id)
3689                .ok()
3690                .flatten()
3691                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3692            let request: ActivityRecordRequest =
3693                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3694            crate::activity::validate(&module, &request)?;
3695            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3696            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3697            let at_ms = std::time::SystemTime::now()
3698                .duration_since(std::time::UNIX_EPOCH)
3699                .unwrap_or_default()
3700                .as_millis() as u64;
3701            ledger.record(module, request, actor, at_ms).await
3702        }
3703        .await;
3704        match result {
3705            Ok(entry) => Ok(vec![control_response_body_frame(
3706                &frame,
3707                &ActivityRecordResponse { seq: entry.seq },
3708                "activity.record",
3709            )?]),
3710            Err(code) => Ok(vec![control_error_frame(
3711                &frame,
3712                code,
3713                "activity recording refused",
3714            )?]),
3715        }
3716    }
3717
3718    fn activity_actor(
3719        &self,
3720        connection: ConnectionId,
3721        module: &str,
3722        on: Option<&subc_control::ActivityOn>,
3723    ) -> Result<subc_control::ActivityActor, &'static str> {
3724        use crate::forwarding::{DataRoute, DataRouteState};
3725        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3726        let principal = |id: &str| Principal::Reserved {
3727            module_id: id.to_owned(),
3728        };
3729        let mut actor = ActivityActor {
3730            principal: principal(module),
3731            scope_owner: None,
3732            agent_id: None,
3733            run_id: None,
3734            flow_id: None,
3735            project_root: None,
3736        };
3737        let scope = match on {
3738            None => return Ok(actor),
3739            Some(ActivityOn::Scope(scope)) => {
3740                let selector = subc_protocol::scope::ScopeSelector::new(
3741                    principal(module),
3742                    scope.scope_ref.clone(),
3743                    Some(scope.epoch),
3744                );
3745                self.scopes
3746                    .read()
3747                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3748                    .attribute(&principal(module), module, &selector, true)
3749                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3750            }
3751            Some(ActivityOn::Route(route)) => {
3752                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3753                    .forwarding
3754                    .lookup_data_route(connection, route.channel, route.epoch)
3755                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3756                else {
3757                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3758                };
3759                if binding.module_id != module {
3760                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3761                }
3762                actor.principal = binding.principal.clone();
3763                actor.project_root = binding
3764                    .project_root
3765                    .as_ref()
3766                    .map(|root| root.as_path().to_path_buf());
3767                let Some(bound) = &binding.scope else {
3768                    return Ok(actor);
3769                };
3770                let owner = principal(&bound.owner);
3771                let selector = subc_protocol::scope::ScopeSelector::new(
3772                    owner.clone(),
3773                    bound.scope_ref.clone(),
3774                    Some(bound.tag.scope_epoch),
3775                );
3776                let admitted = self
3777                    .scopes
3778                    .read()
3779                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3780                    .attribute(&owner, module, &selector, true)
3781                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3782                if admitted.tag != bound.tag {
3783                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3784                }
3785                admitted
3786            }
3787        };
3788        actor.scope_owner = Some(scope.stamp.owner);
3789        actor.agent_id = scope.stamp.attributes.agent_id;
3790        actor.run_id = scope.stamp.attributes.run_id;
3791        actor.flow_id = scope.stamp.attributes.flow_id;
3792        Ok(actor)
3793    }
3794
3795    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3796        self.refresh_capability_requirements();
3797        // A bare connection count is ambiguous between many clients holding a
3798        // route each and one client accumulating hundreds, so publish the
3799        // concentration alongside it. Route state is best-effort here: a
3800        // diagnostic endpoint must still answer if the forwarding lock is
3801        // contended.
3802        let mut counters = self.counters.snapshot();
3803        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3804            obj.insert(
3805                "activity_unparseable_lines".into(),
3806                ledger.read_errors().into(),
3807            );
3808            obj.insert(
3809                "activity_lifecycle_append_failures".into(),
3810                ledger.lifecycle_failures().into(),
3811            );
3812            obj.insert(
3813                "activity_started_digest_skips".into(),
3814                ledger.lifecycle_digest_skips().into(),
3815            );
3816        }
3817        if let (Ok((connections_with_routes, max)), Some(obj)) = (
3818            self.forwarding.client_route_concentration(),
3819            counters.as_object_mut(),
3820        ) {
3821            obj.insert(
3822                "client_connections_with_routes".into(),
3823                connections_with_routes.into(),
3824            );
3825            obj.insert("max_routes_on_one_connection".into(), max.into());
3826        }
3827        // A module that is being fast-refused and a module that is fine look
3828        // identical from a client that retries and succeeds, so name the open
3829        // breakers here. This rides the existing free-form counters object
3830        // rather than a new wire field, so no sibling that deserializes
3831        // `ServerDescribe` has to be rebuilt to keep reading it.
3832        if let (Some(open_breakers), Some(obj)) = (
3833            self.route_bind_breakers.open_snapshot(),
3834            counters.as_object_mut(),
3835        ) {
3836            obj.insert("route_bind_breakers_open".into(), open_breakers);
3837        }
3838        let response = ClientControlResponse::ServerDescribe {
3839            protocol_ver: PROTOCOL_VERSION,
3840            subc_ops: subc_ops(),
3841            capabilities: self.subc_capabilities.as_ref().to_vec(),
3842            connected_clients: self.connected_clients.count(),
3843            counters: Some(counters),
3844            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3845            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3846            capability_requirements: self.capability_requirement_statuses(),
3847            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3848        };
3849        Ok(vec![control_response_body_frame(
3850            &frame,
3851            &response,
3852            "ClientControlResponse::ServerDescribe",
3853        )?])
3854    }
3855
3856    fn handle_catalog_list(
3857        &self,
3858        frame: Frame,
3859        module_id: Option<String>,
3860    ) -> Result<Vec<Frame>, RouterError> {
3861        let (generation, modules) = self.registry.list_modules().map_err(|err| {
3862            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3863        })?;
3864        let entries = modules
3865            .into_iter()
3866            .filter(|registration| {
3867                module_id
3868                    .as_deref()
3869                    .map(|wanted| registration.manifest.module_id == wanted)
3870                    .unwrap_or(true)
3871            })
3872            .map(|registration| {
3873                let not_ready = self.not_ready_reason(&registration);
3874                let roles = registration.manifest.provides;
3875                CatalogEntry::new(
3876                    registration.manifest.module_id,
3877                    roles,
3878                    registration.control_ops,
3879                )
3880                .with_ready(not_ready.is_none())
3881                .with_not_ready(not_ready)
3882                .with_module_version(Some(registration.manifest.module_version))
3883                .with_capabilities(registration.manifest.capabilities)
3884                .with_self_signals(registration.manifest.self_signals)
3885                .with_events(registration.manifest.events)
3886                .with_operation_delivery(registration.manifest.operation_delivery)
3887            })
3888            .collect();
3889        let response = ClientControlResponse::CatalogList {
3890            generation,
3891            modules: entries,
3892            subc_ops: subc_ops(),
3893        };
3894        Ok(vec![control_response_body_frame(
3895            &frame,
3896            &response,
3897            "ClientControlResponse::CatalogList",
3898        )?])
3899    }
3900
3901    fn route_open_principal(
3902        &self,
3903        frame: &Frame,
3904        consumer_identity: Option<ConsumerIdentity>,
3905    ) -> Result<Result<Principal, Frame>, RouterError> {
3906        let Some(consumer_identity) = consumer_identity else {
3907            return Ok(Ok(Principal::Direct));
3908        };
3909
3910        if self.supervisor.spawned_consumer_authorized(
3911            &consumer_identity.module_id,
3912            &consumer_identity.launch_nonce,
3913        ) {
3914            return Ok(Ok(Principal::Reserved {
3915                module_id: consumer_identity.module_id,
3916            }));
3917        }
3918
3919        Ok(Err(control_error_frame(
3920            frame,
3921            "bad_consumer_identity",
3922            format!(
3923                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3924                consumer_identity.module_id
3925            ),
3926        )?))
3927    }
3928
3929    /// Ordinary `route.open` refusals go through here; admission and breaker
3930    /// refusals log separately with their capacity or breaker state. The daemon can
3931    /// attest which code it sent: without the event, a client's "the daemon
3932    /// refused me" and the daemon's own view could only be reconciled by
3933    /// argument. Malformed input (`invalid_project_root`) does not come here;
3934    /// rejecting a request that was never a valid open is not a refusal of one.
3935    fn route_open_refusal_frame(
3936        &self,
3937        ctx: &RouteCtx,
3938        frame: &Frame,
3939        module_id: &str,
3940        reason: &'static str,
3941        code: &'static str,
3942        message: impl Into<String>,
3943    ) -> Result<Frame, RouterError> {
3944        self.observe_route_open_refusal(ctx, module_id, reason, code);
3945        control_error_frame(frame, code, message.into())
3946    }
3947
3948    /// Refuse a `route.open` because the target module's bind-relay breaker is
3949    /// open, without attempting the relay.
3950    ///
3951    /// The wire code is `module_timeout`, which is the truth (the module has
3952    /// not been answering binds) and which both SDKs already classify as
3953    /// retryable with capped backoff. Reusing it is what keeps this change out
3954    /// of both SDKs; the daemon-side distinction lives in the counter key
3955    /// instead.
3956    ///
3957    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
3958    /// While a breaker is open this fires on every open to that module, and the
3959    /// stall written up in `docs/designs/route-open-head-of-line.md` already
3960    /// produced 261 lines about a single module inside 3000 lines of daemon
3961    /// log. The rare transitions are logged at warn/info instead and the volume
3962    /// is carried by the counter, so the evidence survives without the flood.
3963    /// The debug line keeps a per-refusal record reachable for whoever turns
3964    /// the level up.
3965    fn route_open_breaker_refusal_frame(
3966        &self,
3967        ctx: &RouteCtx,
3968        frame: &Frame,
3969        module_id: &str,
3970        consecutive_timeouts: u32,
3971        retry_in: Duration,
3972        probe_in_flight: bool,
3973    ) -> Result<Frame, RouterError> {
3974        self.counters
3975            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3976        debug!(
3977            target: "control",
3978            code = "module_timeout",
3979            module_id = ?module_id,
3980            connection_id = ctx.connection_id.get(),
3981            consecutive_timeouts,
3982            retry_in_ms = retry_in.as_millis() as u64,
3983            probe_in_flight,
3984            "route.open refused by open bind-relay breaker"
3985        );
3986        // Say what a caller can act on. An open bind-relay breaker means the
3987        // module timed out accepting several new routes in a row. The module
3988        // is still running and its established routes keep working; only new
3989        // route.open requests are refused until the cooldown ends and one
3990        // test route (the probe) gets through. A message that only counts
3991        // failed relays reads as "the module is down" to a worker that sees it.
3992        let detail = if probe_in_flight {
3993            "one test route is already being tried; retry once it settles".to_string()
3994        } else {
3995            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3996        };
3997        control_error_frame(
3998            frame,
3999            "module_timeout",
4000            format!(
4001                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4002                 timed out in a row); {detail}; its established routes are unaffected"
4003            ),
4004        )
4005    }
4006
4007    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
4008    /// requester's bytes (an unknown target is whatever the client sent) and
4009    /// is Debug-formatted so control characters land in the log escaped
4010    /// rather than as terminal sequences for whoever tails it.
4011    ///
4012    /// `reason` names the check that refused, because one wire code has
4013    /// several senders: after a module registers, `target_unavailable` can
4014    /// come from a missing role, an inactive registration, a supervisor that
4015    /// has not marked the process live, a missing forwarding connection, or a
4016    /// failed relay, and a log that records only the code cannot say which of
4017    /// them fired. It is a static, daemon-chosen label per branch, so it is
4018    /// safe to print plainly and stays a closed set.
4019    fn observe_route_open_refusal(
4020        &self,
4021        ctx: &RouteCtx,
4022        module_id: &str,
4023        reason: &'static str,
4024        code: &'static str,
4025    ) {
4026        self.counters.increment_route_open_refused(code);
4027        info!(
4028            target: "control",
4029            code,
4030            reason,
4031            module_id = ?module_id,
4032            connection_id = ctx.connection_id.get(),
4033            "route.open refused"
4034        );
4035        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4036            self.route_outages.record_not_serving(module_id, reason);
4037        }
4038    }
4039
4040    /// Record an ACCEPTED route.open.
4041    ///
4042    /// Refusals have been logged and counted since the attestation work; accepts
4043    /// were invisible, so the daemon knew every principal it stamped and wrote
4044    /// none of them down. The party that attests the identity was the only party
4045    /// not recording it, which left a credential vault unable to name the sender
4046    /// of a call that reached it (claustrum #43) and left the launch-nonce
4047    /// concurrency question unanswerable from the outside.
4048    ///
4049    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
4050    /// `code`/`module_id`/`connection_id` returns both directions of the same
4051    /// decision rather than two shapes a reader has to join by hand.
4052    ///
4053    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
4054    /// and the difference carries information rather than being an
4055    /// inconsistency. This line is only reachable after a successful bind to a
4056    /// REGISTERED module, so the value has already passed HELLO validation
4057    /// including the path-hazard refusal and cannot contain control bytes. A
4058    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
4059    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
4060    ///
4061    /// Bare is also what every other daemon line already emits (`module
4062    /// registered`, `configured module supervised`). Shipping `?module_id` here
4063    /// made this instrument the only one in the file whose ids did not answer
4064    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
4065    /// line whose whole purpose is being grepped beside its sibling.
4066    ///
4067    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
4068    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
4069    /// forwards every field type through it and a bare `&str` and a `?`-escaped
4070    /// one are recorded identically. A test written against that harness passes
4071    /// either way -- I wrote one, measured it, and deleted it rather than ship a
4072    /// green assertion that cannot fail. The same limit applies to the escaping
4073    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
4074    /// it reads as a guard on the Debug escaping and cannot detect its removal.
4075    /// Fencing either needs the real formatter, not the capture layer.
4076    ///
4077    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
4078    /// unix-socket options and subc is loopback TCP, so there is no peer identity
4079    /// to record. The ephemeral port would decay within minutes and answer only a
4080    /// live question. The identity question is instead answered by counting
4081    /// distinct live connections presenting one module's `consumer_identity` --
4082    /// "is anyone else holding this secret" rather than "is this the right
4083    /// process".
4084    fn observe_route_open_accept(
4085        &self,
4086        ctx: &RouteCtx,
4087        module_id: &str,
4088        principal: &str,
4089        scope: &str,
4090    ) {
4091        self.route_outages.record_accepted(module_id);
4092        self.counters.increment_route_open_accepted(principal);
4093        info!(
4094            target: "control",
4095            principal,
4096            module_id,
4097            connection_id = ctx.connection_id.get(),
4098            scope = %scope,
4099            "route.open accepted"
4100        );
4101    }
4102
4103    fn supervised_absent_route_open_refusal_frame(
4104        &self,
4105        ctx: &RouteCtx,
4106        frame: &Frame,
4107        module_id: &str,
4108        code: &'static str,
4109        status: &crate::supervise::ModuleStatus,
4110    ) -> Result<Frame, RouterError> {
4111        self.counters.increment_route_open_refused(code);
4112        info!(
4113            target: "control",
4114            code,
4115            reason = "supervised_not_registered",
4116            module_id = ?module_id,
4117            connection_id = ctx.connection_id.get(),
4118            state = %status.state,
4119            enabled = status.enabled,
4120            live = status.live,
4121            "route.open refused"
4122        );
4123        // A supervised module whose process has not registered is not
4124        // serving, whatever the reason; the supervisor knows this id, so it is
4125        // safe to track.
4126        self.route_outages
4127            .record_not_serving(module_id, "supervised_not_registered");
4128        let message = if let Some(pid) = status.waiting_for_previous_pid {
4129            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4130                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4131            })
4132        } else {
4133            format!(
4134                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4135                status.state, status.enabled, status.live
4136            )
4137        };
4138        control_error_frame(frame, code, message)
4139    }
4140
4141    async fn handle_route_open(
4142        &self,
4143        ctx: &RouteCtx,
4144        frame: Frame,
4145        request: RouteOpenRequest,
4146        timings: &mut RouteOpenTimings,
4147    ) -> Result<Vec<Frame>, RouterError> {
4148        let RouteOpenRequest {
4149            target,
4150            mut identity,
4151            consumer_identity,
4152            consumer_capabilities,
4153            role_versions,
4154            admission_facts,
4155            scope,
4156        } = request;
4157        let target_module_id = target_module_id(&target).to_string();
4158        timings.module_id.clone_from(&target_module_id);
4159        if self
4160            .registry
4161            .get_module_by_connection(ctx.connection_id)
4162            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4163            .is_some()
4164        {
4165            return Ok(vec![control_error_frame(
4166                &frame,
4167                "invalid_request",
4168                "module connections cannot open client routes",
4169            )?]);
4170        }
4171        debug!(
4172            connection_id = ctx.connection_id.get(),
4173            corr = frame.header.corr,
4174            module_id = %target_module_id,
4175            "handling route.open"
4176        );
4177
4178        // A malformed declaration is refused first, before anything about the
4179        // target is looked up: the same body would be refused against any
4180        // module, so the caller learns nothing by retrying or waiting. An empty
4181        // map declares nothing and travels as no field at all, so a provider
4182        // only ever sees a missing field or a non-empty one.
4183        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4184        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4185            self.observe_route_open_refusal(
4186                ctx,
4187                &target_module_id,
4188                "invalid_role_versions",
4189                error_codes::INVALID_REQUEST,
4190            );
4191            return Ok(vec![control_error_body_frame(
4192                &frame,
4193                ErrorBody {
4194                    code: error_codes::INVALID_REQUEST.to_string(),
4195                    message: error.to_string(),
4196                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4197                },
4198            )?]);
4199        }
4200
4201        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4202            if status.waiting_for_previous_pid.is_some() {
4203                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4204                    ctx,
4205                    &frame,
4206                    &target_module_id,
4207                    "module_warming",
4208                    &status,
4209                )?]);
4210            }
4211        }
4212
4213        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4214        // opposite. Below, a caller learns whether a module is unregistered,
4215        // supervised-but-down (with state/enabled/live), or registered without the
4216        // requested role. Elsewhere that is an enumeration leak: a probe learning
4217        // the shape of a fleet it cannot otherwise see.
4218        //
4219        // It is not one here, and the reason is the ACCESS MODEL rather than
4220        // anything about these errors. Reaching route.open requires the
4221        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4222        // connection file, so any caller who completes it already runs as this
4223        // user -- and can read subc.jsonc for the module list and `ck module
4224        // status` for live state. The reply discloses nothing the caller cannot
4225        // read more easily from disk, while the precision is load-bearing:
4226        // `unknown_module` is retryable and a missing role is not.
4227        //
4228        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4229        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4230        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4231        let Some(registration) = self
4232            .registry
4233            .get_module(&target_module_id)
4234            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4235        else {
4236            if let Some((status, warming)) =
4237                self.supervisor_status(&target_module_id, frame.header.corr)?
4238            {
4239                // BEFORE the two availability codes below, because for a module
4240                // that speaks no subc wire both of them are false comfort: they
4241                // say "not right now" and are retried, and this module will
4242                // never register no matter how long the caller waits. The
4243                // absence here is the declaration being honoured, not a module
4244                // that is late.
4245                if status.protocol == ModuleProtocol::None {
4246                    return Ok(vec![self.route_open_refusal_frame(
4247                        ctx,
4248                        &frame,
4249                        &target_module_id,
4250                        "protocol_none",
4251                        error_codes::MODULE_NO_PROTOCOL,
4252                        format!(
4253                            "module_id '{target_module_id}' is declared protocol: none; \
4254                             it speaks no subc wire and serves no routes"
4255                        ),
4256                    )?]);
4257                }
4258                let code = if warming {
4259                    "module_warming"
4260                } else {
4261                    "target_unavailable"
4262                };
4263                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4264                    ctx,
4265                    &frame,
4266                    &target_module_id,
4267                    code,
4268                    &status,
4269                )?]);
4270            }
4271            if let Some(removed_ago_ms) =
4272                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4273            {
4274                return Ok(vec![self.route_open_refusal_frame(
4275                    ctx,
4276                    &frame,
4277                    &target_module_id,
4278                    "removed",
4279                    error_codes::MODULE_REMOVED,
4280                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4281                )?]);
4282            }
4283            return Ok(vec![self.route_open_refusal_frame(
4284                ctx,
4285                &frame,
4286                &target_module_id,
4287                "not_registered",
4288                error_codes::UNKNOWN_MODULE,
4289                format!("module_id '{target_module_id}' is not registered"),
4290            )?]);
4291        };
4292
4293        // Best-effort only: registry readiness and forwarding reservation use
4294        // different locks, so a module can flip readiness between this read and
4295        // the relay. Modules must still tolerate an `on_bind` while not ready.
4296        if !registration.ready {
4297            self.counters
4298                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4299            info!(
4300                target: "control",
4301                code = error_codes::MODULE_WARMING,
4302                module_id = ?target_module_id,
4303                connection_id = ctx.connection_id.get(),
4304                reason = "declared_not_ready",
4305                "route.open refused"
4306            );
4307            // The module is registered but says it cannot take work, which is
4308            // an outage from the caller's side even though its process is up.
4309            self.route_outages
4310                .record_not_serving(&target_module_id, "declared_not_ready");
4311            return Ok(vec![control_error_body_frame(
4312                &frame,
4313                ErrorBody {
4314                    code: error_codes::MODULE_WARMING.to_string(),
4315                    message: format!(
4316                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4317                    ),
4318                    detail: Some(serde_json::json!({
4319                        "reason": "declared_not_ready"
4320                    })),
4321                },
4322            )?]);
4323        }
4324
4325        // Effective readiness, second half: a module that declares a capability
4326        // `need: required` is not routable while that capability has no
4327        // registered provider. It is enforced HERE, as a retryable routing
4328        // refusal, and deliberately not as spawn ordering or a boot block. The
4329        // module is still started and registered and can make its own calls;
4330        // spawn ordering is a promise that cannot be kept once a provider
4331        // crashes at runtime, and refusing to boot would stop the whole
4332        // machine, including the tools needed to fix its configuration.
4333        //
4334        // "Provided" is the evaluator's verdict, which counts a provider as
4335        // soon as it has REGISTERED, not once it is ready. Two modules that
4336        // require each other's capabilities are therefore both routable once
4337        // both register; counting readiness instead would deadlock them.
4338        //
4339        // Only new opens are refused. Routes already bound when a provider
4340        // goes away stay bound: nothing here tears them down, and the module
4341        // answers them as it can. Like the readiness read above this is
4342        // best-effort against a provider registering or leaving concurrently.
4343        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
4344            self.counters
4345                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4346            info!(
4347                target: "control",
4348                code = error_codes::MODULE_WARMING,
4349                module_id = ?target_module_id,
4350                connection_id = ctx.connection_id.get(),
4351                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4352                capability = %capability,
4353                "route.open refused"
4354            );
4355            return Ok(vec![control_error_body_frame(
4356                &frame,
4357                ErrorBody {
4358                    code: error_codes::MODULE_WARMING.to_string(),
4359                    message: format!(
4360                        "module_id '{target_module_id}' requires capability '{capability}', \
4361                         which no registered module provides; retry"
4362                    ),
4363                    detail: Some(serde_json::json!({
4364                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4365                        "capability": capability,
4366                    })),
4367                },
4368            )?]);
4369        }
4370
4371        if !target_has_required_role(&target, &registration.manifest.provides) {
4372            return Ok(vec![self.route_open_refusal_frame(
4373                ctx,
4374                &frame,
4375                &target_module_id,
4376                "role_not_provided",
4377                "target_unavailable",
4378                format!("module_id '{target_module_id}' does not provide the requested target"),
4379            )?]);
4380        }
4381
4382        if registration.state != ChannelState::Active {
4383            return Ok(vec![self.route_open_refusal_frame(
4384                ctx,
4385                &frame,
4386                &target_module_id,
4387                "registration_not_active",
4388                "target_unavailable",
4389                format!("module_id '{target_module_id}' is not active"),
4390            )?]);
4391        }
4392
4393        if self
4394            .forwarding
4395            .module_is_draining(&target_module_id)
4396            .map_err(RouterError::Forwarding)?
4397        {
4398            return Ok(vec![self.route_open_refusal_frame(
4399                ctx,
4400                &frame,
4401                &target_module_id,
4402                "reloading",
4403                "module_reloading",
4404                format!("module_id '{target_module_id}' is reloading"),
4405            )?]);
4406        }
4407
4408        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4409            process_liveness.process_live(&target_module_id) == Some(false)
4410        }) {
4411            // A module the supervisor is restarting or reloading can still hold
4412            // a registration: the old process before its connection closes, or
4413            // a new one that registered while the supervisor was draining. The
4414            // forwarding table does not see that as draining, but the consumer
4415            // should still be told to retry soon, exactly as for the drain
4416            // above, rather than that the target is unavailable.
4417            if process_liveness.process_replacing(&target_module_id) {
4418                return Ok(vec![self.route_open_refusal_frame(
4419                    ctx,
4420                    &frame,
4421                    &target_module_id,
4422                    "reloading",
4423                    "module_reloading",
4424                    format!("module_id '{target_module_id}' is reloading"),
4425                )?]);
4426            }
4427            return Ok(vec![self.route_open_refusal_frame(
4428                ctx,
4429                &frame,
4430                &target_module_id,
4431                "supervisor_not_live",
4432                "target_unavailable",
4433                format!("module_id '{target_module_id}' is not live"),
4434            )?]);
4435        }
4436
4437        if !self
4438            .forwarding
4439            .has_live_module_connection(&target_module_id)
4440            .map_err(RouterError::Forwarding)?
4441        {
4442            return Ok(vec![self.route_open_refusal_frame(
4443                ctx,
4444                &frame,
4445                &target_module_id,
4446                "no_forwarding_connection",
4447                "target_unavailable",
4448                format!("module_id '{target_module_id}' has no live forwarding connection"),
4449            )?]);
4450        }
4451
4452        if let Some(error) =
4453            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4454        {
4455            self.observe_route_open_refusal(
4456                ctx,
4457                &target_module_id,
4458                "op_not_allowed",
4459                "op_not_allowed",
4460            );
4461            return Ok(vec![error]);
4462        }
4463
4464        let principal = match self.route_open_principal(&frame, consumer_identity)? {
4465            Ok(principal) => principal,
4466            Err(error) => {
4467                self.observe_route_open_refusal(
4468                    ctx,
4469                    &target_module_id,
4470                    "bad_consumer_identity",
4471                    "bad_consumer_identity",
4472                );
4473                return Ok(vec![error]);
4474            }
4475        };
4476        timings.principal = principal_label(&principal);
4477
4478        // This is attested, control-plane policy for supervised module origins.
4479        // Keep it before route reservation and out of the opaque forwarding hot
4480        // path: data frames must never acquire a per-frame capability check.
4481        if let Principal::Reserved {
4482            module_id: opening_module_id,
4483        } = &principal
4484        {
4485            if let Some(opening_registration) = self
4486                .registry
4487                .get_module(opening_module_id)
4488                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4489            {
4490                if let Some(capability) =
4491                    denied_capability(&opening_registration.manifest, &registration.manifest)
4492                {
4493                    warn!(
4494                        opening_module_id,
4495                        target_module_id,
4496                        capability,
4497                        "refusing route.open because an attested capability deny edge matches"
4498                    );
4499                    return Ok(vec![self.route_open_refusal_frame(
4500                        ctx,
4501                        &frame,
4502                        &target_module_id,
4503                        "capability_deny_edge",
4504                        "capability_forbidden",
4505                        format!(
4506                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4507                        ),
4508                    )?]);
4509                }
4510            }
4511        }
4512
4513        if admission_facts.is_some() {
4514            let carrier_matches = matches!(
4515                &principal,
4516                Principal::Reserved { module_id }
4517                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4518            );
4519            if !carrier_matches {
4520                return Ok(vec![self.route_open_refusal_frame(
4521                    ctx,
4522                    &frame,
4523                    &target_module_id,
4524                    "admission_facts_carrier_not_permitted",
4525                    "admission_facts_not_permitted",
4526                    "admission facts may only be carried by the configured reserved module",
4527                )?]);
4528            }
4529
4530            let target_allowed = self
4531                .admission_facts_targets
4532                .as_ref()
4533                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4534            if !target_allowed {
4535                return Ok(vec![self.route_open_refusal_frame(
4536                    ctx,
4537                    &frame,
4538                    &target_module_id,
4539                    "admission_facts_target_not_listed",
4540                    "admission_facts_target_not_allowed",
4541                    format!(
4542                        "admission facts are not permitted for target module_id '{target_module_id}'"
4543                    ),
4544                )?]);
4545            }
4546
4547            // Keep the value opaque to subc. The downstream admission validator owns
4548            // schema and semantic checks; this daemon only enforces carrier authority
4549            // and the configured destination allowlist.
4550        }
4551
4552        // Scope admission, on the attested principal above and never on the
4553        // request body. The tag read here travels with the pending bind and is
4554        // compared with the published one at commit, so a sync between here
4555        // and the module's ack refuses the open instead of binding a stamp
4556        // that is no longer true.
4557        let (bound_scope, mut scope_stamp) = match &scope {
4558            None => (None, None),
4559            Some(selector) => {
4560                let owner_configured = match &selector.owner {
4561                    // A reserved owner counts as configured from before its
4562                    // process is spawned (see `SupervisorHandle::is_configured`).
4563                    // So an owner that has not synced its scopes yet is refused
4564                    // as retryable (`scope_not_synced`), not as one that will
4565                    // never sync.
4566                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4567                    _ => false,
4568                };
4569                let mut scopes = self
4570                    .scopes
4571                    .write()
4572                    .unwrap_or_else(|poisoned| poisoned.into_inner());
4573                let admitted = scopes
4574                    .admit(&principal, &target_module_id, selector, owner_configured)
4575                    .and_then(|admission| {
4576                        crate::scopes::check_target_flow_support(
4577                            &admission.stamp,
4578                            &target_module_id,
4579                            registration.manifest.capabilities.as_ref(),
4580                        )?;
4581                        crate::scopes::check_target_agent_run_support(
4582                            &admission.stamp,
4583                            &target_module_id,
4584                            registration.manifest.capabilities.as_ref(),
4585                        )?;
4586                        let bound = scopes.reserve_bind(
4587                            &admission,
4588                            &principal,
4589                            ctx.connection_id,
4590                            selector.hold_presence,
4591                        );
4592                        Ok((admission, bound))
4593                    });
4594                match admitted {
4595                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4596                    Err(refusal) => {
4597                        return Ok(vec![self.route_open_refusal_frame(
4598                            ctx,
4599                            &frame,
4600                            &target_module_id,
4601                            refusal.code,
4602                            refusal.code,
4603                            refusal.message,
4604                        )?]);
4605                    }
4606                }
4607            }
4608        };
4609
4610        // Bind admits a root that no longer exists on disk, because refusing here
4611        // closes the only exit from a paused run: cancel needs a bound route, and a
4612        // renamed or reclaimed directory makes that route unopenable forever. The
4613        // run itself is intact and still addressable by its recorded identity.
4614        //
4615        // This does NOT relax the rule the strict constructor protects. That rule is
4616        // that no root is ever aliased into NEW durable state -- a missing component
4617        // can reappear as a symlink elsewhere, which would move the identity and
4618        // split a session's history across two of them. The engine now refuses the
4619        // two operations that create such state (send and import) at admission,
4620        // which is a narrower way to hold the same invariant: reads and terminations
4621        // are admitted, writes are not. That refusal had to ship before this line
4622        // changed, or there is an interval where a send commits under a provisional
4623        // identity -- the exact failure the original policy existed to prevent.
4624        //
4625        // Resolution follows realpath rather than lexical cleanup: the longest
4626        // existing ancestor is canonicalized and the missing tail re-appended, so a
4627        // live root is unchanged and a vanished leaf keeps the identity it was
4628        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
4629        // same caller the moment the directory vanished, which strands the run more
4630        // quietly than refusing it.
4631        timings.enter(RouteOpenPhase::ProjectRoot);
4632        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4633            Ok(project_root) => project_root,
4634            Err(err) => {
4635                return Ok(vec![control_error_frame(
4636                    &frame,
4637                    "invalid_project_root",
4638                    err.to_string(),
4639                )?])
4640            }
4641        };
4642        timings.enter(RouteOpenPhase::Admission);
4643        identity.project_root = project_root.as_path().to_path_buf();
4644
4645        // Last gate before any relay work, and deliberately after the cheap
4646        // registry and availability checks above: those name a more precise
4647        // condition (unknown, removed, reloading) and a caller is better served
4648        // by the precise code than by this one.
4649        //
4650        // Everything below this point costs an egress permit, a reserved handle
4651        // pair and, if the module does not answer, the whole relay budget. The
4652        // reader no longer waits for that budget, so cap each target explicitly;
4653        // serial dispatch used to provide the accidental cap of one relay per
4654        // connection. Admission is a mutex-protected count and never waits.
4655        let _concurrency_guard = match self
4656            .route_bind_concurrency
4657            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4658        {
4659            Ok(guard) => guard,
4660            Err(in_flight) => {
4661                return Ok(vec![self.route_open_target_capacity_refusal(
4662                    ctx,
4663                    &frame,
4664                    &target_module_id,
4665                    in_flight,
4666                )?]);
4667            }
4668        };
4669
4670        // A module that has already burned the whole budget `threshold` times
4671        // in a row does not get to charge it again until a probe says it recovered.
4672        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4673            RouteBindAdmission::Admitted { guard, probe } => {
4674                if probe {
4675                    info!(
4676                        module_id = %target_module_id,
4677                        connection_id = ctx.connection_id.get(),
4678                        "route.bind breaker half-open: admitting one probe"
4679                    );
4680                }
4681                guard
4682            }
4683            RouteBindAdmission::Refused {
4684                consecutive_timeouts,
4685                retry_in,
4686                probe_in_flight,
4687            } => {
4688                return Ok(vec![self.route_open_breaker_refusal_frame(
4689                    ctx,
4690                    &frame,
4691                    &target_module_id,
4692                    consecutive_timeouts,
4693                    retry_in,
4694                    probe_in_flight,
4695                )?]);
4696            }
4697        };
4698
4699        // Resolve the per-module budget here so the wait matches the operator's
4700        // intent for this specific target. A per-module override in
4701        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
4702        // daemons) wins over the daemon-wide default.
4703        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4704        let relay_deadline = Instant::now() + route_bind_relay_timeout;
4705        timings.enter(RouteOpenPhase::EgressReservation);
4706        let pending = match self
4707            .forwarding
4708            .begin_route_bind_relay_for(
4709                ctx.connection_id,
4710                ctx.egress.clone(),
4711                response_version(&frame),
4712                frame.header.corr,
4713                &target_module_id,
4714                principal.clone(),
4715                bound_scope,
4716                Some(project_root),
4717                relay_deadline,
4718            )
4719            .await
4720        {
4721            Ok(pending) => pending,
4722            Err(err) => {
4723                return Ok(vec![self.route_open_refusal_frame(
4724                    ctx,
4725                    &frame,
4726                    &target_module_id,
4727                    "relay_reservation_failed",
4728                    forwarding_error_code(&err),
4729                    err.to_string(),
4730                )?])
4731            }
4732        };
4733        timings.enter(RouteOpenPhase::Admission);
4734        let crate::forwarding::PendingRouteBindRelay {
4735            endpoint,
4736            module_sink,
4737            negotiated_ver,
4738            client_channel,
4739            client_epoch,
4740            module_channel,
4741            module_epoch,
4742            corr: relay_corr,
4743            receiver,
4744        } = pending;
4745        let mut reservation =
4746            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4747
4748        // Reserving egress can wait while a module reconnects or a swap cuts
4749        // over. Check the relay's captured connection, not the earlier by-id
4750        // lookup: the original module's flow/run capabilities cannot authorize
4751        // its replacement. The captured sink stays bound to that connection.
4752        let relay_registration = self
4753            .registry
4754            .get_module_by_connection(endpoint.connection_id)
4755            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4756        let relay_capabilities = relay_registration
4757            .as_ref()
4758            .and_then(|registration| registration.manifest.capabilities.as_ref());
4759        if let Some(selector) = &scope {
4760            // Egress reservation may wait across a scope update. Recheck the
4761            // proof against the current record before sending the captured bind.
4762            if let Err(refusal) = self
4763                .scopes
4764                .read()
4765                .unwrap_or_else(|poisoned| poisoned.into_inner())
4766                .check_bind_proof(&principal, selector)
4767            {
4768                reservation.release_and_disarm();
4769                return Ok(vec![self.route_open_refusal_frame(
4770                    ctx,
4771                    &frame,
4772                    &target_module_id,
4773                    refusal.code,
4774                    refusal.code,
4775                    refusal.message,
4776                )?]);
4777            }
4778        }
4779        if let Some(stamp) = scope_stamp
4780            .as_ref()
4781            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4782        {
4783            if let Err(refusal) = crate::scopes::check_target_flow_support(
4784                stamp,
4785                &target_module_id,
4786                relay_capabilities,
4787            )
4788            .and_then(|()| {
4789                crate::scopes::check_target_agent_run_support(
4790                    stamp,
4791                    &target_module_id,
4792                    relay_capabilities,
4793                )
4794            }) {
4795                reservation.release_and_disarm();
4796                return Ok(vec![self.route_open_refusal_frame(
4797                    ctx,
4798                    &frame,
4799                    &target_module_id,
4800                    refusal.code,
4801                    refusal.code,
4802                    refusal.message,
4803                )?]);
4804            }
4805        }
4806
4807        // Older readers may reject unknown fields. Select optional fields from
4808        // the captured endpoint's declaration, including on unscoped binds.
4809        if !relay_capabilities.is_some_and(|caps| {
4810            caps.provides
4811                .iter()
4812                .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4813        }) {
4814            if let Some(stamp) = scope_stamp.as_mut() {
4815                stamp.attributes.on_behalf_of_agent = None;
4816            }
4817        }
4818        let call_mode = if relay_capabilities
4819            .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4820        {
4821            Some(derive_call_mode(scope_stamp.as_ref()))
4822        } else {
4823            if let Some(stamp) = scope_stamp.as_mut() {
4824                stamp.expires_at_ms = None;
4825            }
4826            None
4827        };
4828
4829        debug!(
4830            connection_id = ctx.connection_id.get(),
4831            client_channel,
4832            client_epoch,
4833            module_channel,
4834            module_epoch,
4835            "reserved route handle pair"
4836        );
4837        // Rendered BEFORE the move into the relay, because the accept arm below
4838        // is where it is logged and the principal is gone by then.
4839        let route_principal_label = principal_label(&principal);
4840        let scope_label = scope_stamp
4841            .as_ref()
4842            .map_or_else(|| "none".to_string(), route_open_scope_label);
4843        let relay = ModuleControlRequest::RouteBind {
4844            route_channel: module_channel,
4845            epoch: module_epoch,
4846            target,
4847            identity,
4848            principal: Some(principal),
4849            consumer_capabilities,
4850            role_versions,
4851            admission_facts,
4852            scope: scope_stamp,
4853            call_mode,
4854        };
4855        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4856            RouterError::backend(
4857                0,
4858                frame.header.corr,
4859                format!("failed to encode route.bind request: {err}"),
4860            )
4861        })?;
4862        let relay_frame = Frame::build_with_version(
4863            negotiated_ver,
4864            FrameType::Request,
4865            control_flags(),
4866            0,
4867            0,
4868            relay_corr,
4869            relay_body,
4870        )
4871        .map_err(RouterError::FrameBuild)?;
4872
4873        timings.enter(RouteOpenPhase::BindSend);
4874        let sent = module_sink.send(relay_frame).await;
4875        timings.enter(RouteOpenPhase::Admission);
4876        if let Err(err) = sent {
4877            reservation.release_and_disarm();
4878            return Ok(vec![self.route_open_refusal_frame(
4879                ctx,
4880                &frame,
4881                &target_module_id,
4882                "relay_send_failed",
4883                "target_unavailable",
4884                err.to_string(),
4885            )?]);
4886        }
4887
4888        #[cfg(test)]
4889        if let Some(hold) = &self.route_bind_relay_sent_hold {
4890            // First wait: tell the test the bind is queued. Second wait: stay
4891            // here until the test has driven whatever it wants to happen
4892            // between queueing the bind and marking it enqueued.
4893            hold.wait().await;
4894            hold.wait().await;
4895        }
4896
4897        // A missing relay is not by itself an abandoned bind: the module may
4898        // already have acked it and the route may be committed. Only a relay
4899        // dropped without any module reply needs this GOODBYE; one a reply
4900        // settled has already had its GOODBYE decided by the reply path.
4901        if self
4902            .forwarding
4903            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4904            .map_err(RouterError::Forwarding)?
4905            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4906        {
4907            self.send_abandoned_route_bind_goodbye(
4908                &module_sink,
4909                negotiated_ver,
4910                module_channel,
4911                module_epoch,
4912            );
4913        }
4914
4915        timings.enter(RouteOpenPhase::BindAckWait);
4916        let outcome = timeout_at(relay_deadline, receiver).await;
4917        timings.enter(RouteOpenPhase::Admission);
4918        match outcome {
4919            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4920                reservation.disarm();
4921                if breaker.record_accepted() {
4922                    info!(
4923                        module_id = %target_module_id,
4924                        "route.bind breaker closed: the probe was accepted"
4925                    );
4926                }
4927                self.observe_route_open_accept(
4928                    ctx,
4929                    &target_module_id,
4930                    &route_principal_label,
4931                    &scope_label,
4932                );
4933                Ok(Vec::new())
4934            }
4935            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4936                reservation.release_and_disarm();
4937                // A module that says no in microseconds is healthy. Rejection
4938                // is a different condition with its own refusal and must not
4939                // move the breaker.
4940                breaker.record_inconclusive();
4941                // The daemon's own commit re-check refused the bind because the
4942                // scope ended or changed after admission. The module accepted;
4943                // counting it as a module rejection would blame the module.
4944                let scope_code = match body.code.as_str() {
4945                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4946                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4947                    _ => None,
4948                };
4949                if let Some(code) = scope_code {
4950                    self.observe_route_open_refusal(
4951                        ctx,
4952                        &target_module_id,
4953                        "scope_changed_before_commit",
4954                        code,
4955                    );
4956                    return Ok(vec![control_error_body_frame(&frame, body)?]);
4957                }
4958                self.counters
4959                    .increment_route_open_refused("module_rejected");
4960                info!(
4961                    target: "control",
4962                    code = "module_rejected",
4963                    module_code = ?body.code,
4964                    module_id = ?target_module_id,
4965                    connection_id = ctx.connection_id.get(),
4966                    "route.open refused"
4967                );
4968                Ok(vec![control_error_body_frame(&frame, body)?])
4969            }
4970            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4971                reservation.release_and_disarm();
4972                breaker.record_inconclusive();
4973                // Fires when the module's connection closes while a relayed
4974                // bind is pending -- typically a caller racing a module restart
4975                // whose bind was relayed BEFORE the drain mark went up. Logged
4976                // because the caller sees only its own error and the fleet has
4977                // already spent one diagnosis round unable to tell this arm
4978                // from a relay timeout without daemon-side evidence.
4979                tracing::warn!(
4980                    module_id = %target_module_id,
4981                    "route.bind relay abandoned: {message}"
4982                );
4983                Ok(vec![self.route_open_refusal_frame(
4984                    ctx,
4985                    &frame,
4986                    &target_module_id,
4987                    "relay_abandoned",
4988                    "target_unavailable",
4989                    message,
4990                )?])
4991            }
4992            Ok(Err(_)) => {
4993                reservation.release_and_disarm();
4994                breaker.record_inconclusive();
4995                Ok(vec![self.route_open_refusal_frame(
4996                    ctx,
4997                    &frame,
4998                    &target_module_id,
4999                    "relay_waiter_canceled",
5000                    "target_unavailable",
5001                    "route.bind relay waiter was canceled before the module responded",
5002                )?])
5003            }
5004            Err(_) => {
5005                reservation.release_and_disarm();
5006                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
5007                // answer at all is the one condition a fast refusal can
5008                // usefully stand in for; every other arm already answered.
5009                if let Some(opened) = breaker.record_timeout(
5010                    self.route_bind_breaker_threshold,
5011                    self.route_bind_breaker_cooldown,
5012                ) {
5013                    warn!(
5014                        module_id = %target_module_id,
5015                        consecutive_timeouts = opened.consecutive_timeouts,
5016                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5017                        reopened_after_probe = opened.reopened_after_probe,
5018                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5019                    );
5020                }
5021                // The generous budget just burned to no answer: the module is
5022                // registered and its connection is up, but its bind handler sat
5023                // on the ack for the full budget (warm-on-bind, cold configure,
5024                // or a wedged handler). Every earlier unavailability shape
5025                // fast-refuses BEFORE the relay, so this arm firing means the
5026                // slowness is module-side -- log it so the per-module timeline
5027                // is reconstructable without client audit rows.
5028                tracing::warn!(
5029                    module_id = %target_module_id,
5030                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5031                    "route.bind relay timed out: module did not ack within budget"
5032                );
5033                Ok(vec![self.route_open_refusal_frame(
5034                    ctx,
5035                    &frame,
5036                    &target_module_id,
5037                    "relay_timed_out",
5038                    "module_timeout",
5039                    format!(
5040                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
5041                        route_bind_relay_timeout
5042                    ),
5043                )?])
5044            }
5045        }
5046    }
5047
5048    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5049        let response = ClientControlResponse::SupervisorSpawnSnapshot {
5050            snapshot: self.supervisor.spawn_snapshot(),
5051        };
5052        Ok(vec![control_response_body_frame(
5053            &frame,
5054            &response,
5055            "ClientControlResponse::SupervisorSpawnSnapshot",
5056        )?])
5057    }
5058
5059    fn handle_supervisor_spawn_subscribe(
5060        &self,
5061        ctx: &RouteCtx,
5062        frame: Frame,
5063        since: Option<SpawnCursor>,
5064    ) -> Result<Vec<Frame>, RouterError> {
5065        match self.supervisor.subscribe_spawns(
5066            ctx.connection_id,
5067            frame.header.corr,
5068            response_version(&frame),
5069            since,
5070            ctx.egress.clone(),
5071        ) {
5072            Ok(()) => Ok(Vec::new()),
5073            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5074                Ok(vec![control_error_body_frame(
5075                    &frame,
5076                    ErrorBody {
5077                        code: "spawn_cursor_incarnation_mismatch".to_string(),
5078                        message: "spawn cursor belongs to a different daemon incarnation"
5079                            .to_string(),
5080                        detail: Some(serde_json::json!({
5081                            "current_daemon_incarnation": current
5082                        })),
5083                    },
5084                )?])
5085            }
5086            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5087                &frame,
5088                ErrorBody {
5089                    code: "spawn_cursor_too_old".to_string(),
5090                    message: "spawn cursor predates the retained event ring".to_string(),
5091                    detail: Some(serde_json::json!({
5092                        "oldest_retained_cursor": oldest
5093                    })),
5094                },
5095            )?]),
5096            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5097                0,
5098                frame.header.corr,
5099                format!("failed to open supervisor spawn subscription: {error}"),
5100            )),
5101        }
5102    }
5103
5104    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5105        let generation = self
5106            .registry
5107            .generation()
5108            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5109        let mut modules = Vec::new();
5110        for module in self.supervisor.list() {
5111            let status = module.status_for_control("list").map_err(|err| {
5112                RouterError::backend(
5113                    0,
5114                    frame.header.corr,
5115                    format!("failed to read supervisor status: {err}"),
5116                )
5117            })?;
5118            let (configured, _) = module.configuration().map_err(|err| {
5119                RouterError::backend(
5120                    0,
5121                    frame.header.corr,
5122                    format!("failed to read module configuration: {err}"),
5123                )
5124            })?;
5125            // Status and configuration snapshots release their locks before the image probe awaits.
5126            let image = module.running_image_agreement().await;
5127            // Read per request so the figure is current when the operator asks;
5128            // the daemon samples nothing in between.
5129            let resources = Some(module.child_resource_usage());
5130            let mut verdict =
5131                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5132            verdict.spec = Some(launch_spec_agreement(
5133                &configured,
5134                status.process_alive,
5135                status.launch_spec_digest.as_ref(),
5136            ));
5137            let pending_reload = Some(verdict);
5138            // Keep the retired policy field on the wire for one release so
5139            // existing status consumers still receive the platform policy.
5140            modules.push(
5141                SupervisorEntry::new(
5142                    status.module_id,
5143                    status.state.to_string(),
5144                    status.enabled,
5145                    status.live,
5146                    status.health.status,
5147                )
5148                .with_launch_nonce_env(Some(!cfg!(unix)))
5149                .with_launch_nonce_source(status.launch_nonce_source)
5150                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5151                .with_protocol(status.protocol)
5152                .with_pending_reload(pending_reload)
5153                .with_last_probe_ms(status.health.last_probe_ms)
5154                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5155                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5156                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5157                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5158                .with_restart_count(Some(status.restart_count))
5159                .with_max_restarts(Some(status.max_restarts))
5160                .with_lifetime_restarts(Some(status.lifetime_restarts))
5161                .with_spawn_generation(Some(status.spawn_generation))
5162                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5163                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5164                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5165                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5166                .with_resources(resources),
5167            );
5168        }
5169        let response = ClientControlResponse::SupervisorList {
5170            generation,
5171            modules,
5172        };
5173        Ok(vec![control_response_body_frame(
5174            &frame,
5175            &response,
5176            "ClientControlResponse::SupervisorList",
5177        )?])
5178    }
5179
5180    fn handle_supervisor_stderr_tail(
5181        &self,
5182        frame: Frame,
5183        module_id: String,
5184        max_lines: Option<u32>,
5185        max_bytes: Option<u32>,
5186    ) -> Result<Vec<Frame>, RouterError> {
5187        let Some(module) = self.supervisor.get(&module_id) else {
5188            return Ok(vec![control_error_frame(
5189                &frame,
5190                "unknown_module",
5191                format!("module_id '{module_id}' is not supervised"),
5192            )?]);
5193        };
5194
5195        let snapshot = module.stderr_tail(
5196            max_lines.map(|value| value as usize),
5197            max_bytes.map(|value| value as usize),
5198        );
5199
5200        let response = ClientControlResponse::SupervisorStderrTail {
5201            module_id,
5202            tail: StderrTail {
5203                capture: match snapshot.capture {
5204                    CaptureState::Captured => StderrCaptureState::Captured,
5205                    CaptureState::Incomplete { reason } => {
5206                        StderrCaptureState::Incomplete { reason }
5207                    }
5208                    CaptureState::NotCaptured { reason } => {
5209                        StderrCaptureState::NotCaptured { reason }
5210                    }
5211                },
5212                entries: snapshot
5213                    .entries
5214                    .into_iter()
5215                    .map(|entry| match entry {
5216                        TailEntry::Line {
5217                            text,
5218                            truncated,
5219                            at_ms,
5220                        } => StderrTailEntry::Line {
5221                            text,
5222                            truncated,
5223                            at_ms,
5224                        },
5225                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5226                    })
5227                    .collect(),
5228                dropped_lines: snapshot.dropped_lines,
5229            },
5230        };
5231        Ok(vec![control_response_body_frame(
5232            &frame,
5233            &response,
5234            "ClientControlResponse::SupervisorStderrTail",
5235        )?])
5236    }
5237
5238    async fn handle_supervisor_terminals(
5239        &self,
5240        frame: Frame,
5241        module_id: String,
5242    ) -> Result<Vec<Frame>, RouterError> {
5243        let Some(module) = self.supervisor.get(&module_id) else {
5244            return Ok(vec![control_error_frame(
5245                &frame,
5246                "unknown_module",
5247                format!("module_id '{module_id}' is not supervised"),
5248            )?]);
5249        };
5250
5251        // The journal read runs on a blocking thread: it can be megabytes of
5252        // file I/O and must not occupy a runtime worker.
5253        let terminals = module
5254            .read_durable_terminal_history()
5255            .await
5256            .map_err(|error| {
5257                RouterError::backend(
5258                    0,
5259                    frame.header.corr,
5260                    format!("failed to read terminal history: {error}"),
5261                )
5262            })?;
5263        let response = ClientControlResponse::SupervisorTerminals {
5264            module_id,
5265            terminals,
5266        };
5267        Ok(vec![control_response_body_frame(
5268            &frame,
5269            &response,
5270            "ClientControlResponse::SupervisorTerminals",
5271        )?])
5272    }
5273
5274    fn handle_supervisor_routes(
5275        &self,
5276        frame: Frame,
5277        module_id: Option<String>,
5278    ) -> Result<Vec<Frame>, RouterError> {
5279        let modules = self
5280            .forwarding
5281            .route_census(module_id.as_deref())
5282            .map_err(RouterError::Forwarding)?
5283            .into_iter()
5284            .map(|(module_id, routes)| SupervisorRouteModule {
5285                module_id,
5286                routes: routes
5287                    .into_iter()
5288                    .map(|route| SupervisorRoute {
5289                        consumer: match route.principal {
5290                            Principal::Reserved { module_id } => {
5291                                SupervisorRouteConsumer::Reserved { module_id }
5292                            }
5293                            Principal::Direct | Principal::Unverified => {
5294                                SupervisorRouteConsumer::Direct {
5295                                    connection_id: route.goodbye_target.connection_id.get(),
5296                                }
5297                            }
5298                        },
5299                        age_ms: Instant::now()
5300                            .saturating_duration_since(route.bound_at)
5301                            .as_millis()
5302                            .try_into()
5303                            .unwrap_or(u64::MAX),
5304                        draining: route.draining,
5305                        drain_reason: route.drain_reason,
5306                    })
5307                    .collect(),
5308            })
5309            .collect();
5310        let response = ClientControlResponse::SupervisorRoutes { modules };
5311        Ok(vec![control_response_body_frame(
5312            &frame,
5313            &response,
5314            "ClientControlResponse::SupervisorRoutes",
5315        )?])
5316    }
5317
5318    async fn handle_supervisor_provenance(
5319        &self,
5320        frame: Frame,
5321        module_id: Option<String>,
5322    ) -> Result<Vec<Frame>, RouterError> {
5323        let mut selected = if let Some(module_id) = module_id {
5324            let Some(module) = self.supervisor.get(&module_id) else {
5325                return Ok(vec![control_error_frame(
5326                    &frame,
5327                    "unknown_module",
5328                    format!("module_id '{module_id}' is not supervised"),
5329                )?]);
5330            };
5331            vec![module]
5332        } else {
5333            self.supervisor.list()
5334        };
5335
5336        let mut modules = Vec::with_capacity(selected.len());
5337        for module in selected.drain(..) {
5338            let (status, observed_image) = module
5339                .status_and_running_image_agreement()
5340                .await
5341                .map_err(|err| {
5342                    RouterError::backend(
5343                        0,
5344                        frame.header.corr,
5345                        format!("failed to read supervisor status: {err}"),
5346                    )
5347                })?;
5348            let module_declared = self
5349                .registry
5350                .get_module(&status.module_id)
5351                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5352                .and_then(|registration| registration.manifest.provenance)
5353                .map(|build| ModuleDeclaredProvenance::Reported { build })
5354                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5355            #[cfg(test)]
5356            let running_image = match &self.provenance_probe_override {
5357                Some(result) => result.clone(),
5358                None => observed_image,
5359            };
5360            #[cfg(not(test))]
5361            let running_image = observed_image;
5362            modules.push(SupervisorModuleProvenance {
5363                module_id: status.module_id,
5364                module_declared,
5365                daemon_observed: SupervisorObservedProcess {
5366                    launch_nonce_source: status.launch_nonce_source,
5367                    pid: status.pid,
5368                    spawned_at_ms: status.spawned_at_ms,
5369                    spawned_from: status.spawned_from,
5370                    running_image,
5371                },
5372            });
5373        }
5374        let daemon = SupervisorDaemonProvenance {
5375            daemon_build: self.daemon_provenance.build.clone(),
5376            daemon_observed: DaemonObservedProcess {
5377                pid: self.daemon_provenance.pid,
5378                started_at_ms: self
5379                    .daemon_provenance
5380                    .start_clock
5381                    .map(|clock| clock.started_at_ms())
5382                    .or(self.daemon_provenance.started_at_ms),
5383                running_image: self
5384                    .daemon_provenance
5385                    .probe
5386                    .observe(
5387                        self.daemon_provenance.pid,
5388                        self.daemon_provenance.executable_path.as_deref(),
5389                        self.daemon_provenance.executable_identity.clone(),
5390                        self.daemon_provenance.process_start_time,
5391                    )
5392                    .await,
5393            },
5394        };
5395        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5396        Ok(vec![control_response_body_frame(
5397            &frame,
5398            &response,
5399            "ClientControlResponse::SupervisorProvenance",
5400        )?])
5401    }
5402
5403    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5404        self.refresh_capability_requirements();
5405        let generation = self
5406            .registry
5407            .generation()
5408            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5409        let modules = self
5410            .supervisor
5411            .list()
5412            .into_iter()
5413            .map(|module| {
5414                let status = module.status_for_control("health").map_err(|err| {
5415                    RouterError::backend(
5416                        0,
5417                        frame.header.corr,
5418                        format!("failed to read supervisor health: {err}"),
5419                    )
5420                })?;
5421                let module_id = status.module_id;
5422                let capability_detail = self
5423                    .capability_evaluator
5424                    .required_problem_detail(&module_id);
5425                Ok(SupervisorHealthEntry {
5426                    module_id,
5427                    status: status.health.status,
5428                    detail: append_capability_problem_detail(
5429                        status.health.detail,
5430                        capability_detail,
5431                    ),
5432                    metrics: status.health.metrics,
5433                    consecutive_failures: status.health.consecutive_failures,
5434                    late_answer_count: status.health.late_answer_count,
5435                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5436                    last_action: status.health.last_action,
5437                    last_action_ms: status.health.last_action_ms,
5438                    last_probe_ms: status.health.last_probe_ms,
5439                })
5440            })
5441            .collect::<Result<Vec<_>, RouterError>>()?;
5442        let response = ClientControlResponse::SupervisorHealth {
5443            generation,
5444            modules,
5445        };
5446        Ok(vec![control_response_body_frame(
5447            &frame,
5448            &response,
5449            "ClientControlResponse::SupervisorHealth",
5450        )?])
5451    }
5452
5453    async fn handle_supervisor_restart(
5454        &self,
5455        frame: Frame,
5456        module_id: String,
5457        drain_timeout_ms: Option<u64>,
5458        requester: Principal,
5459    ) -> Result<Vec<Frame>, RouterError> {
5460        let operation_lock = self.supervisor.operation_lock();
5461        let _operation_guard = operation_lock.lock().await;
5462        let Some(module) = self.supervisor.get(&module_id) else {
5463            return Ok(vec![control_error_frame(
5464                &frame,
5465                "unknown_module",
5466                format!("module_id '{module_id}' is not supervised"),
5467            )?]);
5468        };
5469
5470        self.route_outages.mark_operator_action(&module_id);
5471        if let Err(err) = module
5472            .restart_requested(drain_timeout_ms, Some(requester))
5473            .await
5474        {
5475            self.route_outages
5476                .operator_action_ended_unrefused(&module_id);
5477            let (code, message) = match err {
5478                crate::supervise::SuperviseError::Disabled { .. } => {
5479                    ("module_disabled", err.to_string())
5480                }
5481                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5482                    ("swap_in_progress", err.to_string())
5483                }
5484                _ => (
5485                    "target_unavailable",
5486                    format!("failed to restart module_id '{module_id}': {err}"),
5487                ),
5488            };
5489            return Ok(vec![control_error_frame(&frame, code, message)?]);
5490        }
5491
5492        let response = ClientControlResponse::SupervisorAck {
5493            module_id,
5494            applied: true,
5495        };
5496        Ok(vec![control_response_body_frame(
5497            &frame,
5498            &response,
5499            "ClientControlResponse::SupervisorAck",
5500        )?])
5501    }
5502
5503    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
5504    /// when the old process has finished draining: a caller whose own lane
5505    /// rides the old process must get its reply before that drain waits on it.
5506    async fn handle_supervisor_swap(
5507        &self,
5508        frame: Frame,
5509        module_id: String,
5510        ready_timeout_ms: Option<u64>,
5511        requester: Principal,
5512    ) -> Result<Vec<Frame>, RouterError> {
5513        // The daemon-wide operation lock is held only to resolve the handle,
5514        // not across the swap. The swap can take its whole readiness budget,
5515        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
5516        // holding it here would park an operator's stop behind the swap it is
5517        // meant to abort. A rescan or stop that reaches the module during the
5518        // swap is served by the swap itself (see `supervise_swap`).
5519        let module = {
5520            let operation_lock = self.supervisor.operation_lock();
5521            let _operation_guard = operation_lock.lock().await;
5522            self.supervisor.get(&module_id)
5523        };
5524        let Some(module) = module else {
5525            return Ok(vec![control_error_frame(
5526                &frame,
5527                "unknown_module",
5528                format!("module_id '{module_id}' is not supervised"),
5529            )?]);
5530        };
5531
5532        self.route_outages.mark_operator_action(&module_id);
5533        if let Err(err) = module
5534            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5535            .await
5536        {
5537            self.route_outages
5538                .operator_action_ended_unrefused(&module_id);
5539            use crate::supervise::SuperviseError;
5540            let message = err.to_string();
5541            let error = match err {
5542                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5543                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5544                    code: "swap_refused".to_string(),
5545                    message,
5546                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5547                },
5548                SuperviseError::SwapFailed {
5549                    arm,
5550                    candidate_exit,
5551                    ..
5552                } => ErrorBody {
5553                    code: "swap_failed".to_string(),
5554                    message,
5555                    detail: Some(serde_json::json!({
5556                        "arm": arm.as_str(),
5557                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5558                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5559                    })),
5560                },
5561                _ => ErrorBody::new(
5562                    "target_unavailable",
5563                    format!("failed to swap module_id '{module_id}': {message}"),
5564                ),
5565            };
5566            return Ok(vec![control_error_body_frame(&frame, error)?]);
5567        }
5568        // A completed swap kept the incumbent serving until cutover, so it
5569        // usually opened no outage; a mark left behind would make the next,
5570        // unrelated outage read as requested.
5571        self.route_outages
5572            .operator_action_ended_unrefused(&module_id);
5573
5574        let response = ClientControlResponse::SupervisorAck {
5575            module_id,
5576            applied: true,
5577        };
5578        Ok(vec![control_response_body_frame(
5579            &frame,
5580            &response,
5581            "ClientControlResponse::SupervisorAck",
5582        )?])
5583    }
5584
5585    async fn handle_supervisor_reload(
5586        &self,
5587        frame: Frame,
5588        module_id: String,
5589        requester: Principal,
5590    ) -> Result<Vec<Frame>, RouterError> {
5591        let operation_lock = self.supervisor.operation_lock();
5592        let _operation_guard = operation_lock.lock().await;
5593        let Some(module) = self.supervisor.get(&module_id) else {
5594            return Ok(vec![control_error_frame(
5595                &frame,
5596                "unknown_module",
5597                format!("module_id '{module_id}' is not supervised"),
5598            )?]);
5599        };
5600
5601        self.route_outages.mark_operator_action(&module_id);
5602        if let Err(err) = module.reload_requested(Some(requester)).await {
5603            self.route_outages
5604                .operator_action_ended_unrefused(&module_id);
5605            let (code, message) = match err {
5606                crate::supervise::SuperviseError::Disabled { .. } => {
5607                    ("module_disabled", err.to_string())
5608                }
5609                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5610                    ("swap_in_progress", err.to_string())
5611                }
5612                _ => (
5613                    "reload_failed",
5614                    format!("failed to reload module_id '{module_id}': {err}"),
5615                ),
5616            };
5617            return Ok(vec![control_error_frame(&frame, code, message)?]);
5618        }
5619
5620        let response = ClientControlResponse::SupervisorAck {
5621            module_id,
5622            applied: true,
5623        };
5624        Ok(vec![control_response_body_frame(
5625            &frame,
5626            &response,
5627            "ClientControlResponse::SupervisorAck",
5628        )?])
5629    }
5630
5631    async fn handle_supervisor_rescan(
5632        &self,
5633        frame: Frame,
5634        preview: bool,
5635        requester: Principal,
5636    ) -> Result<Vec<Frame>, RouterError> {
5637        let Some(context) = self.rescan.clone() else {
5638            return Ok(vec![control_error_frame(
5639                &frame,
5640                "rescan_unavailable",
5641                "the daemon was not started with a reloadable config path".to_string(),
5642            )?]);
5643        };
5644
5645        let operation_lock = self.supervisor.operation_lock();
5646        let _operation_guard = operation_lock.lock().await;
5647        let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5648            Ok(config) => config,
5649            Err(err) => {
5650                return Ok(vec![control_error_frame(
5651                    &frame,
5652                    "invalid_daemon_config",
5653                    format!("supervisor rescan rejected daemon config: {err}"),
5654                )?])
5655            }
5656        };
5657        // `load` reports a missing file as Ok(None), which is correct at boot
5658        // (no config, nothing to supervise) and catastrophic here: rescan treats
5659        // "not in the config" as "remove it", so an absent file would read as an
5660        // empty module list and retire the entire running fleet. An editor
5661        // writing via write-new-then-rename, or a half-finished edit, is enough
5662        // to open that window. Refuse instead: a config that cannot be read
5663        // carries no instruction to remove anything.
5664        let Some((config, activity_config)) = loaded else {
5665            return Ok(vec![control_error_frame(
5666                &frame,
5667                "invalid_daemon_config",
5668                format!(
5669                    "daemon config not found at {}; refusing to rescan (an absent config would \
5670                     retire every supervised module)",
5671                    context.config_path.display()
5672                ),
5673            )?]);
5674        };
5675        let (
5676            configured_port,
5677            storage_config,
5678            admission_facts_carrier_module_id,
5679            admission_facts_targets,
5680            scope_authority_owners,
5681            privileged_operator_prompt,
5682            modules,
5683            reserved_capabilities,
5684        ) = (
5685            config.port,
5686            config.storage,
5687            config.admission_facts_carrier_module_id,
5688            config.admission_facts_targets,
5689            config.scope_authority_owners,
5690            config.privileged_operator_prompt,
5691            config.modules,
5692            config.reserved_capabilities,
5693        );
5694
5695        // Collect the sections rescan cannot apply, so the REPLY carries them.
5696        //
5697        // The warning below has always been correct and has always gone only to
5698        // the journal -- addressed to whoever reads logs, while the person who
5699        // just edited the config is looking at the CLI. Naming each section
5700        // individually rather than setting a flag: "something outside modules
5701        // changed" sends the operator back to diffing their own file, which is
5702        // the work this is meant to save.
5703        let mut restart_required = Vec::new();
5704        for section in RestartRequiredSection::ALL {
5705            let changed = match section {
5706                RestartRequiredSection::Port => configured_port != context.configured_port,
5707                RestartRequiredSection::Storage => storage_config != context.storage_config,
5708                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5709                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5710                }
5711                RestartRequiredSection::AdmissionFactsTargets => {
5712                    admission_facts_targets != context.admission_facts_targets
5713                }
5714                RestartRequiredSection::ScopeAuthorityOwners => {
5715                    scope_authority_owners != context.scope_authority_owners
5716                }
5717                RestartRequiredSection::PrivilegedOperatorPrompt => {
5718                    privileged_operator_prompt != context.privileged_operator_prompt
5719                }
5720            };
5721            if changed {
5722                restart_required.push(section.label().to_string());
5723            }
5724        }
5725        if !restart_required.is_empty() {
5726            warn!(
5727                config_path = %context.config_path.display(),
5728                sections = %restart_required.join(", "),
5729                "daemon config changed outside the modules section; restart the daemon to apply those changes"
5730            );
5731        }
5732
5733        for configured in &modules {
5734            if let Err(err) = validate_spec(&configured.module_spec()) {
5735                return Ok(vec![control_error_frame(
5736                    &frame,
5737                    "invalid_daemon_config",
5738                    format!("supervisor rescan rejected daemon config: {err}"),
5739                )?]);
5740            }
5741        }
5742
5743        let configured_capabilities = modules
5744            .iter()
5745            .map(|module| (module.module_id.clone(), module.enabled))
5746            .collect::<Vec<_>>();
5747        let preview_capability_warnings = if preview {
5748            let (_, registrations) = self.runtime_capability_snapshot()?;
5749            let current_modules = self
5750                .supervisor
5751                .list()
5752                .into_iter()
5753                .map(|module| module.module_id().to_string())
5754                .collect::<BTreeSet<_>>();
5755            let resulting_modules = configured_capabilities.clone();
5756            let removed = current_modules
5757                .into_iter()
5758                .filter(|module_id| {
5759                    !resulting_modules
5760                        .iter()
5761                        .any(|(configured_id, _)| configured_id == module_id)
5762                })
5763                .collect::<Vec<_>>();
5764            self.capability_evaluator.preview_removal_warnings(
5765                resulting_modules,
5766                &removed,
5767                &registrations,
5768            )
5769        } else {
5770            Vec::new()
5771        };
5772        let result = match self
5773            .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5774            .await
5775        {
5776            Ok(result) => result,
5777            Err(message) => {
5778                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5779            }
5780        };
5781        if !preview {
5782            if let Some(ledger) = &self.activity {
5783                ledger.module_rescanned(&result.added, &result.removed, requester);
5784            }
5785            if let Err(code) = self.configure_activity(activity_config).await {
5786                warn!(
5787                    code,
5788                    "activity retention could not be applied during rescan"
5789                );
5790            }
5791            self.capability_evaluator
5792                .configure(configured_capabilities, reserved_capabilities);
5793            self.capability_evaluator.wake_deadline_loop();
5794            self.refresh_capability_requirements();
5795        }
5796        let mut result = result;
5797        result.restart_required = restart_required;
5798        result.capability_warnings = preview_capability_warnings;
5799        let response = ClientControlResponse::SupervisorRescan { result };
5800        Ok(vec![control_response_body_frame(
5801            &frame,
5802            &response,
5803            "ClientControlResponse::SupervisorRescan",
5804        )?])
5805    }
5806
5807    async fn handle_supervisor_release_reserved(
5808        &self,
5809        frame: Frame,
5810        module_id: String,
5811    ) -> Result<Vec<Frame>, RouterError> {
5812        let Some(context) = self.rescan.clone() else {
5813            return Ok(vec![control_error_frame(
5814                &frame,
5815                "release_unavailable",
5816                "reserved-id release requires a daemon started with a reloadable config path",
5817            )?]);
5818        };
5819        let operation_lock = self.supervisor.operation_lock();
5820        let _operation_guard = operation_lock.lock().await;
5821        let loaded = match crate::daemon_config::load(&context.config_path) {
5822            Ok(Some(config)) => config,
5823            Ok(None) => {
5824                return Ok(vec![control_error_frame(
5825                    &frame,
5826                    "invalid_daemon_config",
5827                    format!(
5828                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5829                        context.config_path.display()
5830                    ),
5831                )?])
5832            }
5833            Err(err) => {
5834                return Ok(vec![control_error_frame(
5835                    &frame,
5836                    "invalid_daemon_config",
5837                    format!("unable to verify reserved-id release against daemon config: {err}"),
5838                )?])
5839            }
5840        };
5841        if loaded
5842            .modules
5843            .iter()
5844            .any(|configured| configured.module_id == module_id)
5845        {
5846            return Ok(vec![control_error_frame(
5847                &frame,
5848                "reserved_module_configured",
5849                format!(
5850                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5851                ),
5852            )?]);
5853        }
5854        if !self.supervisor.release_retained_reserved_gate(&module_id) {
5855            return Ok(vec![control_error_frame(
5856                &frame,
5857                "reserved_gate_not_retained",
5858                format!(
5859                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5860                ),
5861            )?]);
5862        }
5863
5864        let response = ClientControlResponse::SupervisorAck {
5865            module_id,
5866            applied: true,
5867        };
5868        Ok(vec![control_response_body_frame(
5869            &frame,
5870            &response,
5871            "ClientControlResponse::SupervisorAck",
5872        )?])
5873    }
5874
5875    /// Reconcile the running module set against the configured one.
5876    ///
5877    /// With `preview` set, the diff is computed and returned WITHOUT applying any
5878    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
5879    /// deliberately shares this function with the executing path rather than
5880    /// computing the same diff somewhere else -- two implementations of one
5881    /// decision agree until they do not, and the whole value of a preview is that
5882    /// it describes the operation that will actually run.
5883    async fn reconcile_supervised_modules(
5884        &self,
5885        supervisor: &Supervisor,
5886        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5887        preview: bool,
5888        requester: Principal,
5889    ) -> Result<SupervisorRescanResult, String> {
5890        let mut current = BTreeMap::new();
5891        for module in self.supervisor.list() {
5892            let (spec, health) = module.configuration().map_err(|err| {
5893                format!(
5894                    "failed to read configuration for module_id '{}': {err}",
5895                    module.module_id()
5896                )
5897            })?;
5898            let enabled = module
5899                .status()
5900                .map_err(|err| {
5901                    format!(
5902                        "failed to read status for module_id '{}': {err}",
5903                        module.module_id()
5904                    )
5905                })?
5906                .enabled;
5907            current.insert(
5908                module.module_id().to_string(),
5909                (module, spec, health, enabled),
5910            );
5911        }
5912        let configured = configured_modules
5913            .into_iter()
5914            .map(|module| (module.module_id.clone(), module))
5915            .collect::<BTreeMap<_, _>>();
5916
5917        let added = configured
5918            .keys()
5919            .filter(|module_id| !current.contains_key(*module_id))
5920            .cloned()
5921            .collect::<Vec<_>>();
5922        let removed = current
5923            .keys()
5924            .filter(|module_id| !configured.contains_key(*module_id))
5925            .cloned()
5926            .collect::<Vec<_>>();
5927        let mut changed_pending_reload = Vec::new();
5928        let mut configuration_changes = BTreeSet::new();
5929        let mut enabled_changes = BTreeSet::new();
5930        let mut unchanged = 0_u32;
5931
5932        for (module_id, configured_module) in &configured {
5933            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5934            else {
5935                continue;
5936            };
5937            // Compare the whole launch spec so a future launch field cannot
5938            // accidentally become a live-only policy change. Health is stored
5939            // separately and applies live without replacing the process.
5940            let launch_changed = *current_spec != configured_module.module_spec();
5941            let configuration_changed =
5942                launch_changed || *current_health != configured_module.health;
5943            let enabled_changed = *current_enabled != configured_module.enabled;
5944            if configuration_changed {
5945                configuration_changes.insert(module_id.clone());
5946            }
5947            if launch_changed {
5948                changed_pending_reload.push(module_id.clone());
5949            }
5950            if enabled_changed {
5951                enabled_changes.insert(module_id.clone());
5952            }
5953            if !configuration_changed && !enabled_changed {
5954                unchanged = unchanged.saturating_add(1);
5955            }
5956        }
5957
5958        // Everything above this point is pure computation over two snapshots.
5959        // Everything below MUTATES. The preview returns here so the boundary is a
5960        // single early return rather than a condition repeated at each mutation
5961        // site, where one missed guard would apply part of a change the caller was
5962        // told would not happen.
5963        if preview {
5964            return Ok(SupervisorRescanResult {
5965                added,
5966                removed,
5967                changed_pending_reload,
5968                enabled_changes: enabled_changes.iter().cloned().collect(),
5969                unchanged,
5970                preview: true,
5971                // Filled by the caller on both paths, so the preview reports
5972                // restart-required sections identically to an executed rescan --
5973                // the preview is where an operator is most likely to be looking.
5974                restart_required: Vec::new(),
5975                capability_warnings: Vec::new(),
5976            });
5977        }
5978
5979        for module_id in &removed {
5980            let module = &current
5981                .get(module_id)
5982                .expect("removed module came from current supervisor state")
5983                .0;
5984            module
5985                .retire_requested(Some(requester.clone()))
5986                .await
5987                .map_err(|err| {
5988                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
5989                })?;
5990            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
5991            //
5992            // `handle_route_open` resolves an absent module in three steps:
5993            // registry, then supervisor status, then tombstone. Retiring first
5994            // opens a window where ALL THREE ARE ABSENT -- the registry entry
5995            // went with the teardown above, the supervisor entry went with
5996            // `retire`, and the tombstone does not exist yet -- so a route.open
5997            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
5998            // it") for a module that was deliberately removed and whose caller
5999            // should get `module_removed` (TERMINAL, carrying a removal age).
6000            //
6001            // Writing the tombstone first closes it: during the window the
6002            // supervisor entry still answers, so the caller gets
6003            // `target_unavailable` -- retryable, and TRUE, because the module
6004            // is mid-teardown. After both statements it is `module_removed`.
6005            // No instant remains where a removed module reads as one that
6006            // never existed.
6007            //
6008            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
6009            // the absence of a test beside a fix invites deletion: these are
6010            // two sync statements with no await between them, so reaching the
6011            // window needs a second worker thread to land exactly between them
6012            // and there is no hook to force it. MEASURED: the 25 daemon_config
6013            // tests pass identically with the old order and the new one, so
6014            // the existing suite cannot see this and a green run is not
6015            // evidence either way. What the suite does hold is the
6016            // post-condition -- a removed module answers `module_removed` --
6017            // which this preserves.
6018            //
6019            // Found by an Athena panel reading the shipped tree against a
6020            // design note (2026-09-19), as the one concrete instance of that
6021            // note's class that survived contact with source. Direction is
6022            // benign: retryable where terminal was intended, never the reverse.
6023            self.supervisor.record_rescan_removal(module_id);
6024            self.supervisor.retire(module_id);
6025            self.route_outages.forget(module_id);
6026        }
6027
6028        for module_id in configured.keys() {
6029            let Some((module, _, _, _)) = current.get(module_id) else {
6030                continue;
6031            };
6032            let configured_module = configured
6033                .get(module_id)
6034                .expect("configured module id came from configured map");
6035            if configuration_changes.contains(module_id) {
6036                module
6037                    .update_configuration(
6038                        configured_module.module_spec(),
6039                        configured_module.health.clone(),
6040                        configured_module.drain_timeout_ms,
6041                    )
6042                    .await
6043                    .map_err(|err| {
6044                        format!(
6045                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
6046                        )
6047                    })?;
6048            }
6049            if enabled_changes.contains(module_id) {
6050                // A rescan that starts or stops a module applies an operator's
6051                // edit to the config, so the resulting outage was asked for.
6052                self.route_outages.mark_operator_action(module_id);
6053                module
6054                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6055                    .await
6056                    .map_err(|err| {
6057                        self.route_outages.operator_action_ended_unrefused(module_id);
6058                        format!(
6059                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6060                            configured_module.enabled
6061                        )
6062                    })?;
6063            }
6064        }
6065
6066        for module_id in &added {
6067            let configured_module = configured
6068                .get(module_id)
6069                .expect("added module id came from configured map");
6070            supervisor
6071                .supervise_configured_requested(
6072                    configured_module.module_spec(),
6073                    configured_module.enabled,
6074                    configured_module.health.clone(),
6075                    configured_module.drain_timeout_ms,
6076                    configured_module.restart,
6077                    Some(requester.clone()),
6078                )
6079                .map_err(|err| {
6080                    format!("failed to add module_id '{module_id}' during rescan: {err}")
6081                })?;
6082        }
6083
6084        Ok(SupervisorRescanResult {
6085            added,
6086            removed,
6087            changed_pending_reload,
6088            enabled_changes: enabled_changes.iter().cloned().collect(),
6089            unchanged,
6090            preview: false,
6091            // Filled by the caller, which is the only layer that can see the
6092            // previous config to diff against.
6093            restart_required: Vec::new(),
6094            capability_warnings: Vec::new(),
6095        })
6096    }
6097
6098    async fn handle_supervisor_set_enabled(
6099        &self,
6100        frame: Frame,
6101        module_id: String,
6102        enabled: bool,
6103        requester: Principal,
6104    ) -> Result<Vec<Frame>, RouterError> {
6105        let operation_lock = self.supervisor.operation_lock();
6106        let _operation_guard = operation_lock.lock().await;
6107        let Some(module) = self.supervisor.get(&module_id) else {
6108            return Ok(vec![control_error_frame(
6109                &frame,
6110                "unknown_module",
6111                format!("module_id '{module_id}' is not supervised"),
6112            )?]);
6113        };
6114
6115        // Enabling counts as well as disabling: a module an operator starts
6116        // is refused until it registers, and that wait was asked for.
6117        self.route_outages.mark_operator_action(&module_id);
6118        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6119            Ok(applied) => applied,
6120            Err(err) => {
6121                self.route_outages
6122                    .operator_action_ended_unrefused(&module_id);
6123                return Ok(vec![control_error_frame(
6124                    &frame,
6125                    "target_unavailable",
6126                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6127                )?]);
6128            }
6129        };
6130        if !applied {
6131            // Already in the requested state: nothing was made unavailable,
6132            // so the mark must not outlive this request.
6133            self.route_outages
6134                .operator_action_ended_unrefused(&module_id);
6135        }
6136
6137        self.capability_evaluator.wake_deadline_loop();
6138        self.refresh_capability_requirements();
6139        let response = ClientControlResponse::SupervisorAck { module_id, applied };
6140        Ok(vec![control_response_body_frame(
6141            &frame,
6142            &response,
6143            "ClientControlResponse::SupervisorAck",
6144        )?])
6145    }
6146
6147    async fn handle_supervisor_health_probe(
6148        &self,
6149        frame: Frame,
6150        module_id: String,
6151    ) -> Result<Vec<Frame>, RouterError> {
6152        self.refresh_capability_requirements();
6153        let Some(registration) = self
6154            .registry
6155            .get_module(&module_id)
6156            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6157        else {
6158            return Ok(vec![control_error_frame(
6159                &frame,
6160                "unknown_module",
6161                format!("module_id '{module_id}' is not registered"),
6162            )?]);
6163        };
6164
6165        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
6166        // named for it. Making `module_registration_grants_op` return false
6167        // unconditionally reddens five tests, and every one is named for something
6168        // else -- capability relay, probe/bind demultiplexing, supervision-only
6169        // probing. They exercise a successful advertisement check on the way to their
6170        // own subject.
6171        //
6172        // Real protection, fragile in a specific way: narrowing any of those tests to
6173        // focus on its stated subject would silently remove coverage nobody knows
6174        // they are carrying. Recorded here rather than as a sixth test, because the
6175        // useful fact is WHICH tests hold the guard up -- a new test would add
6176        // coverage without telling the next person what the existing ones quietly do.
6177        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6178        {
6179            return Ok(vec![control_error_frame(
6180                &frame,
6181                "health_not_advertised",
6182                format!("module_id '{module_id}' did not advertise health.check"),
6183            )?]);
6184        }
6185
6186        let deadline = Instant::now() + self.health_probe_timeout;
6187        let pending = match self.forwarding.begin_module_control_rpc_for(
6188            &module_id,
6189            MODULE_CONTROL_OP_HEALTH_CHECK,
6190            deadline,
6191        ) {
6192            Ok(pending) => pending,
6193            Err(err) => {
6194                return Ok(vec![control_error_frame(
6195                    &frame,
6196                    forwarding_error_code(&err),
6197                    err.to_string(),
6198                )?])
6199            }
6200        };
6201
6202        let PendingModuleControlRpc {
6203            endpoint,
6204            module_sink,
6205            negotiated_ver,
6206            corr: probe_corr,
6207            receiver,
6208        } = pending;
6209        let mut guard =
6210            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6211        let probe_body =
6212            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6213                RouterError::backend(
6214                    0,
6215                    frame.header.corr,
6216                    format!("failed to encode health.check request: {err}"),
6217                )
6218            })?;
6219        let probe_frame = Frame::build_with_version(
6220            negotiated_ver,
6221            FrameType::Request,
6222            control_flags(),
6223            0,
6224            0,
6225            probe_corr,
6226            probe_body,
6227        )
6228        .map_err(RouterError::FrameBuild)?;
6229
6230        // Queue admission and the reply share one deadline, so a module that
6231        // stops reading cannot park the requesting connection's reader forever.
6232        match timeout_at(deadline, module_sink.send(probe_frame)).await {
6233            Ok(Ok(())) => {}
6234            Ok(Err(err)) => {
6235                return Ok(vec![control_error_frame(
6236                    &frame,
6237                    "target_unavailable",
6238                    err.to_string(),
6239                )?]);
6240            }
6241            Err(_) => {
6242                return Ok(vec![control_error_frame(
6243                    &frame,
6244                    "module_timeout",
6245                    format!(
6246                        "module_id '{module_id}' did not answer health.check within {:?}",
6247                        self.health_probe_timeout
6248                    ),
6249                )?]);
6250            }
6251        }
6252
6253        match timeout_at(deadline, receiver).await {
6254            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6255                guard.disarm();
6256                let Some(report) = response.health_report() else {
6257                    return Ok(vec![control_error_frame(
6258                        &frame,
6259                        "invalid_control_body",
6260                        "health.check RPC returned a non-health response",
6261                    )?]);
6262                };
6263                // Metrics go out whole here. The supervisor's cached snapshot
6264                // caps this blob (see truncate_health_metrics), and this path
6265                // exists precisely to answer without that cap -- so applying it
6266                // here would leave no way to see what the cached view drops.
6267                let HealthReport {
6268                    status,
6269                    detail,
6270                    metrics,
6271                } = report;
6272                let capability_detail = self
6273                    .capability_evaluator
6274                    .required_problem_detail(&module_id);
6275                let response = ClientControlResponse::SupervisorHealthProbe {
6276                    module_id,
6277                    status,
6278                    detail: append_capability_problem_detail(detail, capability_detail),
6279                    metrics,
6280                };
6281                Ok(vec![control_response_body_frame(
6282                    &frame,
6283                    &response,
6284                    "ClientControlResponse::SupervisorHealthProbe",
6285                )?])
6286            }
6287            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6288                guard.disarm();
6289                Ok(vec![control_error_body_frame(&frame, body)?])
6290            }
6291            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6292                guard.disarm();
6293                Ok(vec![control_error_frame(
6294                    &frame,
6295                    "target_unavailable",
6296                    message,
6297                )?])
6298            }
6299            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6300                guard.disarm();
6301                Ok(vec![control_error_frame(
6302                    &frame,
6303                    "invalid_control_body",
6304                    message,
6305                )?])
6306            }
6307            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6308                guard.disarm();
6309                Ok(vec![control_error_frame(
6310                    &frame,
6311                    "invalid_control_body",
6312                    format!("expected module-control op '{expected}', got '{actual}'"),
6313                )?])
6314            }
6315            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6316                guard.disarm();
6317                Ok(vec![control_error_frame(
6318                    &frame,
6319                    "module_timeout",
6320                    format!(
6321                        "module_id '{module_id}' answered health.check after {:?}",
6322                        self.health_probe_timeout
6323                    ),
6324                )?])
6325            }
6326            Ok(Err(_)) => Ok(vec![control_error_frame(
6327                &frame,
6328                "target_unavailable",
6329                "health.check waiter was canceled before the module responded",
6330            )?]),
6331            Err(_) => Ok(vec![control_error_frame(
6332                &frame,
6333                "module_timeout",
6334                format!(
6335                    "module_id '{module_id}' did not answer health.check within {:?}",
6336                    self.health_probe_timeout
6337                ),
6338            )?]),
6339        }
6340    }
6341
6342    fn supervisor_status(
6343        &self,
6344        module_id: &str,
6345        corr: u64,
6346    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6347        self.supervisor
6348            .get(module_id)
6349            .map(|module| {
6350                let warming = module.is_warming_for_control("status").map_err(|err| {
6351                    RouterError::backend(
6352                        0,
6353                        corr,
6354                        format!(
6355                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
6356                        ),
6357                    )
6358                })?;
6359                module.status_for_control("status").map_err(|err| {
6360                    RouterError::backend(
6361                        0,
6362                        corr,
6363                        format!(
6364                            "failed to read supervisor status for module_id '{module_id}': {err}"
6365                        ),
6366                    )
6367                }).map(|status| (status, warming))
6368            })
6369            .transpose()
6370    }
6371
6372    fn guard_module_control_op(
6373        &self,
6374        frame: &Frame,
6375        module_id: &str,
6376        op: &str,
6377    ) -> Result<Option<Frame>, RouterError> {
6378        if self.module_grants_op(module_id, op, frame.header.corr)? {
6379            return Ok(None);
6380        }
6381
6382        Ok(Some(control_error_frame(
6383            frame,
6384            "op_not_allowed",
6385            format!("module_id '{module_id}' did not grant control op '{op}'"),
6386        )?))
6387    }
6388
6389    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6390        let Some(registration) = self
6391            .registry
6392            .get_module(module_id)
6393            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6394        else {
6395            return Ok(false);
6396        };
6397        Ok(module_registration_grants_op(&registration.control_ops, op))
6398    }
6399
6400    fn handle_status_update(
6401        &self,
6402        endpoint: ModuleEndpointId,
6403        frame: Frame,
6404    ) -> Result<Vec<Frame>, RouterError> {
6405        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6406            Ok(update) => update,
6407            Err(err) => {
6408                // Forward-compat: a newer module may push a channel-0 op this subc
6409                // version doesn't know. The control contract says unknown push ops
6410                // are IGNORED, never answered with an error. Only a malformed body
6411                // for an op we DO know is a real error worth surfacing.
6412                if is_known_module_push_op(&frame.body) {
6413                    return Ok(vec![control_error_frame(
6414                        &frame,
6415                        "invalid_control_body",
6416                        format!("malformed module control push body: {err}"),
6417                    )?]);
6418                }
6419                return Ok(Vec::new());
6420            }
6421        };
6422
6423        match update {
6424            ModuleControlPush::RouteStatus {
6425                route_channel,
6426                route_epoch,
6427                status,
6428            } => {
6429                self.forwarding
6430                    .cache_status(endpoint, route_channel, route_epoch, status)
6431                    .map_err(RouterError::Forwarding)?;
6432            }
6433        }
6434        Ok(Vec::new())
6435    }
6436
6437    fn handle_route_poll(
6438        &self,
6439        ctx: &RouteCtx,
6440        frame: Frame,
6441        route_channel: u16,
6442        route_epoch: u32,
6443        kind: PollKind,
6444    ) -> Result<Vec<Frame>, RouterError> {
6445        let snapshot = self
6446            .forwarding
6447            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6448            .map_err(RouterError::Forwarding)?;
6449        let response = match (kind, snapshot) {
6450            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6451                ClientControlResponse::RoutePoll {
6452                    route_channel,
6453                    route_epoch,
6454                    status,
6455                    live: None,
6456                }
6457            }
6458            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6459                route_channel,
6460                route_epoch,
6461                status: None,
6462                live: None,
6463            },
6464            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6465                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
6466                // is what makes reporting `true` correct rather than a
6467                // confident guess. `process_live` returns None only when the
6468                // module id has no supervisor snapshot at all -- an
6469                // externally-started module the daemon did not spawn -- and
6470                // for those the supervisor has no opinion to offer, ever. It
6471                // is never None for a supervised module in an unknown state:
6472                // a supervised module always has a snapshot, and the answer
6473                // comes from `state == Running && process_alive`.
6474                //
6475                // The route is Bound, so the module completed a HELLO on a
6476                // live connection; "the process this route points at is
6477                // running" is therefore attested by the binding rather than
6478                // assumed. Reporting `false` for an unsupervised module would
6479                // be the actual lie -- it would tell a client its healthy
6480                // route is dead because the daemon does not manage the
6481                // process.
6482                //
6483                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
6484                // module whose liveness is genuinely unknown, e.g. a snapshot
6485                // that has not been populated yet -- THIS DEFAULT BECOMES
6486                // WRONG and must split: unsupervised stays true, unknown
6487                // becomes null so the client can tell the two apart. The
6488                // response field is already `Option<bool>`, so the wire can
6489                // carry that distinction today.
6490                let live = self
6491                    .process_liveness
6492                    .as_ref()
6493                    .and_then(|source| source.process_live(&module_id))
6494                    .unwrap_or(true);
6495                ClientControlResponse::RoutePoll {
6496                    route_channel,
6497                    route_epoch,
6498                    status: None,
6499                    live: Some(live),
6500                }
6501            }
6502            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6503                route_channel,
6504                route_epoch,
6505                status: None,
6506                live: Some(false),
6507            },
6508        };
6509
6510        Ok(vec![control_response_body_frame(
6511            &frame,
6512            &response,
6513            "ClientControlResponse::RoutePoll",
6514        )?])
6515    }
6516
6517    pub(crate) fn observe_module_control_completion(
6518        &self,
6519        completion: ModuleControlRpcCompletion,
6520    ) -> bool {
6521        match completion {
6522            ModuleControlRpcCompletion::Unknown => false,
6523            ModuleControlRpcCompletion::Settled => true,
6524            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6525                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6526                info!(
6527                    module_id = %module_id,
6528                    latency_ms,
6529                    "late health.check answer proves the module is alive"
6530                );
6531                match self
6532                    .supervisor
6533                    .record_late_health_answer(&module_id, latency_ms)
6534                {
6535                    Ok(true) => {}
6536                    Ok(false) => debug!(
6537                        module_id = %module_id,
6538                        latency_ms,
6539                        "late health.check answer has no active supervisor snapshot"
6540                    ),
6541                    Err(err) => warn!(
6542                        module_id = %module_id,
6543                        latency_ms,
6544                        error = %err,
6545                        "failed to record late health.check answer"
6546                    ),
6547                }
6548                true
6549            }
6550        }
6551    }
6552
6553    /// Decide whether a failure while settling a relayed `route.bind` belongs to
6554    /// the module connection whose frame is being handled, or to the client that
6555    /// relay was opened for.
6556    ///
6557    /// This runs on the MODULE connection's frame handler, where returning `Err`
6558    /// ends that connection -- and a module connection carries every client's
6559    /// routes to that module, so ending it costs the whole fleet its tools.
6560    /// `ConnectionClosing` carries the id of the connection that is closing, and
6561    /// when that id is a CLIENT's, the condition is entirely about that one
6562    /// client's route.open. A client-scoped condition has no authority over a
6563    /// shared module connection, so it is logged and the single relay is dropped:
6564    /// the client is going away, and `complete_pending_relay` already removed the
6565    /// relay before failing, so there is nothing left to settle. Anything that
6566    /// relay still reserved is released by that client's own connection teardown,
6567    /// which is already under way -- that is what "closing" means.
6568    ///
6569    /// Every other failure is a statement about THIS connection and stays fatal:
6570    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
6571    /// id in `ConnectionClosing` all mean this connection cannot keep serving
6572    /// frames correctly.
6573    fn refuse_to_end_module_connection_for_a_client(
6574        &self,
6575        module_connection_id: ConnectionId,
6576        corr: u64,
6577        err: ForwardingError,
6578    ) -> Result<(), RouterError> {
6579        if let ForwardingError::ConnectionClosing { connection_id } = err {
6580            if connection_id != module_connection_id {
6581                warn!(
6582                    module_connection_id = module_connection_id.get(),
6583                    client_connection_id = connection_id.get(),
6584                    corr,
6585                    "dropping a route.bind response for a closing client; the module connection keeps serving"
6586                );
6587                return Ok(());
6588            }
6589        }
6590        Err(RouterError::Forwarding(err))
6591    }
6592
6593    fn handle_module_relay_response(
6594        &self,
6595        connection_id: ConnectionId,
6596        frame: Frame,
6597    ) -> Result<Vec<Frame>, RouterError> {
6598        let mut secondary_error = None;
6599        let outcome = match frame.header.ty {
6600            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6601                Ok(probe) if probe.op == "route.bind" => {
6602                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6603                        Ok(ModuleControlResponse::RouteBindAck {}) => {
6604                            RouteBindRelayOutcome::Accepted
6605                        }
6606                        Ok(other) => {
6607                            let message =
6608                                format!("route.bind response carried unexpected body: {other:?}");
6609                            secondary_error = Some(control_error_frame(
6610                                &frame,
6611                                "invalid_control_body",
6612                                message.clone(),
6613                            )?);
6614                            RouteBindRelayOutcome::ModuleGone(message)
6615                        }
6616                        Err(err) => {
6617                            let message = format!("malformed route.bind response body: {err}");
6618                            secondary_error = Some(control_error_frame(
6619                                &frame,
6620                                "invalid_control_body",
6621                                message.clone(),
6622                            )?);
6623                            RouteBindRelayOutcome::ModuleGone(message)
6624                        }
6625                    }
6626                }
6627                Ok(probe) => {
6628                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6629                    {
6630                        Ok(response) => ModuleControlRpcOutcome::Response(response),
6631                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6632                            "malformed {} response body: {err}",
6633                            probe.op
6634                        )),
6635                    };
6636                    let completion = self
6637                        .forwarding
6638                        .complete_module_control_rpc(
6639                            connection_id,
6640                            frame.header.corr,
6641                            Some(&probe.op),
6642                            outcome,
6643                        )
6644                        .map_err(RouterError::Forwarding)?;
6645                    if !self.observe_module_control_completion(completion) {
6646                        debug!(
6647                            connection_id = connection_id.get(),
6648                            corr = frame.header.corr,
6649                            op = %probe.op,
6650                            "dropping late or unknown module-control RPC response"
6651                        );
6652                    }
6653                    return Ok(Vec::new());
6654                }
6655                Err(err) => {
6656                    if let Some(expected_op) = self
6657                        .forwarding
6658                        .pending_module_control_op(connection_id, frame.header.corr)
6659                        .map_err(RouterError::Forwarding)?
6660                    {
6661                        let completion = self
6662                            .forwarding
6663                            .complete_module_control_rpc(
6664                                connection_id,
6665                                frame.header.corr,
6666                                None,
6667                                ModuleControlRpcOutcome::MalformedResponse(format!(
6668                                    "malformed {expected_op} response body: {err}"
6669                                )),
6670                            )
6671                            .map_err(RouterError::Forwarding)?;
6672                        if !self.observe_module_control_completion(completion) {
6673                            debug!(
6674                                connection_id = connection_id.get(),
6675                                corr = frame.header.corr,
6676                                "dropping late malformed module-control RPC response"
6677                            );
6678                        }
6679                        return Ok(Vec::new());
6680                    }
6681                    let message = format!("malformed route.bind response body: {err}");
6682                    secondary_error = Some(control_error_frame(
6683                        &frame,
6684                        "invalid_control_body",
6685                        message.clone(),
6686                    )?);
6687                    RouteBindRelayOutcome::ModuleGone(message)
6688                }
6689            },
6690            FrameType::Error => {
6691                if self
6692                    .forwarding
6693                    .pending_module_control_op(connection_id, frame.header.corr)
6694                    .map_err(RouterError::Forwarding)?
6695                    .is_some()
6696                {
6697                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6698                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6699                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6700                            "malformed module-control ERROR body: {err}"
6701                        )),
6702                    };
6703                    let completion = self
6704                        .forwarding
6705                        .complete_module_control_rpc(
6706                            connection_id,
6707                            frame.header.corr,
6708                            None,
6709                            outcome,
6710                        )
6711                        .map_err(RouterError::Forwarding)?;
6712                    if !self.observe_module_control_completion(completion) {
6713                        debug!(
6714                            connection_id = connection_id.get(),
6715                            corr = frame.header.corr,
6716                            "dropping late or unknown module-control RPC error"
6717                        );
6718                    }
6719                    return Ok(Vec::new());
6720                }
6721                match serde_json::from_slice::<ErrorBody>(&frame.body) {
6722                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
6723                    Err(err) => {
6724                        let message = format!("malformed route.bind ERROR body: {err}");
6725                        secondary_error = Some(control_error_frame(
6726                            &frame,
6727                            "invalid_control_body",
6728                            message.clone(),
6729                        )?);
6730                        RouteBindRelayOutcome::ModuleGone(message)
6731                    }
6732                }
6733            }
6734            ty => {
6735                return Ok(vec![control_error_frame(
6736                    &frame,
6737                    "unsupported_control_frame",
6738                    format!("unsupported module channel-0 frame {ty:?}"),
6739                )?])
6740            }
6741        };
6742
6743        let settled =
6744            self.forwarding
6745                .complete_pending_relay(connection_id, frame.header.corr, outcome);
6746        let completion = match settled {
6747            Ok(completion) => completion,
6748            Err(err) => {
6749                self.refuse_to_end_module_connection_for_a_client(
6750                    connection_id,
6751                    frame.header.corr,
6752                    err,
6753                )?;
6754                return Ok(secondary_error.into_iter().collect());
6755            }
6756        };
6757        if let Some(target) = completion.abandoned.as_ref() {
6758            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6759        }
6760        if !completion.settled {
6761            debug!(
6762                connection_id = connection_id.get(),
6763                corr = frame.header.corr,
6764                frame_type = ?frame.header.ty,
6765                "dropping late or unknown route.bind relay response"
6766            );
6767        }
6768        Ok(secondary_error.into_iter().collect())
6769    }
6770
6771    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6772        debug!(connection_id = connection_id.get(), "handling GOODBYE");
6773        // GOODBYE ends the connection's logical session even when its socket
6774        // stays open. Use disconnect teardown so verdicts, client notices and
6775        // scope authority are released at the same lifecycle boundary.
6776        self.cleanup_connection_with_end_reason(
6777            connection_id,
6778            RegistrationEndReason::ExplicitGoodbye,
6779        )
6780        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6781        Ok(Vec::new())
6782    }
6783}
6784
6785impl Default for ControlHandler {
6786    fn default() -> Self {
6787        Self::new(Arc::new(Registry::default()))
6788    }
6789}
6790
6791impl crate::supervise::SwapPromotionObserver for ControlHandler {
6792    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6793        self.apply_registration_capabilities(registration);
6794    }
6795}
6796
6797fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6798    CapabilityRequirementStatus {
6799        consumer: status.consumer,
6800        capability: status.capability,
6801        need: match status.need {
6802            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6803            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6804        },
6805        verdict: status.verdict.as_str().to_string(),
6806        episode_seq: status.episode_seq,
6807        config_satisfiable: status.config_satisfiable,
6808        runtime_available: status.runtime_available,
6809        detail: status.detail,
6810    }
6811}
6812
6813fn append_capability_problem_detail(
6814    detail: Option<String>,
6815    capability_detail: Option<String>,
6816) -> Option<String> {
6817    match (detail, capability_detail) {
6818        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6819        (Some(detail), None) => Some(detail),
6820        (None, Some(capability_detail)) => Some(capability_detail),
6821        (None, None) => None,
6822    }
6823}
6824
6825fn subc_ops() -> Vec<String> {
6826    SUBC_CONTROL_OPS
6827        .iter()
6828        .map(|op| (*op).to_string())
6829        .collect()
6830}
6831
6832fn module_subc_ops() -> Vec<String> {
6833    SUBC_CONTROL_OPS
6834        .iter()
6835        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6836        .map(|op| (*op).to_string())
6837        .collect()
6838}
6839
6840#[cfg(test)]
6841fn module_baseline_control_ops() -> Vec<String> {
6842    MODULE_BASELINE_CONTROL_OPS
6843        .iter()
6844        .map(|op| (*op).to_string())
6845        .collect()
6846}
6847
6848fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6849    let mut seen = HashSet::new();
6850    let mut effective = Vec::new();
6851    for op in MODULE_BASELINE_CONTROL_OPS {
6852        if seen.insert((*op).to_string()) {
6853            effective.push((*op).to_string());
6854        }
6855    }
6856    for op in declared.unwrap_or_default() {
6857        if seen.insert(op.clone()) {
6858            effective.push(op);
6859        }
6860    }
6861    effective
6862}
6863
6864fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6865    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6866}
6867
6868fn target_module_id(target: &RouteTarget) -> &str {
6869    match target {
6870        RouteTarget::ToolProvider { module_id }
6871        | RouteTarget::ManagementSurface { module_id }
6872        | RouteTarget::InternalService { module_id, .. } => module_id,
6873    }
6874}
6875
6876fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6877    roles.iter().any(|role| match (target, role) {
6878        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6879        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6880        (
6881            RouteTarget::InternalService { service_id, .. },
6882            ProviderRole::InternalService {
6883                service_id: provided,
6884                ..
6885            },
6886        ) => service_id == provided,
6887        _ => false,
6888    })
6889}
6890
6891fn is_routable_role(role: &ProviderRole) -> bool {
6892    matches!(
6893        role,
6894        ProviderRole::ToolProvider { .. }
6895            | ProviderRole::ManagementSurface { .. }
6896            | ProviderRole::InternalService { .. }
6897    )
6898}
6899
6900#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6901enum ControlRequestBodyError {
6902    UnknownOp,
6903    InvalidBody,
6904}
6905
6906#[derive(Debug, Deserialize)]
6907struct ControlOpProbe {
6908    op: String,
6909}
6910
6911/// Channel-0 push ops this subc version understands. A push whose `op` is not in
6912/// this set is treated as a forward-compat unknown and ignored rather than errored.
6913const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6914
6915fn is_known_module_push_op(body: &[u8]) -> bool {
6916    serde_json::from_slice::<ControlOpProbe>(body)
6917        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6918        .unwrap_or(false)
6919}
6920
6921fn is_known_module_request_op(body: &[u8]) -> bool {
6922    serde_json::from_slice::<ControlOpProbe>(body)
6923        .map(|probe| is_module_to_subc_op(&probe.op))
6924        .unwrap_or(false)
6925}
6926
6927fn is_module_to_subc_op(op: &str) -> bool {
6928    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6929}
6930
6931fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6932    debug!(
6933        op = %op,
6934        connection_id = connection_id.get(),
6935        corr,
6936        "control dispatch"
6937    );
6938}
6939
6940fn log_slow_control_dispatch(
6941    dispatch_started_at: Option<StdInstant>,
6942    op: &'static str,
6943    connection_id: ConnectionId,
6944    corr: u64,
6945    route_open_timings: Option<&RouteOpenTimings>,
6946) {
6947    let Some(dispatch_started_at) = dispatch_started_at else {
6948        return;
6949    };
6950    // Use the same monotonic clock as the phase timers and relay deadline.
6951    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
6952    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6953        if let Some(timings) = route_open_timings {
6954            warn!(
6955                op = %op,
6956                connection_id = connection_id.get(),
6957                corr,
6958                elapsed_ms = elapsed.as_millis() as u64,
6959                module_id = ?timings.module_id,
6960                principal = %timings.principal,
6961                admission_ms = timings.admission.as_millis() as u64,
6962                project_root_ms = timings.project_root.as_millis() as u64,
6963                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
6964                bind_send_ms = timings.bind_send.as_millis() as u64,
6965                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
6966                "slow control dispatch"
6967            );
6968            return;
6969        }
6970        warn!(
6971            op = %op,
6972            connection_id = connection_id.get(),
6973            corr,
6974            elapsed_ms = elapsed.as_millis() as u64,
6975            "slow control dispatch"
6976        );
6977    }
6978}
6979
6980/// A test stand-in for the OS caller lookup, given the recorded endpoints.
6981#[cfg(test)]
6982type TestCallerLookup =
6983    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
6984
6985/// Every new client operation needs an explicit privilege decision here.
6986fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
6987    match request {
6988        ClientControlRequest::SupervisorRestart { module_id, .. } => {
6989            Some(PrivilegedOperation::Restart { module_id })
6990        }
6991        ClientControlRequest::SupervisorSwap { module_id, .. } => {
6992            Some(PrivilegedOperation::Swap { module_id })
6993        }
6994        ClientControlRequest::SupervisorReload { module_id } => {
6995            Some(PrivilegedOperation::Reload { module_id })
6996        }
6997        ClientControlRequest::SupervisorRescan { preview } => {
6998            if *preview {
6999                None
7000            } else {
7001                Some(PrivilegedOperation::Rescan)
7002            }
7003        }
7004        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7005            Some(PrivilegedOperation::SetEnabled {
7006                module_id,
7007                enabled: *enabled,
7008            })
7009        }
7010        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7011            Some(PrivilegedOperation::ReleaseReserved { module_id })
7012        }
7013        ClientControlRequest::ServerDescribe {}
7014        | ClientControlRequest::CatalogList { .. }
7015        | ClientControlRequest::RouteOpen { .. }
7016        | ClientControlRequest::RoutePoll { .. }
7017        | ClientControlRequest::SupervisorList {}
7018        | ClientControlRequest::SupervisorHealth {}
7019        | ClientControlRequest::SupervisorHealthProbe { .. }
7020        | ClientControlRequest::SupervisorRoutes { .. }
7021        | ClientControlRequest::SupervisorProvenance { .. }
7022        | ClientControlRequest::SupervisorStderrTail { .. }
7023        | ClientControlRequest::SupervisorTerminals { .. }
7024        | ClientControlRequest::SupervisorSpawnSnapshot {}
7025        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7026    }
7027}
7028
7029fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7030    match request {
7031        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7032        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7033        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7034        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7035        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7036        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7037        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7038        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7039        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7040        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7041        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7042        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7043        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7044        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7045        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7046        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7047        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7048        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7049        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7050    }
7051}
7052
7053/// The `scope=` field of the `route.open accepted` line, from the stamp that
7054/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
7055/// so a ref holding control characters cannot forge a log line, and cut to 64
7056/// characters with a trailing `…` to bound the line. The bind proof and the
7057/// scope's attributes never appear.
7058fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7059    use subc_protocol::scope::ScopeKind;
7060    const MAX_REF_CHARS: usize = 64;
7061    let kind = match stamp.kind {
7062        ScopeKind::Head => "head",
7063        ScopeKind::Worker => "worker",
7064        ScopeKind::Ephemeral => "ephemeral",
7065    };
7066    let mut scope_ref: String = stamp.scope_ref.chars().take(MAX_REF_CHARS).collect();
7067    if stamp.scope_ref.chars().nth(MAX_REF_CHARS).is_some() {
7068        scope_ref.push('…');
7069    }
7070    format!(
7071        "{kind}/{}/{scope_ref:?}@{}",
7072        principal_label(&stamp.owner),
7073        stamp.scope_epoch
7074    )
7075}
7076
7077fn principal_label(principal: &Principal) -> String {
7078    match principal {
7079        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7080        Principal::Direct => "direct".to_string(),
7081        other => format!("{other:?}"),
7082    }
7083}
7084
7085fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7086    match request {
7087        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7088        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7089        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7090        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7091        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7092    }
7093}
7094
7095fn parse_client_control_request(
7096    body: &[u8],
7097) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7098    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7099        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7100            Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
7101                ControlRequestBodyError::InvalidBody
7102            }
7103            Ok(_) => ControlRequestBodyError::UnknownOp,
7104            Err(_) => ControlRequestBodyError::InvalidBody,
7105        };
7106        (err, classification)
7107    })
7108}
7109
7110fn parse_module_control_request_from_module(
7111    body: &[u8],
7112) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7113    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7114        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7115            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7116            Ok(_) => ControlRequestBodyError::UnknownOp,
7117            Err(_) => ControlRequestBodyError::InvalidBody,
7118        };
7119        (err, classification)
7120    })
7121}
7122
7123#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7124enum ProviderRoleKind {
7125    ToolProvider,
7126    PipelineStage,
7127    ManagementSurface,
7128    InternalService,
7129}
7130
7131fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7132    match role {
7133        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7134        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7135        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7136        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7137    }
7138}
7139
7140fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7141    roles.iter().map(provider_role_kind).collect()
7142}
7143
7144/// Most refused scope records named individually in the log per sync; the
7145/// `refused` count on the accepted line is always complete.
7146const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7147
7148/// Per-outcome counts of one accepted `scope.sync`, for its log line.
7149#[derive(Debug, Default, PartialEq, Eq)]
7150struct ScopeOutcomeCounts {
7151    created: usize,
7152    replaced: usize,
7153    updated: usize,
7154    unchanged: usize,
7155    refused: usize,
7156}
7157
7158impl ScopeOutcomeCounts {
7159    fn of(results: &[ScopeRecordResult]) -> Self {
7160        let mut counts = Self::default();
7161        for result in results {
7162            let slot = match result.outcome {
7163                ScopeRecordOutcome::Created => &mut counts.created,
7164                ScopeRecordOutcome::Replaced => &mut counts.replaced,
7165                ScopeRecordOutcome::Updated => &mut counts.updated,
7166                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7167                ScopeRecordOutcome::Refused => &mut counts.refused,
7168            };
7169            *slot += 1;
7170        }
7171        counts
7172    }
7173}
7174
7175#[cfg(test)]
7176mod scope_outcome_count_tests {
7177    use super::*;
7178
7179    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7180        ScopeRecordResult {
7181            scope_ref: "r".to_string(),
7182            scope_epoch: 1,
7183            outcome,
7184            code: None,
7185            message: None,
7186            version: None,
7187            parent_state: None,
7188        }
7189    }
7190
7191    /// Each outcome lands in its own count, so a refused record can never be
7192    /// hidden inside the total the log already printed.
7193    #[test]
7194    fn every_outcome_is_counted_in_its_own_field() {
7195        let results = [
7196            result(ScopeRecordOutcome::Created),
7197            result(ScopeRecordOutcome::Created),
7198            result(ScopeRecordOutcome::Replaced),
7199            result(ScopeRecordOutcome::Updated),
7200            result(ScopeRecordOutcome::Unchanged),
7201            result(ScopeRecordOutcome::Refused),
7202            result(ScopeRecordOutcome::Refused),
7203            result(ScopeRecordOutcome::Refused),
7204        ];
7205        assert_eq!(
7206            ScopeOutcomeCounts::of(&results),
7207            ScopeOutcomeCounts {
7208                created: 2,
7209                replaced: 1,
7210                updated: 1,
7211                unchanged: 1,
7212                refused: 3,
7213            }
7214        );
7215    }
7216}
7217
7218/// Return whether a catalog change can create a newly violating live route.
7219/// Removing an attested claim is intentionally excluded: it makes fewer routes
7220/// forbidden and therefore must leave the existing route census untouched.
7221fn capability_census_trigger(
7222    old: Option<&CapabilityDeclarations>,
7223    new: Option<&CapabilityDeclarations>,
7224) -> bool {
7225    let old_provides = old
7226        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7227        .unwrap_or_default();
7228    let old_denies = old
7229        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7230        .unwrap_or_default();
7231    let new = new.cloned().unwrap_or(CapabilityDeclarations {
7232        provides: Vec::new(),
7233        requires: Vec::new(),
7234        must_never_reach: Vec::new(),
7235    });
7236
7237    new.provides
7238        .iter()
7239        .any(|capability| !old_provides.contains(capability))
7240        || new
7241            .must_never_reach
7242            .iter()
7243            .any(|capability| !old_denies.contains(capability))
7244}
7245
7246/// Find the first capability an attested opener denies that an attested target
7247/// claims. Both manifests are live registry records, never cached or client data.
7248fn denied_capability<'a>(
7249    opening_manifest: &'a ModuleManifest,
7250    target_manifest: &ModuleManifest,
7251) -> Option<&'a str> {
7252    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7253    let target_capabilities = target_manifest.capabilities.as_ref()?;
7254    opening_capabilities
7255        .must_never_reach
7256        .iter()
7257        .find(|denied| {
7258            target_capabilities
7259                .provides
7260                .iter()
7261                .any(|provided| provided == *denied)
7262        })
7263        .map(String::as_str)
7264}
7265
7266fn catalog_update_frozen_field_message(
7267    registered: &ModuleManifest,
7268    provides: &[ProviderRole],
7269) -> Option<String> {
7270    let old_has_provides = !registered.provides.is_empty();
7271    let new_has_provides = !provides.is_empty();
7272    if old_has_provides != new_has_provides {
7273        return Some(format!(
7274            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7275            registered.module_id
7276        ));
7277    }
7278
7279    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
7280        return Some(format!(
7281            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7282            registered.module_id
7283        ));
7284    }
7285
7286    let registered_concurrency = manifest_concurrency(registered);
7287    let mut candidate = registered.clone();
7288    candidate.provides = provides.to_vec();
7289    let candidate_concurrency = manifest_concurrency(&candidate);
7290    if candidate_concurrency != registered_concurrency {
7291        return Some(format!(
7292            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7293            registered.module_id, registered_concurrency, candidate_concurrency
7294        ));
7295    }
7296
7297    // control_ops live beside the manifest in the HELLO body, not inside
7298    // ModuleManifest, so a provides-only catalog.update cannot change them.
7299    None
7300}
7301
7302fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7303    manifest.provides.iter().any(is_routable_role)
7304}
7305
7306/// Returns the routable-provider concurrency subc should enforce for this manifest.
7307///
7308/// ToolProvider and ManagementSurface store their delivery concurrency directly.
7309/// InternalService has no role-specific concurrency field, so it retains the
7310/// existing ModuleManaged default for backward compatibility.
7311fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7312    manifest
7313        .provides
7314        .iter()
7315        .find_map(|provider| match provider {
7316            ProviderRole::ToolProvider { concurrency, .. }
7317            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7318            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7319        })
7320        .unwrap_or(Concurrency::ModuleManaged)
7321}
7322
7323/// True when the manifest carries a ManagementSurface role whose concurrency
7324/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
7325/// bytes because the typed manifest deliberately erases that distinction: the
7326/// default exists for wire compatibility, and this probe exists so the default
7327/// stays observable. Any parse irregularity returns false -- the caller only
7328/// logs, and a malformed body already failed registration upstream.
7329fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7330    let has_management_surface = manifest
7331        .provides
7332        .iter()
7333        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7334    if !has_management_surface {
7335        return false;
7336    }
7337    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7338        return false;
7339    };
7340    let Some(provides) = raw
7341        .get("manifest")
7342        .and_then(|manifest| manifest.get("provides"))
7343        .and_then(serde_json::Value::as_array)
7344    else {
7345        return false;
7346    };
7347    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
7348    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
7349    // against the management_surface_manifest_without_concurrency golden, not
7350    // recalled (the externally-tagged guess was this function's first bug).
7351    provides.iter().any(|role| {
7352        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7353            && role.get("concurrency").is_none()
7354    })
7355}
7356
7357fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7358    if peer_version != PROTOCOL_VERSION {
7359        return Err(format!(
7360            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7361        ));
7362    }
7363    Ok(PROTOCOL_VERSION)
7364}
7365
7366fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7367    Frame::build_with_version(
7368        response_version(frame),
7369        FrameType::Pong,
7370        frame.header.flags,
7371        0,
7372        0,
7373        frame.header.corr,
7374        Vec::new(),
7375    )
7376    .map_err(RouterError::FrameBuild)
7377}
7378
7379pub(crate) fn control_error_frame(
7380    frame: &Frame,
7381    code: &'static str,
7382    message: impl Into<String>,
7383) -> Result<Frame, RouterError> {
7384    control_error_body_frame(
7385        frame,
7386        ErrorBody {
7387            code: code.to_string(),
7388            message: message.into(),
7389            detail: None,
7390        },
7391    )
7392}
7393
7394fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7395    let body = serde_json::to_vec(&error).map_err(|err| {
7396        RouterError::backend(
7397            0,
7398            frame.header.corr,
7399            format!("failed to encode control ERROR: {err}"),
7400        )
7401    })?;
7402
7403    Frame::build_with_version(
7404        response_version(frame),
7405        FrameType::Error,
7406        control_flags(),
7407        0,
7408        0,
7409        frame.header.corr,
7410        body,
7411    )
7412    .map_err(RouterError::FrameBuild)
7413}
7414
7415fn control_response_body_frame<T: Serialize>(
7416    frame: &Frame,
7417    reply: &T,
7418    label: &'static str,
7419) -> Result<Frame, RouterError> {
7420    let body = serde_json::to_vec(reply).map_err(|err| {
7421        RouterError::backend(
7422            0,
7423            frame.header.corr,
7424            format!("failed to encode {label}: {err}"),
7425        )
7426    })?;
7427
7428    Frame::build_with_version(
7429        response_version(frame),
7430        FrameType::Response,
7431        control_flags(),
7432        0,
7433        0,
7434        frame.header.corr,
7435        body,
7436    )
7437    .map_err(RouterError::FrameBuild)
7438}
7439
7440/// Map a forwarding failure to the wire code a client sees.
7441///
7442/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
7443/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
7444/// chosen here decides whether a caller retries or gives up.
7445///
7446/// That makes attribution the load-bearing property, not merely having a code. A
7447/// permanent fault published as a retryable one produces a fleet-wide retry storm
7448/// against something that can never recover; a transient fault published as
7449/// permanent gives up on work that would have succeeded. Both look correct in a
7450/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
7451/// the mapping per variant rather than merely asserting that some code exists.
7452///
7453/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
7454/// two codes on the same side of the boundary passes it. Measured rather than
7455/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
7456/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
7457/// a test named for something else that happens to assert the string.
7458///
7459/// That accidental coverage is deliberately left alone rather than promoted to a
7460/// named test, because it guards a property this function does not promise.
7461/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
7462/// specific code within a class, so identity is free to change and only the
7463/// partition is a contract. Splitting it out would assert a guarantee nothing
7464/// depends on — and a suite that promises more than the code does is the harder
7465/// thing to correct later, because the next reader cannot tell which assertions
7466/// are load-bearing.
7467///
7468/// Pin identity here the moment a consumer branches on a specific code.
7469fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7470    match err {
7471        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7472        ForwardingError::NoModuleConnection => "target_unavailable",
7473        ForwardingError::ModuleReloading { .. } => "module_reloading",
7474        ForwardingError::ClientRouteChannelExhausted { .. }
7475        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7476        ForwardingError::StaleModuleEndpoint
7477        | ForwardingError::UnknownReservation { .. }
7478        | ForwardingError::ConnectionClosing { .. }
7479        | ForwardingError::ClientEgressClosed { .. }
7480        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7481        // Only a swap candidate's registration can produce this, and it means
7482        // exactly what a second active HELLO for a live id means.
7483        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7484        ForwardingError::RelayCorrelationExhausted
7485        | ForwardingError::RouteOpenBuild(_)
7486        | ForwardingError::Poisoned => "forwarding_error",
7487    }
7488}
7489
7490fn response_version(frame: &Frame) -> u8 {
7491    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7492        frame.header.ver
7493    } else {
7494        PROTOCOL_VERSION
7495    }
7496}
7497
7498fn control_flags() -> Flags {
7499    Flags::new(false, Priority::Passive, false)
7500}
7501
7502/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
7503/// channel. The target is the module (a client never saw the route), so this
7504/// takes the module path: delivered late rather than dropped when the module's
7505/// queue is momentarily full, and never closing its connection.
7506fn send_goodbye_target_best_effort(
7507    counters: &DaemonCounters,
7508    target: &GoodbyeTarget,
7509    context: &'static str,
7510) {
7511    let Ok(frame) = Frame::build_with_version(
7512        target.negotiated_ver,
7513        FrameType::Goodbye,
7514        control_flags(),
7515        target.channel,
7516        target.epoch,
7517        0,
7518        Vec::new(),
7519    ) else {
7520        return;
7521    };
7522    crate::forwarding::send_module_route_goodbye(
7523        counters,
7524        &target.sink,
7525        frame,
7526        target.module_id.as_deref(),
7527        context,
7528    );
7529}
7530
7531pub(crate) fn send_route_control_pushes(
7532    forwarding: &ForwardingTable,
7533    routes: Vec<EndpointRoute>,
7534    push: ClientControlPush,
7535) {
7536    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7537    for route in routes {
7538        let target = route.goodbye_target;
7539        if let Some((existing, channels)) = targets
7540            .iter_mut()
7541            .find(|(existing, _)| existing.connection_id == target.connection_id)
7542        {
7543            debug_assert_eq!(
7544                existing.negotiated_ver, target.negotiated_ver,
7545                "one connection cannot negotiate multiple frame versions"
7546            );
7547            if !channels.contains(&target.channel) {
7548                channels.push(target.channel);
7549            }
7550            continue;
7551        }
7552        let channel = target.channel;
7553        targets.push((target, vec![channel]));
7554    }
7555    for (target, mut channels) in targets {
7556        channels.sort_unstable();
7557        let mut push = push.clone();
7558        match &mut push {
7559            ClientControlPush::RouteClosing {
7560                channels: covered, ..
7561            }
7562            | ClientControlPush::RouteClosed {
7563                channels: covered, ..
7564            } => *covered = channels,
7565        }
7566        let body = match serde_json::to_vec(&push) {
7567            Ok(body) => body,
7568            Err(err) => {
7569                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7570                continue;
7571            }
7572        };
7573        let frame = match Frame::build_with_version(
7574            target.negotiated_ver,
7575            FrameType::Push,
7576            control_flags(),
7577            0,
7578            0,
7579            0,
7580            body.clone(),
7581        ) {
7582            Ok(frame) => frame,
7583            Err(err) => {
7584                warn!(
7585                    route_channel = target.channel,
7586                    error = %err,
7587                    "failed to build route lifecycle control PUSH frame"
7588                );
7589                continue;
7590            }
7591        };
7592        if let Err(err) = target.sink.try_send(frame) {
7593            if target.close_on_delivery_failure() {
7594                warn!(
7595                    target_connection_id = target.connection_id.get(),
7596                    route_channel = target.channel,
7597                    error = %err,
7598                    "route lifecycle control PUSH was not delivered to client; closing target connection"
7599                );
7600                let _ = forwarding.escalate_client_delivery_failure(
7601                    target.connection_id,
7602                    target.channel,
7603                    target.epoch,
7604                    CloseReason::new(
7605                        "route_lifecycle_push_delivery_failed",
7606                        format!(
7607                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7608                            target.channel
7609                        ),
7610                    ),
7611                    crate::forwarding::UndeliveredFrame {
7612                        module_id: target.module_id.as_deref(),
7613                        sink: &target.sink,
7614                    },
7615                );
7616            }
7617        }
7618    }
7619}
7620
7621#[cfg(test)]
7622mod tests {
7623    mod activity_read {
7624        include!("activity_read_tests.rs");
7625    }
7626
7627    mod lifecycle_activity {
7628        include!("control_lifecycle_tests.rs");
7629    }
7630    #[cfg(unix)]
7631    #[tokio::test]
7632    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7633        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7634        let path = dir.join("subc.jsonc");
7635        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7636            "program":"/bin/sleep","args":["60"],"protocol":"none",
7637            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7638        }}}).to_string()).unwrap();
7639        let mut configured = crate::daemon_config::load(&path)
7640            .unwrap()
7641            .unwrap()
7642            .modules
7643            .pop()
7644            .unwrap();
7645        let registry = std::sync::Arc::new(crate::Registry::default());
7646        let handle = crate::SupervisorHandle::new();
7647        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7648            .with_handle(handle.clone());
7649        let module = supervisor
7650            .supervise_configured_with_health(
7651                configured.module_spec(),
7652                true,
7653                configured.health.clone(),
7654                None,
7655                configured.restart,
7656            )
7657            .unwrap();
7658        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7659        let before = module.status().unwrap().pid;
7660        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7661        configured.health.cadence = std::time::Duration::from_secs(3600);
7662        let health_only = handler
7663            .reconcile_supervised_modules(
7664                &supervisor,
7665                vec![configured.clone()],
7666                false,
7667                Principal::Direct,
7668            )
7669            .await
7670            .unwrap();
7671        assert!(
7672            health_only.changed_pending_reload.is_empty(),
7673            "health policy is already applied live"
7674        );
7675        assert_eq!(module.status().unwrap().pid, before);
7676        assert_eq!(
7677            module.configuration().unwrap().1.http,
7678            configured.health.http
7679        );
7680        configured.args = vec!["61".into()];
7681        let launch = handler
7682            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7683            .await
7684            .unwrap();
7685        assert_eq!(launch.changed_pending_reload, ["stock"]);
7686        assert_eq!(
7687            module.status().unwrap().pid,
7688            before,
7689            "a launch edit is stored until reload"
7690        );
7691        module.drain().await.unwrap();
7692    }
7693    use cortexkit_test_support::ScratchDir;
7694    use std::{
7695        collections::BTreeMap,
7696        fmt,
7697        path::PathBuf,
7698        sync::{Arc, Mutex},
7699        time::Duration,
7700    };
7701
7702    use serde_json::{json, Value};
7703    use subc_protocol::{
7704        manifest::{
7705            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7706            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7707        },
7708        session::HealthStatus,
7709        FrameType,
7710    };
7711
7712    use super::*;
7713    use crate::{
7714        forwarding::{DataRoute, DataRouteState},
7715        registry::ChannelState,
7716        router::FrameSink,
7717        stderr_tail::DEFAULT_MAX_LINE_BYTES,
7718        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7719        RouteCtx, Router,
7720    };
7721    use tokio::{
7722        sync::mpsc,
7723        time::{sleep, Instant},
7724    };
7725    use tracing::{
7726        field::{Field, Visit},
7727        Event, Subscriber,
7728    };
7729    use tracing_subscriber::{layer::Context, prelude::*, Layer};
7730
7731    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
7732    ///
7733    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
7734    /// is only populated for `tests/*.rs` integration test binaries -- this file
7735    /// compiles as part of the library target, which gets neither. This test's
7736    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
7737    /// and the sibling binary lives two directories up at
7738    /// `<target-dir>/<profile>/fake-aft-stub`.
7739    ///
7740    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
7741    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
7742    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
7743    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
7744    /// `NotFound`, which reads as a broken test rather than an unbuilt
7745    /// dependency -- so state the cause and the remedy instead. Deliberately a
7746    /// panic and not a silent skip: a test that quietly passes when it could not
7747    /// run is worse than one that fails, because it reports health it never
7748    /// verified.
7749    fn fake_aft_stub_path() -> PathBuf {
7750        let mut path = std::env::current_exe().expect("current_exe available in tests");
7751        path.pop(); // .../deps/
7752        path.pop(); // .../<profile>/
7753        path.push(if cfg!(windows) {
7754            "fake-aft-stub.exe"
7755        } else {
7756            "fake-aft-stub"
7757        });
7758        assert!(
7759            path.exists(),
7760            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7761             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7762            path.display()
7763        );
7764        path
7765    }
7766
7767    /// Whether clients retry `code` in place: the predicate itself, never a copy
7768    /// of its set. A copied list breaks silently when a code is added to or
7769    /// removed from the real one, and a stale copy here would let exactly the
7770    /// failure this test exists to catch pass.
7771    fn client_retries(code: &str) -> bool {
7772        subc_protocol::error_codes::is_retryable_route_open(code)
7773    }
7774
7775    /// A code is not a label — clients branch on it, so publishing the wrong KIND
7776    /// of failure is worse than publishing none. A permanent fault dressed as
7777    /// retryable makes every client in the fleet retry forever against something
7778    /// that cannot recover; a transient fault dressed as permanent abandons work
7779    /// that would have succeeded.
7780    ///
7781    /// Asserting "a code exists" cannot catch either, because the string is free
7782    /// to say anything. This enumerates every variant and pins which side of the
7783    /// retry boundary it lands on, so a new variant must be classified here
7784    /// deliberately rather than inheriting whichever arm it was appended to.
7785    #[test]
7786    fn retryability_of_forwarding_codes_matches_the_failure() {
7787        // Transient by nature: the target is booting, reloading, or its endpoint
7788        // was swapped mid-flight. Retrying is how these resolve.
7789        let transient = [
7790            ForwardingError::NoModuleConnection,
7791            ForwardingError::ModuleReloading {
7792                module_id: "m".into(),
7793            },
7794            ForwardingError::StaleModuleEndpoint,
7795            ForwardingError::UnknownReservation {
7796                client_channel: 1,
7797                module_channel: 1,
7798            },
7799            ForwardingError::ConnectionClosing {
7800                connection_id: ConnectionId::new(1),
7801            },
7802            ForwardingError::ClientEgressClosed {
7803                connection_id: ConnectionId::new(1),
7804            },
7805            ForwardingError::ModuleEgressUnavailable {
7806                connection_id: ConnectionId::new(1),
7807            },
7808        ];
7809        for err in transient {
7810            let code = forwarding_error_code(&err);
7811            assert!(
7812                client_retries(code),
7813                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7814            );
7815        }
7816
7817        // Not fixed by retrying. Channel and correlation exhaustion need the
7818        // caller to close routes, and a poisoned lock is a daemon that cannot
7819        // recover at all — the worst thing to advertise as retryable, since every
7820        // client would storm a daemon that will never answer.
7821        let permanent = [
7822            ForwardingError::ConnectionRoleConflict {
7823                connection_id: ConnectionId::new(1),
7824            },
7825            ForwardingError::ClientRouteChannelExhausted {
7826                connection_id: ConnectionId::new(1),
7827            },
7828            ForwardingError::ModuleRouteChannelExhausted {
7829                endpoint: ModuleEndpointId {
7830                    connection_id: ConnectionId::new(1),
7831                    generation: 1,
7832                },
7833            },
7834            ForwardingError::RelayCorrelationExhausted,
7835            ForwardingError::RouteOpenBuild("x".into()),
7836            ForwardingError::Poisoned,
7837        ];
7838        for err in permanent {
7839            let code = forwarding_error_code(&err);
7840            assert!(
7841                !client_retries(code),
7842                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7843            );
7844        }
7845    }
7846
7847    /// The principal is the daemon's answer to "who is calling", and modules
7848    /// branch on it: aft gates bash on it, cerebellum gates browser control,
7849    /// plexus gates connector invocation. So a stamp is an authorization input in
7850    /// another process, not a label — and both possible answers SUCCEED, which is
7851    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
7852    /// first-party capability to something that never proved it; a supervised one
7853    /// stamped `Direct` silently strips a module of capability it is entitled to.
7854    ///
7855    /// Neither shows up in a test that only checks the bind succeeded. Before this
7856    /// test the only coverage was accidental —
7857    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
7858    /// stamped principal on its way past, so narrowing that wire-shape test to its
7859    /// stated subject would have deleted the last assertion on this value. It
7860    /// still asserts the stamp, which is now redundancy rather than the only
7861    /// guard: both fail under the same mutation, and this one names the reason.
7862    /// SCOPE: this handler's supervisor has spawned nothing, so
7863    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
7864    /// is unreachable here. Both assertions below are refusals, and a mutant that
7865    /// refuses everything would satisfy them.
7866    ///
7867    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
7868    /// spawns a supervised consumer, reads its live nonce, and asserts the module
7869    /// observed `principal.kind == "reserved"` carrying that module_id — verified
7870    /// at source rather than assumed, since a citation is a claim about another
7871    /// file and ages like one. Recorded because a harness that structurally
7872    /// cannot reach an arm reports "none" for that arm identically to one that
7873    /// covers it and found nothing.
7874    #[tokio::test]
7875    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7876        let handler = ControlHandler::default();
7877        let frame =
7878            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7879
7880        // Absent consumer_identity is the ordinary case: a human at a terminal, or
7881        // any process holding the connection file. Nothing was proved, so nothing
7882        // may be granted beyond the unattested floor.
7883        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7884        assert_eq!(
7885            stamped,
7886            Principal::Direct,
7887            "a caller that proved nothing must not be stamped as a supervised module"
7888        );
7889
7890        // A claimed module_id with a nonce no supervised child was given is a
7891        // forgery attempt, not a weaker caller: it must be REFUSED rather than
7892        // quietly demoted to Direct, or an impersonation attempt looks identical
7893        // to an ordinary unattested connection.
7894        let forged = handler
7895            .route_open_principal(
7896                &frame,
7897                Some(ConsumerIdentity {
7898                    module_id: "aft".to_string(),
7899                    launch_nonce: "not-a-real-nonce".to_string(),
7900                }),
7901            )
7902            .unwrap();
7903        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7904        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7905    }
7906
7907    /// The test above hands `route_open_principal` an identity it built itself,
7908    /// which proves the stamping rule and nothing about where the identity comes
7909    /// from. The real producer is a wire body, and the two are joined by a serde
7910    /// field name that nothing else asserts.
7911    ///
7912    /// That join fails quietly in one specific way: an unrecognised key is simply
7913    /// absent after parsing, so a renamed or misspelled `consumer_identity`
7914    /// yields `None` and every supervised module silently drops to `Direct`.
7915    /// Capability-wise that is the safe direction, but it surfaces far from its
7916    /// cause — as a module mysteriously refused bash — and it would pass every
7917    /// test that builds its own input.
7918    ///
7919    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
7920    /// would break every client the moment the daemon gains a field, trading a
7921    /// quiet demotion for a hard refusal on additive change. Asserting the join
7922    /// instead means a rename breaks a test here rather than the fleet.
7923    #[test]
7924    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7925        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7926        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7927        let ClientControlRequest::RouteOpen {
7928            consumer_identity, ..
7929        } = parsed
7930        else {
7931            panic!("route.open body must parse as RouteOpen");
7932        };
7933        assert_eq!(
7934            consumer_identity,
7935            Some(ConsumerIdentity {
7936                module_id: "aft".to_string(),
7937                launch_nonce: "n".to_string(),
7938            }),
7939            "the wire field name must reach the value route_open_principal reads"
7940        );
7941    }
7942
7943    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7944        ModuleManifest::builder(module_id, "0.1.0")
7945            .protocol_ver(protocol_ver)
7946            .provides(vec![ProviderRole::ToolProvider {
7947                tools: vec![Tool {
7948                    name: "read".to_string(),
7949                    description: None,
7950                    execution_mode: ExecutionMode::Pure,
7951                    schema: json!({"type": "object"}),
7952                }],
7953                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7954                concurrency: Concurrency::ModuleManaged,
7955                emits_push: true,
7956                sub_supervises: true,
7957            }])
7958            .build()
7959    }
7960
7961    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7962        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7963    }
7964
7965    fn hello_frame_with_control_ops(
7966        module_id: &str,
7967        protocol_ver: u8,
7968        corr: u64,
7969        control_ops: Option<Vec<String>>,
7970    ) -> Frame {
7971        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7972    }
7973
7974    fn hello_frame_with_nonce(
7975        module_id: &str,
7976        protocol_ver: u8,
7977        corr: u64,
7978        launch_nonce: Option<&str>,
7979    ) -> Frame {
7980        hello_frame_full(
7981            module_id,
7982            protocol_ver,
7983            corr,
7984            None,
7985            launch_nonce.map(ToOwned::to_owned),
7986        )
7987    }
7988
7989    fn hello_frame_full(
7990        module_id: &str,
7991        protocol_ver: u8,
7992        corr: u64,
7993        control_ops: Option<Vec<String>>,
7994        launch_nonce: Option<String>,
7995    ) -> Frame {
7996        let body = serde_json::to_vec(&ModuleHelloBody {
7997            manifest: manifest(module_id, protocol_ver),
7998            protocol_ver,
7999            control_ops,
8000            launch_nonce,
8001        })
8002        .unwrap();
8003        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8004    }
8005
8006    fn non_routable_hello_frame_with_control_ops(
8007        module_id: &str,
8008        corr: u64,
8009        control_ops: Option<Vec<String>>,
8010    ) -> Frame {
8011        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8012        manifest.provides.clear();
8013        let body = serde_json::to_vec(&ModuleHelloBody {
8014            manifest,
8015            protocol_ver: PROTOCOL_VERSION,
8016            control_ops,
8017            launch_nonce: None,
8018        })
8019        .unwrap();
8020        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8021    }
8022
8023    fn capability_grammar_hello_frame(
8024        capabilities: Value,
8025        runtime_computed: Option<Value>,
8026        corr: u64,
8027    ) -> Frame {
8028        let mut body = serde_json::to_value(ModuleHelloBody {
8029            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8030            protocol_ver: PROTOCOL_VERSION,
8031            control_ops: None,
8032            launch_nonce: None,
8033        })
8034        .expect("HELLO body serializes");
8035        body["manifest"]["capabilities"] = capabilities;
8036        if let Some(runtime_computed) = runtime_computed {
8037            body["runtime_computed"] = runtime_computed;
8038        }
8039        Frame::build(
8040            FrameType::Hello,
8041            control_flags(),
8042            0,
8043            0,
8044            corr,
8045            serde_json::to_vec(&body).expect("HELLO body reserializes"),
8046        )
8047        .expect("HELLO frame builds")
8048    }
8049
8050    fn channel_request(channel: u16, corr: u64) -> Frame {
8051        Frame::build(
8052            FrameType::Request,
8053            Flags::new(true, Priority::Interactive, false),
8054            channel,
8055            0,
8056            corr,
8057            b"opaque".to_vec(),
8058        )
8059        .unwrap()
8060    }
8061
8062    fn route_ctx(
8063        connection_id: ConnectionId,
8064    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8065        let (tx, rx) = mpsc::channel(8);
8066        (
8067            RouteCtx {
8068                connection_id,
8069                egress: FrameSink::new(tx),
8070            },
8071            rx,
8072        )
8073    }
8074
8075    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8076        serde_json::from_slice(&frame.body).unwrap()
8077    }
8078
8079    /// Register a module over a connection that has a sink and return the
8080    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
8081    /// module's own sink rather than returning it as a reply, so the ack is
8082    /// taken off `rx` here and whatever the test reads next is what followed it.
8083    async fn hello_via_sink(
8084        handler: &ControlHandler,
8085        ctx: &RouteCtx,
8086        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8087        hello: Frame,
8088    ) -> Frame {
8089        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8090        assert!(
8091            replies.is_empty(),
8092            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8093        );
8094        let ack = rx
8095            .try_recv()
8096            .expect("HELLO_ACK is queued on the module sink")
8097            .frame;
8098        assert_eq!(ack.header.ty, FrameType::HelloAck);
8099        ack
8100    }
8101
8102    fn parse_error(frame: &Frame) -> Value {
8103        serde_json::from_slice(&frame.body).unwrap()
8104    }
8105
8106    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8107        serde_json::from_slice(&frame.body).unwrap()
8108    }
8109
8110    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8111        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8112            route_channel,
8113            route_epoch: 0,
8114            kind,
8115        })
8116        .unwrap();
8117        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8118    }
8119
8120    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8121        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8122            module_id: module_id.to_string(),
8123        })
8124        .unwrap();
8125        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8126    }
8127
8128    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8129        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8130    }
8131
8132    fn route_open_frame_with_consumer_capabilities(
8133        corr: u64,
8134        module_id: &str,
8135        project_root: ScratchDir,
8136        consumer_capabilities: Option<Vec<String>>,
8137    ) -> Frame {
8138        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8139            target: RouteTarget::ToolProvider {
8140                module_id: module_id.to_string(),
8141            },
8142            identity: BindIdentity::new(
8143                project_root.path().to_path_buf(),
8144                "unit".to_string(),
8145                "session".to_string(),
8146            ),
8147            consumer_identity: None,
8148            consumer_capabilities,
8149            role_versions: None,
8150            admission_facts: None,
8151            scope: None,
8152        })
8153        .unwrap();
8154        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8155    }
8156
8157    fn route_open_frame_with_role_versions(
8158        corr: u64,
8159        module_id: &str,
8160        project_root: ScratchDir,
8161        role_versions: Option<BTreeMap<String, String>>,
8162    ) -> Frame {
8163        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8164            target: RouteTarget::ToolProvider {
8165                module_id: module_id.to_string(),
8166            },
8167            identity: BindIdentity::new(
8168                project_root.path().to_path_buf(),
8169                "unit".to_string(),
8170                format!("session-{corr}"),
8171            ),
8172            consumer_identity: None,
8173            consumer_capabilities: None,
8174            role_versions,
8175            admission_facts: None,
8176            scope: None,
8177        })
8178        .unwrap();
8179        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8180    }
8181
8182    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8183        entries
8184            .iter()
8185            .map(|(role, version)| (role.to_string(), version.to_string()))
8186            .collect()
8187    }
8188
8189    fn route_open_frame_with_admission_facts(
8190        corr: u64,
8191        module_id: &str,
8192        project_root: ScratchDir,
8193        consumer_identity: Option<subc_control::ConsumerIdentity>,
8194        facts: Option<Value>,
8195    ) -> Frame {
8196        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8197            target: RouteTarget::ToolProvider {
8198                module_id: module_id.to_string(),
8199            },
8200            identity: BindIdentity::new(
8201                project_root.path().to_path_buf(),
8202                "unit".to_string(),
8203                format!("session-{corr}"),
8204            ),
8205            consumer_identity,
8206            consumer_capabilities: None,
8207            role_versions: None,
8208            admission_facts: facts,
8209            scope: None,
8210        })
8211        .unwrap();
8212        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8213    }
8214
8215    #[derive(Clone, Default)]
8216    struct EventCapture {
8217        events: Arc<Mutex<Vec<CapturedEvent>>>,
8218    }
8219
8220    #[derive(Clone, Debug)]
8221    struct CapturedEvent {
8222        target: String,
8223        level: tracing::Level,
8224        fields: BTreeMap<String, String>,
8225    }
8226
8227    impl EventCapture {
8228        fn events(&self) -> Vec<CapturedEvent> {
8229            self.events.lock().unwrap().clone()
8230        }
8231    }
8232
8233    impl<S> Layer<S> for EventCapture
8234    where
8235        S: Subscriber,
8236    {
8237        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8238            let mut visitor = EventFieldVisitor::default();
8239            event.record(&mut visitor);
8240            self.events.lock().unwrap().push(CapturedEvent {
8241                target: event.metadata().target().to_string(),
8242                level: *event.metadata().level(),
8243                fields: visitor.fields,
8244            });
8245        }
8246    }
8247
8248    #[derive(Default)]
8249    struct EventFieldVisitor {
8250        fields: BTreeMap<String, String>,
8251    }
8252
8253    impl Visit for EventFieldVisitor {
8254        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8255            self.fields
8256                .insert(field.name().to_string(), format!("{value:?}"));
8257        }
8258    }
8259
8260    fn health_response(corr: u64, status: HealthStatus) -> Frame {
8261        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8262            status,
8263            detail: Some("warming".to_string()),
8264            metrics: Some(json!({"queue_depth": 3})),
8265        })
8266        .unwrap();
8267        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8268    }
8269
8270    fn route_bind_ack(corr: u64) -> Frame {
8271        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8272        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8273    }
8274
8275    fn unique_project_root(label: &str) -> ScratchDir {
8276        ScratchDir::new(label)
8277    }
8278
8279    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8280        match parse_route_poll(frame) {
8281            ClientControlResponse::RoutePoll {
8282                status: None,
8283                live: Some(live),
8284                ..
8285            } => assert_eq!(live, expected_live),
8286            other => panic!("unexpected route.poll response: {other:?}"),
8287        }
8288    }
8289
8290    fn bind_liveness_route(
8291        registry: &Registry,
8292        forwarding: &ForwardingTable,
8293        module_id: &str,
8294    ) -> (RouteCtx, u16, u32) {
8295        let module_connection = ConnectionId::new(101);
8296        let client_connection = ConnectionId::new(202);
8297        let registration = registry
8298            .register_with_control_ops(
8299                manifest(module_id, PROTOCOL_VERSION),
8300                PROTOCOL_VERSION,
8301                module_connection,
8302                module_baseline_control_ops(),
8303            )
8304            .unwrap();
8305        let (module_tx, _module_rx) = mpsc::channel(8);
8306        let endpoint = forwarding
8307            .register_module_connection(
8308                module_connection,
8309                module_id.to_string(),
8310                PROTOCOL_VERSION,
8311                manifest_concurrency(&registration.manifest),
8312                FrameSink::new(module_tx),
8313            )
8314            .unwrap();
8315        let (client_ctx, _client_rx) = route_ctx(client_connection);
8316        let pending = forwarding
8317            .begin_route_bind_relay_for_test(
8318                client_connection,
8319                client_ctx.egress.clone(),
8320                1,
8321                module_id,
8322            )
8323            .unwrap();
8324        assert_eq!(pending.endpoint, endpoint);
8325        let route_channel = pending.client_channel;
8326        let route_epoch = pending.client_epoch;
8327        forwarding
8328            .complete_pending_relay(
8329                module_connection,
8330                pending.corr,
8331                RouteBindRelayOutcome::Accepted,
8332            )
8333            .unwrap();
8334        (client_ctx, route_channel, route_epoch)
8335    }
8336
8337    struct FakeProcessLiveness {
8338        live: Option<bool>,
8339    }
8340
8341    impl ModuleProcessLiveness for FakeProcessLiveness {
8342        fn process_live(&self, _module_id: &str) -> Option<bool> {
8343            self.live
8344        }
8345    }
8346
8347    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8348    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8349    {
8350        let registry = Arc::new(Registry::default());
8351        let supervisor_handle = SupervisorHandle::new();
8352        let supervisor = Supervisor::new_for_test(
8353            Arc::clone(&registry),
8354            RestartPolicy::new(1, Duration::from_millis(10)),
8355        )
8356        .with_handle(supervisor_handle.clone());
8357        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8358        let module = supervisor
8359            .spawn(ModuleSpec {
8360                module_id: "stderr-tail-wire".to_string(),
8361                program: fake_aft_stub_path(),
8362                args: Vec::new(),
8363                env: vec![
8364                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8365                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8366                ],
8367                reserved: false,
8368                reserved_prefixes: Vec::new(),
8369                protocol: ModuleProtocol::Subc,
8370                overlap: Default::default(),
8371            })
8372            .unwrap();
8373
8374        let deadline = Instant::now() + Duration::from_secs(5);
8375        loop {
8376            let tail = module.stderr_tail(None, None);
8377            if tail
8378                .entries
8379                .iter()
8380                .any(|entry| matches!(entry, TailEntry::ProcessStart))
8381                && tail.entries.iter().any(|entry| {
8382                    matches!(
8383                        entry,
8384                        TailEntry::Line {
8385                            truncated: true,
8386                            ..
8387                        }
8388                    )
8389                })
8390            {
8391                break;
8392            }
8393            assert!(
8394                Instant::now() < deadline,
8395                "module did not produce a truncated line and restart boundary: {tail:?}"
8396            );
8397            sleep(Duration::from_millis(10)).await;
8398        }
8399
8400        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8401        let request = ClientControlRequest::SupervisorStderrTail {
8402            module_id: "stderr-tail-wire".to_string(),
8403            max_lines: None,
8404            max_bytes: None,
8405        };
8406        let frame = Frame::build(
8407            FrameType::Request,
8408            control_flags(),
8409            0,
8410            0,
8411            1,
8412            serde_json::to_vec(&request).unwrap(),
8413        )
8414        .unwrap();
8415        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8416        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8417        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8418            serde_json::from_slice(&responses[0].body).unwrap()
8419        else {
8420            panic!("expected supervisor.stderr_tail response");
8421        };
8422
8423        assert!(
8424            tail.entries
8425                .iter()
8426                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8427            "the control response lost the restart boundary"
8428        );
8429        let Some(StderrTailEntry::Line {
8430            text,
8431            truncated,
8432            at_ms,
8433        }) = tail.entries.iter().find(|entry| {
8434            matches!(
8435                entry,
8436                StderrTailEntry::Line {
8437                    truncated: true,
8438                    ..
8439                }
8440            )
8441        })
8442        else {
8443            panic!("the control response lost the truncated line");
8444        };
8445        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8446        assert!(*truncated);
8447        assert!(
8448            at_ms.is_some(),
8449            "the control response lost the line's capture time"
8450        );
8451    }
8452
8453    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
8454    /// read done on the worker thread would stall every other task until it
8455    /// finished; the read must run off the worker so this test's own task keeps
8456    /// running while the read is paused.
8457    #[tokio::test(flavor = "current_thread")]
8458    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8459        let dir = ScratchDir::new("terminals-off-worker");
8460        let journal_path = dir.join("terminals.jsonl");
8461        let registry = Arc::new(Registry::default());
8462        let supervisor_handle = SupervisorHandle::new();
8463        let supervisor =
8464            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8465                .with_handle(supervisor_handle.clone())
8466                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8467        let module = supervisor
8468            .spawn(ModuleSpec {
8469                module_id: "terminal-off-worker".to_string(),
8470                program: fake_aft_stub_path(),
8471                args: Vec::new(),
8472                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8473                reserved: false,
8474                reserved_prefixes: Vec::new(),
8475                protocol: ModuleProtocol::Subc,
8476                overlap: Default::default(),
8477            })
8478            .unwrap();
8479        let deadline = Instant::now() + Duration::from_secs(5);
8480        while module.terminal_history().entries.len() != 2 {
8481            assert!(Instant::now() < deadline, "module did not record two exits");
8482            sleep(Duration::from_millis(10)).await;
8483        }
8484
8485        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8486        let handler =
8487            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
8488        let frame = Frame::build(
8489            FrameType::Request,
8490            control_flags(),
8491            0,
8492            0,
8493            1,
8494            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8495                module_id: "terminal-off-worker".to_string(),
8496            })
8497            .unwrap(),
8498        )
8499        .unwrap();
8500        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8501        let spawned_at = std::time::Instant::now();
8502        let read = tokio::spawn({
8503            let handler = Arc::clone(&handler);
8504            async move { handler.handle_control_frame(&ctx, frame).await }
8505        });
8506        // Waiting for the pause from a blocking thread keeps this task pending,
8507        // so the runtime's single worker is free to run the read task.
8508        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8509            .await
8510            .unwrap()
8511            .expect("the history read reached its pause");
8512        let elapsed = spawned_at.elapsed();
8513        assert!(
8514            elapsed < Duration::from_secs(2) && !read.is_finished(),
8515            "this task could not run while the history read was paused \
8516             (resumed after {elapsed:?}, read finished: {})",
8517            read.is_finished()
8518        );
8519
8520        drop(release);
8521        let responses = read.await.unwrap().unwrap();
8522        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8523        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8524            panic!("expected supervisor.terminals response");
8525        };
8526        assert_eq!(terminals.entries.len(), 2);
8527        assert_eq!(terminals.journal_skipped_lines, 0);
8528        assert_eq!(terminals.journal_read_errors, 0);
8529    }
8530
8531    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8532    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8533        let registry = Arc::new(Registry::default());
8534        let supervisor_handle = SupervisorHandle::new();
8535        let supervisor =
8536            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8537                .with_handle(supervisor_handle.clone());
8538        let module = supervisor
8539            .spawn(ModuleSpec {
8540                module_id: "terminal-golden".to_string(),
8541                program: fake_aft_stub_path(),
8542                args: Vec::new(),
8543                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8544                reserved: false,
8545                reserved_prefixes: Vec::new(),
8546                protocol: ModuleProtocol::Subc,
8547                overlap: Default::default(),
8548            })
8549            .unwrap();
8550
8551        let deadline = Instant::now() + Duration::from_secs(5);
8552        while module.terminal_history().entries.len() != 2 {
8553            assert!(
8554                Instant::now() < deadline,
8555                "module did not retain two terminal exits: {:?}",
8556                module.terminal_history()
8557            );
8558            sleep(Duration::from_millis(10)).await;
8559        }
8560
8561        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8562        let request = ClientControlRequest::SupervisorTerminals {
8563            module_id: "terminal-golden".to_string(),
8564        };
8565        let frame = Frame::build(
8566            FrameType::Request,
8567            control_flags(),
8568            0,
8569            0,
8570            1,
8571            serde_json::to_vec(&request).unwrap(),
8572        )
8573        .unwrap();
8574        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8575        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8576        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8577        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8578            panic!("expected supervisor.terminals response");
8579        };
8580        assert_eq!(terminals.entries.len(), 2);
8581        assert_eq!(terminals.dropped, 0);
8582
8583        let mut rendered = serde_json::to_value(response).unwrap();
8584        // Wall-clock fields are the observation contract, but not stable fixture
8585        // bytes; normalize only them after the real handler has shaped the response.
8586        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8587        for (index, entry) in rendered["entries"]
8588            .as_array_mut()
8589            .expect("terminal response entries array")
8590            .iter_mut()
8591            .enumerate()
8592        {
8593            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8594        }
8595
8596        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8597            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8598        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8599        if std::env::var_os("UPDATE_GOLDEN").is_some() {
8600            std::fs::write(&golden_path, &serialized).unwrap();
8601        }
8602        let expected: Value =
8603            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8604        assert_eq!(rendered, expected);
8605    }
8606
8607    #[test]
8608    fn hello_registers_manifest_and_returns_ack() {
8609        let registry = Arc::new(Registry::default());
8610        let handler = ControlHandler::new(Arc::clone(&registry));
8611        let conn = ConnectionId::new(1);
8612
8613        let responses = handler
8614            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8615            .unwrap();
8616
8617        assert_eq!(responses.len(), 1);
8618        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8619        assert_eq!(responses[0].header.channel, 0);
8620        assert_eq!(responses[0].header.corr, 7);
8621        let ack = parse_ack(&responses[0]);
8622        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8623        assert!(ack
8624            .subc_capabilities
8625            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8626        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8627        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8628        assert!(ack
8629            .subc_ops
8630            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8631        assert!(ack
8632            .subc_ops
8633            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8634
8635        let registration = registry.get_module("aft").unwrap().unwrap();
8636        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8637        assert_eq!(registration.state, ChannelState::Active);
8638        assert_eq!(registration.connection_id, conn);
8639        assert_eq!(registration.control_ops, module_baseline_control_ops());
8640    }
8641
8642    #[test]
8643    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8644        let invalid_identifiers = [
8645            ("case_change", "credentials-Provider/v1"),
8646            ("leading_zero", "credentials-provider/v01"),
8647            ("trailing_hyphen", "credentials-provider-/v1"),
8648            ("consecutive_hyphens", "credentials--provider/v1"),
8649            ("uppercase", "Credentials-provider/v1"),
8650            ("missing_v", "credentials-provider/1"),
8651            ("whitespace", "credentials provider/v1"),
8652            ("zero_version", "credentials-provider/v0"),
8653            ("out_of_range_version", "credentials-provider/v4294967296"),
8654            (
8655                "overlength_name",
8656                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8657            ),
8658        ];
8659        let mut cases = invalid_identifiers
8660            .into_iter()
8661            .map(|(name, identifier)| {
8662                (
8663                    format!("identifier_{name}"),
8664                    "capabilities.provides[0]".to_string(),
8665                    identifier.to_string(),
8666                    json!({ "provides": [identifier] }),
8667                    None,
8668                )
8669            })
8670            .collect::<Vec<_>>();
8671        cases.extend([
8672            (
8673                "unknown_need".to_string(),
8674                "capabilities.requires[0].need".to_string(),
8675                "deferred".to_string(),
8676                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8677                None,
8678            ),
8679            (
8680                "duplicate_provides".to_string(),
8681                "capabilities.provides[1]".to_string(),
8682                "credentials-provider/v1".to_string(),
8683                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8684                None,
8685            ),
8686            (
8687                "duplicate_must_never_reach".to_string(),
8688                "capabilities.must_never_reach[1]".to_string(),
8689                "credentials-provider/v1".to_string(),
8690                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8691                None,
8692            ),
8693            (
8694                "duplicate_requires_same_need".to_string(),
8695                "capabilities.requires[1]".to_string(),
8696                "credentials-provider/v1".to_string(),
8697                json!({ "requires": [
8698                    { "capability": "credentials-provider/v1", "need": "required" },
8699                    { "capability": "credentials-provider/v1", "need": "required" }
8700                ] }),
8701                None,
8702            ),
8703            (
8704                "duplicate_requires_conflicting_need".to_string(),
8705                "capabilities.requires[1]".to_string(),
8706                "credentials-provider/v1".to_string(),
8707                json!({ "requires": [
8708                    { "capability": "credentials-provider/v1", "need": "required" },
8709                    { "capability": "credentials-provider/v1", "need": "optional" }
8710                ] }),
8711                None,
8712            ),
8713            (
8714                "capabilities_root_pointer".to_string(),
8715                "runtime_computed[0]".to_string(),
8716                "/capabilities".to_string(),
8717                json!({}),
8718                Some(json!(["/capabilities"])),
8719            ),
8720            (
8721                "capabilities_descendant_pointer".to_string(),
8722                "runtime_computed[0]".to_string(),
8723                "/capabilities/provides".to_string(),
8724                json!({}),
8725                Some(json!(["/capabilities/provides"])),
8726            ),
8727            (
8728                "malformed_pointer_without_leading_slash".to_string(),
8729                "runtime_computed[0]".to_string(),
8730                "capabilities".to_string(),
8731                json!({}),
8732                Some(json!(["capabilities"])),
8733            ),
8734            (
8735                "malformed_pointer_escape".to_string(),
8736                "runtime_computed[0]".to_string(),
8737                "/roles/~2/tools".to_string(),
8738                json!({}),
8739                Some(json!(["/roles/~2/tools"])),
8740            ),
8741            (
8742                "unknown_capabilities_field".to_string(),
8743                "capabilities.future".to_string(),
8744                "<array>".to_string(),
8745                json!({ "future": [] }),
8746                None,
8747            ),
8748        ]);
8749
8750        for (index, (name, field, value, capabilities, runtime_computed)) in
8751            cases.into_iter().enumerate()
8752        {
8753            let registry = Arc::new(Registry::default());
8754            let handler = ControlHandler::new(Arc::clone(&registry));
8755            let response = handler
8756                .handle_control(
8757                    ConnectionId::new((index + 1) as u64),
8758                    capability_grammar_hello_frame(
8759                        capabilities,
8760                        runtime_computed,
8761                        index as u64 + 1,
8762                    ),
8763                )
8764                .expect("invalid HELLO returns a refusal");
8765
8766            assert_eq!(response.len(), 1, "{name} must emit one refusal");
8767            let error = parse_error(&response[0]);
8768            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8769            let message = error["message"]
8770                .as_str()
8771                .expect("error message is a string");
8772            assert!(
8773                message.contains(&field),
8774                "{name}: field missing from {message}"
8775            );
8776            assert!(
8777                message.contains(&value),
8778                "{name}: value missing from {message}"
8779            );
8780            assert_eq!(
8781                registry
8782                    .active_registration_count()
8783                    .expect("registry reads"),
8784                0,
8785                "{name}: refused HELLO must not create a catalog entry"
8786            );
8787        }
8788    }
8789
8790    #[test]
8791    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8792        let registry = Arc::new(Registry::default());
8793        let handler = ControlHandler::new(Arc::clone(&registry));
8794        let capabilities = json!({
8795            "provides": ["credentials-provider/v1"],
8796            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8797            "must_never_reach": ["federation-transport/v1"]
8798        });
8799        let response = handler
8800            .handle_control(
8801                ConnectionId::new(99),
8802                capability_grammar_hello_frame(
8803                    capabilities.clone(),
8804                    Some(json!(["/roles/0/tools"])),
8805                    99,
8806                ),
8807            )
8808            .expect("valid HELLO registers");
8809        assert_eq!(response[0].header.ty, FrameType::HelloAck);
8810
8811        let request = Frame::build(
8812            FrameType::Request,
8813            control_flags(),
8814            0,
8815            0,
8816            100,
8817            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8818                .expect("catalog request serializes"),
8819        )
8820        .expect("catalog request frame builds");
8821        let response = handler
8822            .handle_catalog_list(request, None)
8823            .expect("catalog list succeeds");
8824        let ClientControlResponse::CatalogList { modules, .. } =
8825            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8826        else {
8827            panic!("catalog request must return catalog.list");
8828        };
8829        assert_eq!(modules.len(), 1);
8830        assert_eq!(
8831            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8832            capabilities
8833        );
8834    }
8835
8836    #[test]
8837    fn catalog_list_mirrors_management_operation_description() {
8838        let registry = Arc::new(Registry::default());
8839        let handler = ControlHandler::new(Arc::clone(&registry));
8840        let description = "List managed records and return their identifiers and metadata.";
8841        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8842        manifest.provides = vec![ProviderRole::ManagementSurface {
8843            operations: vec![ManagementOperation {
8844                name: "records.list".to_string(),
8845                kind: ManagementOperationKind::Query,
8846                description: Some(description.to_string()),
8847            }],
8848            config_schema: json!({"type": "object"}),
8849            observability: vec![ObservabilitySurface {
8850                name: "records.stats".to_string(),
8851                kind: ObservabilityKind::Snapshot,
8852            }],
8853            identity_scope: vec![IdentityScope::Project],
8854            concurrency: Concurrency::ModuleManaged,
8855        }];
8856        registry
8857            .register_with_control_ops(
8858                manifest,
8859                PROTOCOL_VERSION,
8860                ConnectionId::new(99),
8861                Vec::new(),
8862            )
8863            .expect("described management manifest registers");
8864
8865        let request = Frame::build(
8866            FrameType::Request,
8867            control_flags(),
8868            0,
8869            0,
8870            100,
8871            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8872                .expect("catalog request serializes"),
8873        )
8874        .expect("catalog request frame builds");
8875        let response = handler
8876            .handle_catalog_list(request, None)
8877            .expect("catalog list succeeds");
8878        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8879        assert_eq!(
8880            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8881            "catalog.list must preserve the declared operation description verbatim"
8882        );
8883    }
8884
8885    #[test]
8886    fn catalog_list_mirrors_registered_operation_delivery() {
8887        let registry = Arc::new(Registry::default());
8888        let handler = ControlHandler::new(Arc::clone(&registry));
8889        let delivery =
8890            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
8891        let mut body = serde_json::to_value(ModuleHelloBody {
8892            manifest: manifest("watch-management", PROTOCOL_VERSION),
8893            protocol_ver: PROTOCOL_VERSION,
8894            control_ops: None,
8895            launch_nonce: None,
8896        })
8897        .unwrap();
8898        body["manifest"]["provides"] = json!([{
8899            "role": "management_surface",
8900            "operations": [
8901                {"name": "status", "kind": "query"},
8902                {"name": "future", "kind": "query"},
8903                {"name": "reset", "kind": "mutate"}
8904            ],
8905            "config_schema": {}, "observability": [], "identity_scope": []
8906        }]);
8907        body["manifest"]["operation_delivery"] = delivery.clone();
8908        let hello = Frame::build(
8909            FrameType::Hello,
8910            control_flags(),
8911            0,
8912            0,
8913            99,
8914            serde_json::to_vec(&body).unwrap(),
8915        )
8916        .unwrap();
8917        let response = handler
8918            .handle_control(ConnectionId::new(99), hello)
8919            .unwrap();
8920        assert_eq!(response[0].header.ty, FrameType::HelloAck);
8921
8922        let (_, registrations) = registry.list_modules().unwrap();
8923        let registered = &registrations[0].manifest;
8924        assert!(registered.is_watch_operation("status"));
8925        assert!(!registered.is_watch_operation("future"));
8926        assert!(!registered.is_watch_operation("reset"));
8927        assert!(!registered.is_watch_operation("missing"));
8928
8929        let request = Frame::build(
8930            FrameType::Request,
8931            control_flags(),
8932            0,
8933            0,
8934            100,
8935            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
8936        )
8937        .unwrap();
8938        let response = handler.handle_catalog_list(request, None).unwrap();
8939        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
8940        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
8941        let ClientControlResponse::CatalogList { modules, .. } =
8942            serde_json::from_slice(&response[0].body).unwrap()
8943        else {
8944            panic!("expected catalog.list");
8945        };
8946        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
8947    }
8948
8949    #[test]
8950    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8951        let registry = Arc::new(Registry::default());
8952        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
8953            [("vault".to_string(), true), ("squatter".to_string(), true)],
8954            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8955        );
8956        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8957        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8958            provides: vec!["credentials-provider/v1".to_string()],
8959            requires: Vec::new(),
8960            must_never_reach: Vec::new(),
8961        });
8962        let frame = Frame::build(
8963            FrameType::Hello,
8964            control_flags(),
8965            0,
8966            0,
8967            77,
8968            serde_json::to_vec(&ModuleHelloBody {
8969                manifest: squatter,
8970                protocol_ver: PROTOCOL_VERSION,
8971                control_ops: None,
8972                launch_nonce: None,
8973            })
8974            .expect("HELLO serializes"),
8975        )
8976        .expect("HELLO frame builds");
8977        let response = handler
8978            .handle_control(ConnectionId::new(77), frame)
8979            .expect("reserved claim receives a typed refusal");
8980        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8981        assert_eq!(
8982            registry
8983                .active_registration_count()
8984                .expect("registry reads"),
8985            0,
8986            "a reserved capability refusal must not leave a catalog entry"
8987        );
8988    }
8989
8990    #[test]
8991    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8992        for settlement in ["timeout", "inconclusive", "drop"] {
8993            let breakers = RouteBindBreakers::default();
8994            let RouteBindAdmission::Admitted {
8995                guard: mut old,
8996                probe: false,
8997            } = breakers.admit("prov")
8998            else {
8999                panic!("ordinary relay admitted")
9000            };
9001            let RouteBindAdmission::Admitted {
9002                guard: mut opener, ..
9003            } = breakers.admit("prov")
9004            else {
9005                panic!("second relay admitted")
9006            };
9007            assert!(
9008                !opener
9009                    .record_timeout(1, Duration::ZERO)
9010                    .unwrap()
9011                    .reopened_after_probe
9012            );
9013            let RouteBindAdmission::Admitted {
9014                guard: mut probe,
9015                probe: true,
9016            } = breakers.admit("prov")
9017            else {
9018                panic!("one cooldown probe admitted")
9019            };
9020            match settlement {
9021                "timeout" => assert!(
9022                    !old.record_timeout(1, Duration::ZERO)
9023                        .unwrap()
9024                        .reopened_after_probe
9025                ),
9026                "inconclusive" => old.record_inconclusive(),
9027                "drop" => drop(old),
9028                _ => unreachable!(),
9029            }
9030            assert!(
9031                matches!(
9032                    breakers.admit("prov"),
9033                    RouteBindAdmission::Refused {
9034                        probe_in_flight: true,
9035                        ..
9036                    }
9037                ),
9038                "{settlement} of a pre-open relay cannot release the real probe"
9039            );
9040            assert!(
9041                probe
9042                    .record_timeout(1, Duration::ZERO)
9043                    .unwrap()
9044                    .reopened_after_probe
9045            );
9046            assert!(matches!(
9047                breakers.admit("prov"),
9048                RouteBindAdmission::Admitted { probe: true, .. }
9049            ));
9050        }
9051        let breakers = RouteBindBreakers::default();
9052        let admit = || match breakers.admit("prov") {
9053            RouteBindAdmission::Admitted { guard, .. } => guard,
9054            _ => panic!("relay admitted"),
9055        };
9056        admit().record_timeout(1, Duration::ZERO);
9057        let mut old_probe = admit();
9058        breakers.reset_for_new_module_connection("prov");
9059        admit().record_timeout(1, Duration::ZERO);
9060        let _new_probe = admit();
9061        old_probe.record_inconclusive();
9062        assert!(matches!(
9063            breakers.admit("prov"),
9064            RouteBindAdmission::Refused {
9065                probe_in_flight: true,
9066                ..
9067            }
9068        ));
9069    }
9070
9071    #[tokio::test]
9072    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9073        for candidate in [false, true] {
9074            let registry = Arc::new(Registry::default());
9075            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9076                [("vault".to_string(), true), ("squatter".to_string(), true)],
9077                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9078            );
9079            let conn = ConnectionId::new(77);
9080            let (ctx, mut rx) = route_ctx(conn);
9081            let initial = capability_manifest("squatter", &[], &[]);
9082            if candidate {
9083                registry
9084                    .register_candidate_with_control_ops(
9085                        initial.clone(),
9086                        PROTOCOL_VERSION,
9087                        conn,
9088                        module_baseline_control_ops(),
9089                    )
9090                    .unwrap();
9091                handler
9092                    .forwarding
9093                    .register_candidate_module_connection(
9094                        conn,
9095                        "squatter".to_string(),
9096                        PROTOCOL_VERSION,
9097                        manifest_concurrency(&initial),
9098                        ctx.egress.clone(),
9099                    )
9100                    .unwrap();
9101            } else {
9102                hello_via_sink(
9103                    &handler,
9104                    &ctx,
9105                    &mut rx,
9106                    hello_frame_with_manifest(initial.clone(), 1),
9107                )
9108                .await;
9109            }
9110            let response = handler
9111                .handle_control_frame(
9112                    &ctx,
9113                    catalog_update_with_capabilities_frame(
9114                        2,
9115                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
9116                            .capabilities
9117                            .unwrap(),
9118                    ),
9119                )
9120                .await
9121                .unwrap();
9122            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9123            assert_eq!(
9124                registry
9125                    .get_module_by_connection(conn)
9126                    .unwrap()
9127                    .unwrap()
9128                    .manifest,
9129                initial
9130            );
9131        }
9132    }
9133
9134    #[test]
9135    fn server_describe_surfaces_required_capability_verdict_fields() {
9136        let registry = Arc::new(Registry::default());
9137        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
9138            [
9139                ("consumer".to_string(), true),
9140                ("provider".to_string(), false),
9141            ],
9142            BTreeMap::new(),
9143        );
9144        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9145        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9146            provides: Vec::new(),
9147            requires: vec![subc_protocol::manifest::CapabilityRequirement {
9148                capability: "credentials-provider/v1".to_string(),
9149                need: subc_protocol::manifest::CapabilityNeed::Required,
9150            }],
9151            must_never_reach: Vec::new(),
9152        });
9153        let hello = Frame::build(
9154            FrameType::Hello,
9155            control_flags(),
9156            0,
9157            0,
9158            78,
9159            serde_json::to_vec(&ModuleHelloBody {
9160                manifest: consumer,
9161                protocol_ver: PROTOCOL_VERSION,
9162                control_ops: None,
9163                launch_nonce: None,
9164            })
9165            .expect("HELLO serializes"),
9166        )
9167        .expect("HELLO frame builds");
9168        handler
9169            .handle_control(ConnectionId::new(78), hello)
9170            .expect("consumer registers");
9171        let describe = Frame::build(
9172            FrameType::Request,
9173            control_flags(),
9174            0,
9175            0,
9176            79,
9177            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9178                .expect("request serializes"),
9179        )
9180        .expect("describe frame builds");
9181        let response = handler
9182            .handle_server_describe(describe)
9183            .expect("server.describe succeeds");
9184        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9185        let requirement = &rendered["capability_requirements"][0];
9186        assert_eq!(requirement["consumer"], "consumer");
9187        assert_eq!(requirement["verdict"], "never_provided");
9188        assert_eq!(requirement["episode_seq"], 1);
9189        assert_eq!(requirement["config_satisfiable"], false);
9190        assert_eq!(requirement["runtime_available"], false);
9191        assert!(requirement["detail"]
9192            .as_str()
9193            .expect("detail string")
9194            .contains("credentials-provider/v1"));
9195    }
9196
9197    /// Register a consumer that requires `project-identity/v1` and the provider
9198    /// of it, optionally mark daemon shutdown as begun, then deregister the
9199    /// provider as its connection closes. Returns the `capability.requirement`
9200    /// events that deregistration logged and the verdict the evaluator then
9201    /// holds for the requirement (the one `route.open` refusals read).
9202    fn deregister_required_provider(
9203        daemon_shutdown_begun: bool,
9204    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
9205        const CAPABILITY: &str = "project-identity/v1";
9206        let registry = Arc::new(Registry::default());
9207        let daemon_shutdown = DaemonShutdownFlag::default();
9208        let handler = ControlHandler::new(Arc::clone(&registry))
9209            .with_capability_config(
9210                [
9211                    ("consumer".to_string(), true),
9212                    ("provider".to_string(), true),
9213                ],
9214                BTreeMap::new(),
9215            )
9216            .with_daemon_shutdown_flag(daemon_shutdown.clone());
9217        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9218        consumer.capabilities = Some(CapabilityDeclarations {
9219            provides: Vec::new(),
9220            requires: vec![subc_protocol::manifest::CapabilityRequirement {
9221                capability: CAPABILITY.to_string(),
9222                need: subc_protocol::manifest::CapabilityNeed::Required,
9223            }],
9224            must_never_reach: Vec::new(),
9225        });
9226        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
9227        let provider_conn = ConnectionId::new(182);
9228        for (conn, manifest, corr) in [
9229            (ConnectionId::new(181), consumer, 181),
9230            (provider_conn, provider, 182),
9231        ] {
9232            let hello = handler
9233                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
9234                .expect("module registers");
9235            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9236        }
9237        assert_eq!(
9238            handler.capability_evaluator.verdict("consumer", CAPABILITY),
9239            Some(CapabilityVerdict::Provided),
9240            "the registered provider must satisfy the requirement first"
9241        );
9242        if daemon_shutdown_begun {
9243            daemon_shutdown.set_for_test();
9244        }
9245
9246        let capture = EventCapture::default();
9247        let _subscriber =
9248            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9249        handler
9250            .cleanup_connection(provider_conn)
9251            .expect("provider deregisters");
9252        let events = capture
9253            .events()
9254            .into_iter()
9255            .filter(|event| {
9256                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
9257            })
9258            .collect();
9259        (
9260            events,
9261            handler.capability_evaluator.verdict("consumer", CAPABILITY),
9262        )
9263    }
9264
9265    #[test]
9266    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
9267        let (events, verdict) = deregister_required_provider(true);
9268        assert_eq!(
9269            verdict,
9270            Some(CapabilityVerdict::NeverProvided),
9271            "shutdown must not change the verdict route.open refusals read"
9272        );
9273        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9274        let event = &events[0];
9275        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
9276        assert_eq!(
9277            event.fields.get("cause").map(String::as_str),
9278            Some("\"daemon_shutdown\""),
9279            "{event:?}"
9280        );
9281        assert_eq!(
9282            event.fields.get("verdict").map(String::as_str),
9283            Some("\"never_provided\""),
9284            "{event:?}"
9285        );
9286    }
9287
9288    #[test]
9289    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
9290        let (events, verdict) = deregister_required_provider(false);
9291        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
9292        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9293        let event = &events[0];
9294        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
9295        assert_eq!(event.fields.get("cause"), None, "{event:?}");
9296        assert_eq!(
9297            event.fields.get("verdict").map(String::as_str),
9298            Some("\"never_provided\""),
9299            "{event:?}"
9300        );
9301    }
9302
9303    #[test]
9304    fn catalog_list_omits_capabilities_for_legacy_manifest() {
9305        let registry = Arc::new(Registry::default());
9306        let handler = ControlHandler::new(Arc::clone(&registry));
9307        let hello = handler
9308            .handle_control(
9309                ConnectionId::new(101),
9310                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9311            )
9312            .expect("legacy HELLO registers");
9313        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9314
9315        let request = Frame::build(
9316            FrameType::Request,
9317            control_flags(),
9318            0,
9319            0,
9320            102,
9321            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9322                .expect("catalog request serializes"),
9323        )
9324        .expect("catalog request frame builds");
9325        let response = handler
9326            .handle_catalog_list(request, None)
9327            .expect("catalog list succeeds");
9328        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9329        assert!(
9330            body["modules"][0].get("capabilities").is_none(),
9331            "legacy manifest must retain an absent capabilities field on catalog.list"
9332        );
9333    }
9334
9335    #[test]
9336    fn hello_ack_omits_storage_when_no_storage_config() {
9337        let registry = Arc::new(Registry::default());
9338        let handler = ControlHandler::new(Arc::clone(&registry));
9339        let responses = handler
9340            .handle_control(
9341                ConnectionId::new(1),
9342                hello_frame("aft", PROTOCOL_VERSION, 7),
9343            )
9344            .unwrap();
9345        let ack = parse_ack(&responses[0]);
9346        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9347        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9348    }
9349
9350    #[tokio::test]
9351    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9352        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9353        let registry = Arc::new(Registry::default());
9354        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
9355        let responses = handler
9356            .handle_control(
9357                ConnectionId::new(1),
9358                hello_frame("aft", PROTOCOL_VERSION, 7),
9359            )
9360            .unwrap();
9361        let ack = parse_ack(&responses[0]);
9362        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9363
9364        let described = handler
9365            .handle_control_frame(
9366                &route_ctx(ConnectionId::new(2)).0,
9367                Frame::build(
9368                    FrameType::Request,
9369                    control_flags(),
9370                    0,
9371                    0,
9372                    9,
9373                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9374                )
9375                .unwrap(),
9376            )
9377            .await
9378            .unwrap();
9379        let ClientControlResponse::ServerDescribe { machine_id, .. } =
9380            serde_json::from_slice(&described[0].body).unwrap()
9381        else {
9382            panic!("server.describe answered with another shape");
9383        };
9384        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9385    }
9386
9387    #[test]
9388    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9389        // With a central sqlite storage policy, each registering module gets its
9390        // own resolved descriptor in HELLO_ACK, keyed by its module id.
9391        let registry = Arc::new(Registry::default());
9392        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
9393            crate::daemon_config::StorageConfig::Sqlite {
9394                data_home: std::path::PathBuf::from("/data"),
9395            },
9396        ));
9397
9398        let responses = handler
9399            .handle_control(
9400                ConnectionId::new(1),
9401                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9402            )
9403            .unwrap();
9404        let ack = parse_ack(&responses[0]);
9405        assert_eq!(
9406            ack.storage,
9407            Some(serde_json::json!({
9408                "module_id": "alfonso-routing",
9409                "storage_namespace": "default",
9410                "isolation": { "kind": "module" },
9411                "backend": {
9412                    "backend": "sqlite",
9413                    "path": "/data/cortexkit/alfonso-routing/store.db"
9414                }
9415            })),
9416            "the delivered descriptor is the module's own sqlite store path"
9417        );
9418    }
9419
9420    #[test]
9421    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9422        let registry = Arc::new(Registry::default());
9423        let handler = ControlHandler::new(Arc::clone(&registry));
9424        let conn = ConnectionId::new(1);
9425        let responses = handler
9426            .handle_control(
9427                conn,
9428                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9429            )
9430            .unwrap();
9431        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9432        let registration = registry.get_module("aft").unwrap().unwrap();
9433        assert_eq!(registration.control_ops, module_baseline_control_ops());
9434
9435        let frame =
9436            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9437        assert!(handler
9438            .guard_module_control_op(&frame, "aft", "route.bind")
9439            .unwrap()
9440            .is_none());
9441        let error = handler
9442            .guard_module_control_op(&frame, "aft", "test.synthetic")
9443            .unwrap()
9444            .expect("synthetic ungranted op should be rejected");
9445        assert_eq!(error.header.ty, FrameType::Error);
9446        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9447    }
9448
9449    #[test]
9450    fn hello_control_ops_some_adds_optional_grants() {
9451        let registry = Arc::new(Registry::default());
9452        let handler = ControlHandler::new(Arc::clone(&registry));
9453        handler
9454            .handle_control(
9455                ConnectionId::new(1),
9456                hello_frame_with_control_ops(
9457                    "aft",
9458                    PROTOCOL_VERSION,
9459                    7,
9460                    Some(vec![
9461                        "future.synthetic".to_string(),
9462                        "route.bind".to_string(),
9463                    ]),
9464                ),
9465            )
9466            .unwrap();
9467        let registration = registry.get_module("aft").unwrap().unwrap();
9468        assert_eq!(
9469            registration.control_ops,
9470            vec![
9471                "route.bind".to_string(),
9472                "route.status".to_string(),
9473                "future.synthetic".to_string(),
9474            ]
9475        );
9476        let frame =
9477            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9478        assert!(handler
9479            .guard_module_control_op(&frame, "aft", "future.synthetic")
9480            .unwrap()
9481            .is_none());
9482    }
9483
9484    #[tokio::test]
9485    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9486        let registry = Arc::new(Registry::default());
9487        let forwarding = Arc::new(ForwardingTable::default());
9488        let handler =
9489            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9490        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9491        hello_via_sink(
9492            &handler,
9493            &module_ctx,
9494            &mut module_rx,
9495            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9496        )
9497        .await;
9498
9499        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9500        let responses = handler
9501            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9502            .await
9503            .unwrap();
9504        assert_eq!(responses.len(), 1);
9505        assert_eq!(responses[0].header.ty, FrameType::Error);
9506        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9507        assert!(module_rx.try_recv().is_err());
9508    }
9509
9510    #[tokio::test]
9511    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9512        let registry = Arc::new(Registry::default());
9513        let forwarding = Arc::new(ForwardingTable::default());
9514        let handler =
9515            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9516        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9517        hello_via_sink(
9518            &handler,
9519            &module_ctx,
9520            &mut module_rx,
9521            hello_frame_with_control_ops(
9522                "aft",
9523                PROTOCOL_VERSION,
9524                7,
9525                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9526            ),
9527        )
9528        .await;
9529
9530        let project_root = unique_project_root("demux");
9531        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9532        let route_handler = handler.clone();
9533        let route_task = tokio::spawn(async move {
9534            route_handler
9535                .handle_control_frame(
9536                    &route_client_ctx,
9537                    route_open_frame(100, "aft", project_root),
9538                )
9539                .await
9540                .unwrap()
9541        });
9542        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9543            .await
9544            .unwrap()
9545            .unwrap();
9546        assert!(matches!(
9547            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9548            ModuleControlRequest::RouteBind { .. }
9549        ));
9550
9551        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9552        let health_handler = handler.clone();
9553        let health_task = tokio::spawn(async move {
9554            health_handler
9555                .handle_control_frame(
9556                    &health_client_ctx,
9557                    supervisor_health_probe_frame(101, "aft"),
9558                )
9559                .await
9560                .unwrap()
9561        });
9562        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9563            .await
9564            .unwrap()
9565            .unwrap();
9566        assert_eq!(
9567            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9568            ModuleControlRequest::HealthCheck {}
9569        );
9570
9571        handler
9572            .handle_control_frame(
9573                &module_ctx,
9574                health_response(health_frame.header.corr, HealthStatus::Degraded),
9575            )
9576            .await
9577            .unwrap();
9578        let health_response = health_task.await.unwrap();
9579        assert_eq!(health_response.len(), 1);
9580        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9581            ClientControlResponse::SupervisorHealthProbe {
9582                module_id,
9583                status,
9584                detail,
9585                metrics,
9586            } => {
9587                assert_eq!(module_id, "aft");
9588                assert_eq!(status, HealthStatus::Degraded);
9589                assert_eq!(detail.as_deref(), Some("warming"));
9590                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9591            }
9592            other => panic!("unexpected health response: {other:?}"),
9593        }
9594
9595        handler
9596            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9597            .await
9598            .unwrap();
9599        let route_response = route_task.await.unwrap();
9600        assert!(route_response.is_empty());
9601        let published = route_client_rx.recv().await.unwrap();
9602        assert!(matches!(
9603            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9604            ClientControlResponse::RouteOpen { .. }
9605        ));
9606    }
9607
9608    /// Start one `route.open` on `client_connection` and return its still-running
9609    /// handler task together with the `route.bind` the module received for it.
9610    /// The handler blocks until the module answers, so it has to run as a task
9611    /// while the test drives the module side.
9612    async fn relay_route_open(
9613        handler: &ControlHandler,
9614        client_connection: ConnectionId,
9615        client_egress: &FrameSink,
9616        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9617        corr: u64,
9618        module_id: &str,
9619        project_root_label: &str,
9620    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9621        let ctx = RouteCtx {
9622            connection_id: client_connection,
9623            egress: client_egress.clone(),
9624        };
9625        let handler = handler.clone();
9626        let project_root = unique_project_root(project_root_label);
9627        let module_id = module_id.to_string();
9628        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9629        let task = tokio::spawn(async move {
9630            let _guard = tracing::dispatcher::set_default(&dispatch);
9631            handler
9632                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9633                .await
9634                .unwrap()
9635        });
9636        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9637            .await
9638            .expect("module receives the relayed route.bind")
9639            .expect("module egress is open");
9640        (task, bind.frame)
9641    }
9642
9643    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9644        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9645            ModuleControlRequest::RouteBind {
9646                route_channel,
9647                epoch,
9648                ..
9649            } => (route_channel, epoch),
9650            other => panic!("expected a route.bind request, got {other:?}"),
9651        }
9652    }
9653
9654    fn published_route(frame: &Frame) -> (u16, u32) {
9655        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9656            ClientControlResponse::RouteOpen {
9657                route_channel,
9658                route_epoch,
9659            } => (route_channel, route_epoch),
9660            other => panic!("expected a route.open response, got {other:?}"),
9661        }
9662    }
9663
9664    /// Reproduction of a production outage. A client had `route.open`s in
9665    /// flight to a module and was already marked closing -- its egress had refused a
9666    /// module frame, so the daemon asked its connection to end -- while its sink
9667    /// was still open. When the module acked those binds, the daemon refused to
9668    /// commit a route for a closing client, and that refusal was returned from
9669    /// the MODULE connection's frame handler, where a router error that has no
9670    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
9671    /// the supervisor correctly did not respawn a clean exit, and every seat lost
9672    /// its tools for hours -- one client's teardown took down a connection
9673    /// carrying ~170 other routes.
9674    ///
9675    /// The window is opened here by calling the production path that opens it
9676    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
9677    /// state that matters is "in `closing_connections`, sink still open, relay
9678    /// still pending", and it lasts only from the close request until the
9679    /// connection loop reacts to it; a socket-level test can flood a client into
9680    /// that escalation but cannot pin the module's ack inside the window. Closing
9681    /// the socket instead takes the other path entirely -- connection teardown
9682    /// removes the pending relay under the same lock, so the ack finds nothing.
9683    #[tokio::test]
9684    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9685        let registry = Arc::new(Registry::default());
9686        let forwarding = Arc::new(ForwardingTable::default());
9687        let handler =
9688            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9689
9690        let module_connection = ConnectionId::new(30);
9691        let (module_ctx, mut module_rx) = route_ctx(module_connection);
9692        hello_via_sink(
9693            &handler,
9694            &module_ctx,
9695            &mut module_rx,
9696            hello_frame("aft", PROTOCOL_VERSION, 7),
9697        )
9698        .await;
9699
9700        let dying_client = ConnectionId::new(31);
9701        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9702
9703        // A published route on the dying client. The escalation below only marks
9704        // a connection closing for a route it has already published.
9705        let (first_task, first_bind) = relay_route_open(
9706            &handler,
9707            dying_client,
9708            &dying_ctx.egress,
9709            &mut module_rx,
9710            100,
9711            "aft",
9712            "closing-first",
9713        )
9714        .await;
9715        handler
9716            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9717            .await
9718            .unwrap();
9719        assert!(first_task.await.unwrap().is_empty());
9720        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9721
9722        // A second route.open from the same client, relayed and awaiting its ack.
9723        let (second_task, second_bind) = relay_route_open(
9724            &handler,
9725            dying_client,
9726            &dying_ctx.egress,
9727            &mut module_rx,
9728            101,
9729            "aft",
9730            "closing-second",
9731        )
9732        .await;
9733        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9734
9735        // The window: the client is closing, its sink is still open, and its
9736        // second bind is still pending.
9737        assert!(forwarding
9738            .escalate_client_delivery_failure(
9739                dying_client,
9740                first_channel,
9741                first_epoch,
9742                CloseReason::new(
9743                    "module_to_client_delivery_failed",
9744                    "client egress refused a module frame",
9745                ),
9746                crate::forwarding::UndeliveredFrame {
9747                    module_id: None,
9748                    sink: &dying_ctx.egress,
9749                },
9750            )
9751            .unwrap());
9752        assert!(!dying_ctx.egress.is_closed());
9753
9754        // The frame that used to end the module connection.
9755        let ack = handler
9756            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9757            .await;
9758        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9759        if module_loop_error.is_some() {
9760            // What the server's connection loop does with a router error that has
9761            // no ERROR-frame translation: end the connection, which releases the
9762            // module's registration and every route on it.
9763            handler.cleanup_connection(module_connection).unwrap();
9764        }
9765        // Read the module's next frame before opening the co-tenant's route, so
9766        // the GOODBYE assertion below is about THIS ack and not about later
9767        // traffic. `None` means the module was told nothing.
9768        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9769            .await
9770            .ok()
9771            .flatten();
9772
9773        // 1. The module connection is still registered.
9774        assert!(
9775            registry
9776                .get_module_by_connection(module_connection)
9777                .unwrap()
9778                .is_some(),
9779            "one client's closing connection ended the shared module connection: \
9780             {module_loop_error:?}"
9781        );
9782        // ...and still serving: another client can open and use a route on it.
9783        let cotenant = ConnectionId::new(32);
9784        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9785        let (cotenant_task, cotenant_bind) = relay_route_open(
9786            &handler,
9787            cotenant,
9788            &cotenant_ctx.egress,
9789            &mut module_rx,
9790            102,
9791            "aft",
9792            "closing-cotenant",
9793        )
9794        .await;
9795        handler
9796            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9797            .await
9798            .unwrap();
9799        assert!(cotenant_task.await.unwrap().is_empty());
9800        let (cotenant_channel, cotenant_epoch) =
9801            published_route(&cotenant_rx.recv().await.unwrap());
9802        assert!(matches!(
9803            forwarding
9804                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9805                .unwrap(),
9806            DataRoute::Client(DataRouteState::Bound(_))
9807        ));
9808
9809        // 2. The module was told to drop the binding it created for the route
9810        //    that will never be published.
9811        let goodbye = post_ack_module_frame
9812            .expect("module receives a GOODBYE for the abandoned route channel");
9813        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9814        assert_eq!(goodbye.header.channel, abandoned_channel);
9815        assert_eq!(goodbye.header.epoch, abandoned_epoch);
9816
9817        // 3. The dying client received nothing: no route was ever published to
9818        //    it. Its route.open is answered as unavailable, which the connection
9819        //    loop would write to a socket that is already going away.
9820        assert!(dying_rx.try_recv().is_err());
9821        let second_response = second_task.await.unwrap();
9822        assert_eq!(second_response.len(), 1);
9823        assert_eq!(
9824            parse_error(&second_response[0])["code"],
9825            "target_unavailable"
9826        );
9827    }
9828
9829    /// One `catalog.list` as a client sees it: the generation and the listed
9830    /// module ids with their readiness.
9831    async fn catalog_snapshot(
9832        handler: &ControlHandler,
9833        client_ctx: &RouteCtx,
9834        corr: u64,
9835    ) -> (u64, Vec<(String, bool)>) {
9836        let body =
9837            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9838        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9839        let reply = handler
9840            .handle_control_frame(client_ctx, request)
9841            .await
9842            .unwrap()
9843            .pop()
9844            .unwrap();
9845        let ClientControlResponse::CatalogList {
9846            generation,
9847            modules,
9848            ..
9849        } = serde_json::from_slice(&reply.body).unwrap()
9850        else {
9851            panic!("not a catalog.list reply");
9852        };
9853        (
9854            generation,
9855            modules
9856                .into_iter()
9857                .map(|entry| (entry.module_id, entry.ready))
9858                .collect(),
9859        )
9860    }
9861
9862    /// A handler whose HELLO for module "aft" on connection 40 is parked
9863    /// between reserving its registry entry and installing its forwarding
9864    /// endpoint. Returns the handler, the two-party barrier (wait once to know
9865    /// the HELLO is parked, once more to release it), the HELLO's thread, and
9866    /// the module's context and receiver.
9867    fn hello_held_before_forwarding() -> (
9868        ControlHandler,
9869        Arc<std::sync::Barrier>,
9870        std::thread::JoinHandle<Vec<Frame>>,
9871        RouteCtx,
9872        mpsc::Receiver<crate::router::OutboundFrame>,
9873    ) {
9874        let hold = Arc::new(std::sync::Barrier::new(2));
9875        let handler = ControlHandler::with_forwarding(
9876            Arc::new(Registry::default()),
9877            Arc::new(ForwardingTable::default()),
9878        )
9879        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9880        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9881        let hello = {
9882            let handler = handler.clone();
9883            let module_ctx = module_ctx.clone();
9884            std::thread::spawn(move || {
9885                handler
9886                    .handle_hello(
9887                        module_ctx.connection_id,
9888                        Some(module_ctx.egress.clone()),
9889                        hello_frame("aft", PROTOCOL_VERSION, 7),
9890                    )
9891                    .unwrap()
9892            })
9893        };
9894        (handler, hold, hello, module_ctx, module_rx)
9895    }
9896
9897    /// While a HELLO has reserved its registry entry but its forwarding
9898    /// endpoint is not installed yet, `route.open` to the module can only be
9899    /// refused. `catalog.list` must not list the module in that window: a
9900    /// client that opens a route as soon as it sees a module listed and ready
9901    /// would be refused for a module the daemon had just advertised. The
9902    /// refusal in the window stays the retryable `target_unavailable`, never
9903    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
9904    /// Once the HELLO completes, the module is listed and a route to it opens.
9905    #[tokio::test]
9906    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
9907        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
9908        let client = ConnectionId::new(41);
9909        let (client_ctx, mut client_rx) = route_ctx(client);
9910
9911        hold.wait();
9912        // Parked: the registry holds "aft", forwarding has no endpoint for it.
9913        assert!(handler.registry.get_module("aft").unwrap().is_some());
9914        assert!(!handler
9915            .forwarding
9916            .has_live_module_connection("aft")
9917            .unwrap());
9918        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
9919        assert!(
9920            listed.is_empty(),
9921            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
9922        );
9923        let refused = handler
9924            .handle_control_frame(
9925                &client_ctx,
9926                route_open_frame(301, "aft", unique_project_root("listed-window")),
9927            )
9928            .await
9929            .unwrap();
9930        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
9931        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
9932        let duplicate = handler
9933            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
9934            .await
9935            .unwrap();
9936        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9937
9938        hold.wait();
9939        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9940        let ack = module_rx.try_recv().unwrap().frame;
9941        assert_eq!(ack.header.ty, FrameType::HelloAck);
9942        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9943        assert_eq!(listed, vec![("aft".to_string(), true)]);
9944
9945        let (open_task, bind) = relay_route_open(
9946            &handler,
9947            client,
9948            &client_ctx.egress,
9949            &mut module_rx,
9950            303,
9951            "aft",
9952            "listed-after",
9953        )
9954        .await;
9955        assert!(handler
9956            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9957            .await
9958            .unwrap()
9959            .is_empty());
9960        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
9961        published_route(&client_rx.try_recv().unwrap().frame);
9962    }
9963
9964    /// A catalog watcher keys on the generation: when it changes, it re-reads
9965    /// the listing. So the generation must advance exactly when the module
9966    /// appears. If it advanced while the module was still hidden, a watcher
9967    /// would read the new generation without the module, and the module would
9968    /// later appear under that same generation with nothing telling the
9969    /// watcher to look again.
9970    #[tokio::test]
9971    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
9972        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
9973        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9974        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9975
9976        hold.wait();
9977        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
9978        assert!(listed.is_empty());
9979        assert_eq!(
9980            during, before,
9981            "the generation moved while the module was not listed"
9982        );
9983
9984        hold.wait();
9985        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9986        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9987        assert_eq!(listed, vec![("aft".to_string(), true)]);
9988        assert_eq!(
9989            after,
9990            before + 1,
9991            "the module appeared without exactly one generation bump"
9992        );
9993    }
9994
9995    /// A HELLO whose forwarding endpoint cannot be installed (here the
9996    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
9997    /// refused and leaves nothing behind: no registry entry, no catalog entry,
9998    /// and the catalog generation where it was.
9999    #[tokio::test]
10000    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10001        let registry = Arc::new(Registry::default());
10002        let handler = ControlHandler::with_forwarding(
10003            Arc::clone(&registry),
10004            Arc::new(ForwardingTable::default()),
10005        );
10006        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10007        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10008        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10009        drop(module_rx);
10010
10011        let replies = handler
10012            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10013            .await
10014            .unwrap();
10015        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10016        assert!(registry.get_module("aft").unwrap().is_none());
10017        assert!(registry
10018            .get_module_by_connection(module_ctx.connection_id)
10019            .unwrap()
10020            .is_none());
10021        assert_eq!(
10022            catalog_snapshot(&handler, &client_ctx, 301).await,
10023            (before, Vec::new())
10024        );
10025    }
10026
10027    /// A handler whose `route.open` parks after queueing the route.bind and
10028    /// before marking it enqueued, with module "aft" registered on connection
10029    /// 40. Returns the two-party barrier that parks and releases it.
10030    async fn route_open_held_after_bind_send() -> (
10031        ControlHandler,
10032        Arc<tokio::sync::Barrier>,
10033        RouteCtx,
10034        mpsc::Receiver<crate::router::OutboundFrame>,
10035    ) {
10036        let hold = Arc::new(tokio::sync::Barrier::new(2));
10037        let handler = ControlHandler::with_forwarding(
10038            Arc::new(Registry::default()),
10039            Arc::new(ForwardingTable::default()),
10040        )
10041        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10042        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10043        hello_via_sink(
10044            &handler,
10045            &module_ctx,
10046            &mut module_rx,
10047            hello_frame("aft", PROTOCOL_VERSION, 7),
10048        )
10049        .await;
10050        (handler, hold, module_ctx, module_rx)
10051    }
10052
10053    /// A fast module acks the route.bind, and the reader commits the route,
10054    /// before `route.open` has marked the relay enqueued. The daemon used to
10055    /// read the missing relay as an abandoned bind and send the module a
10056    /// GOODBYE for the route it had just committed: the client was told its
10057    /// route was open, the module unbound the channel, and the client's first
10058    /// request was dropped on the module side as unbound.
10059    #[tokio::test]
10060    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10061        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10062        let client = ConnectionId::new(41);
10063        let (client_ctx, mut client_rx) = route_ctx(client);
10064
10065        let (open_task, bind) = relay_route_open(
10066            &handler,
10067            client,
10068            &client_ctx.egress,
10069            &mut module_rx,
10070            200,
10071            "aft",
10072            "acked-before-mark",
10073        )
10074        .await;
10075        let (module_channel, module_epoch) = route_bind_channel(&bind);
10076        // route.open is now parked between queueing the bind and marking it.
10077        hold.wait().await;
10078
10079        // The module answers inside that window, and the route commits.
10080        assert!(handler
10081            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10082            .await
10083            .unwrap()
10084            .is_empty());
10085        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10086
10087        hold.wait().await;
10088        assert!(
10089            open_task.await.unwrap().is_empty(),
10090            "route.open answers accepted"
10091        );
10092
10093        // Every frame route.open could have sent is already queued, because it
10094        // has returned; the module must have been sent nothing.
10095        match module_rx.try_recv() {
10096            Err(mpsc::error::TryRecvError::Empty) => {}
10097            Ok(outbound) => panic!(
10098                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10099                outbound.frame.header.ty,
10100                outbound.frame.header.channel,
10101                outbound.frame.header.epoch
10102            ),
10103            Err(err) => panic!("module egress closed: {err}"),
10104        }
10105
10106        // The client's first request on the route is the next thing the module
10107        // reads, on the binding it just created.
10108        let router = Router::with_control_handler(Arc::new(handler.clone()));
10109        let request = Frame::build(
10110            FrameType::Request,
10111            Flags::new(true, Priority::Interactive, false),
10112            client_channel,
10113            client_epoch,
10114            201,
10115            b"first request".to_vec(),
10116        )
10117        .unwrap();
10118        router
10119            .route_for_connection(&client_ctx, request)
10120            .await
10121            .unwrap();
10122        let delivered = module_rx.try_recv().unwrap().frame;
10123        assert_eq!(delivered.header.ty, FrameType::Request);
10124        assert_eq!(
10125            (delivered.header.channel, delivered.header.epoch),
10126            (module_channel, module_epoch)
10127        );
10128        assert_eq!(delivered.body, b"first request");
10129    }
10130
10131    /// The client goes away while `route.open` is still queueing the bind, so
10132    /// the relay is dropped with no module reply. The teardown cannot send the
10133    /// GOODBYE itself (it could reach the module ahead of the bind), so
10134    /// `route.open` must send it once the bind is queued; otherwise the module
10135    /// keeps a binding the daemon will never commit.
10136    #[tokio::test]
10137    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10138        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10139        let client = ConnectionId::new(42);
10140        let (client_ctx, _client_rx) = route_ctx(client);
10141
10142        let (open_task, bind) = relay_route_open(
10143            &handler,
10144            client,
10145            &client_ctx.egress,
10146            &mut module_rx,
10147            300,
10148            "aft",
10149            "released-before-mark",
10150        )
10151        .await;
10152        let (module_channel, module_epoch) = route_bind_channel(&bind);
10153        hold.wait().await;
10154
10155        handler.cleanup_connection(client).unwrap();
10156        assert!(
10157            module_rx.try_recv().is_err(),
10158            "client teardown must not GOODBYE a bind route.open is still queueing"
10159        );
10160
10161        hold.wait().await;
10162        let response = open_task.await.unwrap();
10163        assert_eq!(response.len(), 1);
10164        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10165
10166        let goodbye = module_rx
10167            .try_recv()
10168            .expect("module is told to drop the binding the daemon abandoned")
10169            .frame;
10170        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10171        assert_eq!(
10172            (goodbye.header.channel, goodbye.header.epoch),
10173            (module_channel, module_epoch)
10174        );
10175        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10176    }
10177
10178    /// The module acks inside the same window, but the client's egress closed
10179    /// first, so the reader refuses to commit. The ack proves the module holds
10180    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
10181    /// send a second one.
10182    #[tokio::test]
10183    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
10184        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10185        let client = ConnectionId::new(43);
10186        let (client_ctx, client_rx) = route_ctx(client);
10187
10188        let (open_task, bind) = relay_route_open(
10189            &handler,
10190            client,
10191            &client_ctx.egress,
10192            &mut module_rx,
10193            400,
10194            "aft",
10195            "gone-client-before-mark",
10196        )
10197        .await;
10198        let (module_channel, module_epoch) = route_bind_channel(&bind);
10199        hold.wait().await;
10200
10201        drop(client_rx);
10202        assert!(client_ctx.egress.is_closed());
10203        handler
10204            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10205            .await
10206            .unwrap();
10207
10208        hold.wait().await;
10209        let response = open_task.await.unwrap();
10210        assert_eq!(response.len(), 1);
10211        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10212
10213        let goodbye = module_rx
10214            .try_recv()
10215            .expect("module is told to drop the binding it acked")
10216            .frame;
10217        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10218        assert_eq!(
10219            (goodbye.header.channel, goodbye.header.epoch),
10220            (module_channel, module_epoch)
10221        );
10222        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10223        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
10224    }
10225
10226    /// The fence at the module-loop boundary, stated as its own contract: which
10227    /// forwarding failures are allowed to end the module connection that is being
10228    /// served. A `ConnectionClosing` naming some client is about that client, and
10229    /// a module connection is shared; the same error naming the module's own
10230    /// connection is about this connection and must stay fatal, as must failures
10231    /// that are about the forwarding table itself.
10232    #[test]
10233    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
10234        let handler = ControlHandler::default();
10235        let module_connection = ConnectionId::new(30);
10236        let client_connection = ConnectionId::new(31);
10237
10238        handler
10239            .refuse_to_end_module_connection_for_a_client(
10240                module_connection,
10241                77,
10242                ForwardingError::ConnectionClosing {
10243                    connection_id: client_connection,
10244                },
10245            )
10246            .expect("a closing client must never end the module connection");
10247
10248        assert!(matches!(
10249            handler.refuse_to_end_module_connection_for_a_client(
10250                module_connection,
10251                78,
10252                ForwardingError::ConnectionClosing {
10253                    connection_id: module_connection,
10254                },
10255            ),
10256            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
10257                connection_id
10258            })) if connection_id == module_connection
10259        ));
10260        assert!(matches!(
10261            handler.refuse_to_end_module_connection_for_a_client(
10262                module_connection,
10263                79,
10264                ForwardingError::Poisoned,
10265            ),
10266            Err(RouterError::Forwarding(ForwardingError::Poisoned))
10267        ));
10268        assert!(matches!(
10269            handler.refuse_to_end_module_connection_for_a_client(
10270                module_connection,
10271                80,
10272                ForwardingError::StaleModuleEndpoint,
10273            ),
10274            Err(RouterError::Forwarding(
10275                ForwardingError::StaleModuleEndpoint
10276            ))
10277        ));
10278    }
10279
10280    /// The spawn-attestation guard is what stops a connected module from claiming
10281    /// another module's identity and being stamped `Reserved` for it. Every other
10282    /// test that supplies a consumer_identity supplies a CORRECT one, because a
10283    /// correct one is what the rest of the flow needs -- so the guard's rejection
10284    /// branch was never the subject of an assertion, only its acceptance branch.
10285    ///
10286    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
10287    /// whole subc-core library suite green; only the forwarding integration tests
10288    /// notice, and they notice for unrelated reasons. This test exists so the
10289    /// refusal itself is asserted where the guard lives: it fails if the identity
10290    /// check stops refusing, which is the direction that matters, since a guard
10291    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
10292    #[tokio::test]
10293    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
10294        let registry = Arc::new(Registry::default());
10295        let forwarding = Arc::new(ForwardingTable::default());
10296        let supervisor = SupervisorHandle::new();
10297        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10298        let handler =
10299            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10300                .with_supervisor(supervisor);
10301
10302        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10303        hello_via_sink(
10304            &handler,
10305            &target_ctx,
10306            &mut target_rx,
10307            hello_frame("target", PROTOCOL_VERSION, 1),
10308        )
10309        .await;
10310
10311        // A real supervised module id presenting the wrong nonce. This is the
10312        // impersonation case: the attacker knows a privileged module_id, which is
10313        // public, and guesses at the nonce, which is not.
10314        let wrong_nonce = handler
10315            .handle_control_frame(
10316                &route_ctx(ConnectionId::new(91)).0,
10317                route_open_frame_with_admission_facts(
10318                    20,
10319                    "target",
10320                    unique_project_root("admission-facts"),
10321                    Some(subc_control::ConsumerIdentity {
10322                        module_id: "fed".to_string(),
10323                        launch_nonce: "not-the-real-nonce".to_string(),
10324                    }),
10325                    None,
10326                ),
10327            )
10328            .await
10329            .unwrap();
10330        assert_eq!(
10331            parse_error(&wrong_nonce[0])["code"],
10332            "bad_consumer_identity",
10333            "a mismatched launch nonce must be refused, not stamped Reserved"
10334        );
10335
10336        // A module id the supervisor never spawned at all, so no nonce exists to
10337        // compare against. An implementation that treats "no record" as "nothing
10338        // to check" fails open here while passing the case above.
10339        let never_spawned = handler
10340            .handle_control_frame(
10341                &route_ctx(ConnectionId::new(92)).0,
10342                route_open_frame_with_admission_facts(
10343                    21,
10344                    "target",
10345                    unique_project_root("admission-facts"),
10346                    Some(subc_control::ConsumerIdentity {
10347                        module_id: "never-spawned".to_string(),
10348                        launch_nonce: "any-nonce".to_string(),
10349                    }),
10350                    None,
10351                ),
10352            )
10353            .await
10354            .unwrap();
10355        assert_eq!(
10356            parse_error(&never_spawned[0])["code"],
10357            "bad_consumer_identity",
10358            "an unspawned module_id must be refused rather than accepted for lack of a record"
10359        );
10360    }
10361
10362    /// The refusal test above proves the guard says NO. Nothing proved it can say
10363    /// YES, and the difference is not academic: replacing the whole authorization
10364    /// with `false` -- admitting no consumer identity at all, revoking Reserved
10365    /// standing for every supervised module in the fleet -- leaves 110 of the 111
10366    /// library tests GREEN. The one that notices does so by HANGING, because it
10367    /// waits for a bind that can no longer happen.
10368    ///
10369    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
10370    /// or flaky test, invites a RETRY rather than an investigation, and the retry
10371    /// hangs too and gets blamed on the runner. So a total revocation of the
10372    /// daemon's trust grant would have shipped behind a symptom nobody attributes
10373    /// to code.
10374    ///
10375    /// The bias is structural rather than accidental. A REFUSAL looks like a
10376    /// failure someone writes a test for; a GRANT looks like the happy path. Every
10377    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
10378    /// suite for that reason, and this one is the purest case in the daemon.
10379    ///
10380    /// This test asserts the EFFECT rather than the absence of an error: the module
10381    /// receives a RouteBind and it carries `Reserved` naming the attested module.
10382    /// A guard that admitted nobody would produce no bind at all; one that admitted
10383    /// everybody would stamp the wrong principal, which the refusal test catches.
10384    #[tokio::test]
10385    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10386        let registry = Arc::new(Registry::default());
10387        let forwarding = Arc::new(ForwardingTable::default());
10388        let supervisor = SupervisorHandle::new();
10389        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10390        let handler =
10391            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10392                .with_supervisor(supervisor);
10393
10394        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10395        hello_via_sink(
10396            &handler,
10397            &target_ctx,
10398            &mut target_rx,
10399            hello_frame("target", PROTOCOL_VERSION, 1),
10400        )
10401        .await;
10402
10403        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10404        let route_handler = handler.clone();
10405        let route_task = tokio::spawn(async move {
10406            route_handler
10407                .handle_control_frame(
10408                    &client_ctx,
10409                    route_open_frame_with_admission_facts(
10410                        30,
10411                        "target",
10412                        unique_project_root("admission-facts"),
10413                        Some(subc_control::ConsumerIdentity {
10414                            module_id: "fed".to_string(),
10415                            launch_nonce: "fed-nonce".to_string(),
10416                        }),
10417                        None,
10418                    ),
10419                )
10420                .await
10421                .unwrap()
10422        });
10423
10424        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
10425        // the very mutation it exists to catch -- a guard that admits nobody -- no
10426        // bind is ever sent, so it HUNG rather than failing. That reproduces the
10427        // exact defect being fixed: a total revocation detected only as a stalled
10428        // suite, which reads as flakiness and invites a retry. An acceptance test
10429        // that waits for an effect must bound the wait, or a red becomes a hang.
10430        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10431            .await
10432            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10433            .expect("module control channel closed before route.bind");
10434        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10435        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10436            panic!("expected route.bind")
10437        };
10438        assert_eq!(
10439            principal,
10440            Some(Principal::Reserved {
10441                module_id: "fed".to_string()
10442            }),
10443            "a correctly attested consumer must be stamped Reserved for its own id"
10444        );
10445
10446        handler
10447            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10448            .await
10449            .unwrap();
10450        assert!(route_task.await.unwrap().is_empty());
10451        assert!(
10452            matches!(
10453                serde_json::from_slice::<ClientControlResponse>(
10454                    &client_rx.recv().await.unwrap().body
10455                )
10456                .unwrap(),
10457                ClientControlResponse::RouteOpen { .. }
10458            ),
10459            "the route must actually open, not merely avoid an error"
10460        );
10461    }
10462
10463    #[tokio::test(start_paused = true)]
10464    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10465        let registry = Arc::new(Registry::default());
10466        let forwarding = Arc::new(ForwardingTable::default());
10467        let supervisor = SupervisorHandle::new();
10468        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10469        let handler =
10470            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10471                .with_supervisor(supervisor);
10472
10473        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10474        hello_via_sink(
10475            &handler,
10476            &target_ctx,
10477            &mut target_rx,
10478            hello_frame("target", PROTOCOL_VERSION, 1),
10479        )
10480        .await;
10481
10482        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10483        let direct_handler = handler.clone();
10484        let direct_open = tokio::spawn(async move {
10485            direct_handler
10486                .handle_control_frame(
10487                    &direct_ctx,
10488                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
10489                )
10490                .await
10491                .unwrap()
10492        });
10493        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10494            .await
10495            .expect("no direct route.bind within 5s")
10496            .expect("target control channel closed before direct route.bind");
10497        handler
10498            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10499            .await
10500            .unwrap();
10501        assert!(direct_open.await.unwrap().is_empty());
10502        let _ = direct_rx.recv().await.unwrap();
10503
10504        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10505        let reserved_handler = handler.clone();
10506        let reserved_open = tokio::spawn(async move {
10507            reserved_handler
10508                .handle_control_frame(
10509                    &reserved_ctx,
10510                    route_open_frame_with_admission_facts(
10511                        3,
10512                        "target",
10513                        unique_project_root("admission-facts"),
10514                        Some(ConsumerIdentity {
10515                            module_id: "fed".to_string(),
10516                            launch_nonce: "fed-nonce".to_string(),
10517                        }),
10518                        None,
10519                    ),
10520                )
10521                .await
10522                .unwrap()
10523        });
10524        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10525            .await
10526            .expect("no reserved route.bind within 5s")
10527            .expect("target control channel closed before reserved route.bind");
10528        handler
10529            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10530            .await
10531            .unwrap();
10532        assert!(reserved_open.await.unwrap().is_empty());
10533        let _ = reserved_rx.recv().await.unwrap();
10534
10535        forwarding
10536            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10537            .unwrap();
10538        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10539        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10540            module_id: Some("target".to_string()),
10541        })
10542        .unwrap();
10543        let census_frame =
10544            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10545        let response = handler
10546            .handle_control_frame(&census_ctx, census_frame)
10547            .await
10548            .unwrap()
10549            .pop()
10550            .unwrap();
10551        let actual: Value = serde_json::from_slice(&response.body).unwrap();
10552        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10553        assert!(matches!(
10554            decoded,
10555            ClientControlResponse::SupervisorRoutes { .. }
10556        ));
10557        let routes = actual["modules"][0]["routes"].as_array().unwrap();
10558        assert_eq!(routes.len(), 2);
10559        assert!(routes.iter().all(|route| route["draining"] == true));
10560        // The census carries WHY: the reason the drain was begun with, in the
10561        // route.closing vocabulary, on every draining route this drain marked.
10562        assert!(
10563            routes.iter().all(|route| route["drain_reason"] == "reload"),
10564            "draining routes must name the drain's reason: {routes:?}"
10565        );
10566        assert!(routes.iter().any(|route| {
10567            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10568        }));
10569        assert!(routes.iter().any(|route| {
10570            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10571        }));
10572
10573        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10574            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10575        if std::env::var_os("UPDATE_GOLDEN").is_some() {
10576            std::fs::write(
10577                &golden_path,
10578                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10579            )
10580            .unwrap();
10581        }
10582        let expected: Value =
10583            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10584        assert_eq!(actual, expected);
10585    }
10586
10587    async fn query_live_roots(
10588        handler: &ControlHandler,
10589        module_ctx: &RouteCtx,
10590    ) -> ModuleControlResponseToModule {
10591        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10592        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10593        let response = handler
10594            .handle_control_frame(module_ctx, frame)
10595            .await
10596            .unwrap()
10597            .pop()
10598            .unwrap();
10599        serde_json::from_slice(&response.body).unwrap()
10600    }
10601
10602    #[tokio::test(start_paused = true)]
10603    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10604        let registry = Arc::new(Registry::default());
10605        let forwarding = Arc::new(ForwardingTable::default());
10606        let handler = ControlHandler::with_forwarding(registry, forwarding);
10607        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10608        hello_via_sink(
10609            &handler,
10610            &target_ctx,
10611            &mut target_rx,
10612            hello_frame("target", PROTOCOL_VERSION, 1),
10613        )
10614        .await;
10615        let root = unique_project_root("live-roots-known");
10616        let path = ProjectRootId::from_path_allowing_missing(root.path())
10617            .unwrap()
10618            .as_path()
10619            .to_path_buf();
10620        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10621        let open_handler = handler.clone();
10622        let opened = tokio::spawn(async move {
10623            open_handler
10624                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10625                .await
10626                .unwrap()
10627        });
10628        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10629            .await
10630            .unwrap()
10631            .unwrap();
10632        handler
10633            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10634            .await
10635            .unwrap();
10636        assert!(opened.await.unwrap().is_empty());
10637        let _ = client_rx.recv().await.unwrap();
10638
10639        let root = unique_project_root("live-roots-pending");
10640        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10641            .unwrap()
10642            .as_path()
10643            .to_path_buf();
10644        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10645        let open_handler = handler.clone();
10646        let pending = tokio::spawn(async move {
10647            open_handler
10648                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10649                .await
10650                .unwrap()
10651        });
10652        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10653            .await
10654            .unwrap()
10655            .unwrap();
10656        let actual = query_live_roots(&handler, &target_ctx).await;
10657        let ModuleControlResponseToModule::LiveRoots {
10658            roots,
10659            unknown_root_bindings,
10660            total_bindings,
10661        } = actual
10662        else {
10663            panic!("expected live roots")
10664        };
10665        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10666        assert_eq!(unknown_root_bindings, 0);
10667        assert_eq!(
10668            roots.len(),
10669            2,
10670            "root-known arm must retain each canonical root"
10671        );
10672        assert_eq!(
10673            total_bindings,
10674            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10675        );
10676        let counts = roots
10677            .iter()
10678            .map(|root| (root.project_root.clone(), root.bound, root.pending))
10679            .collect::<Vec<_>>();
10680        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10681        expected.sort_by(|a, b| a.0.cmp(&b.0));
10682        assert_eq!(
10683            counts, expected,
10684            "roots must sort by path and count pending separately"
10685        );
10686        handler
10687            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10688            .await
10689            .unwrap();
10690        assert!(pending.await.unwrap().is_empty());
10691    }
10692
10693    #[tokio::test(start_paused = true)]
10694    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10695        let forwarding = Arc::new(ForwardingTable::default());
10696        let handler =
10697            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10698        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10699        hello_via_sink(
10700            &handler,
10701            &target_ctx,
10702            &mut target_rx,
10703            hello_frame("target", PROTOCOL_VERSION, 1),
10704        )
10705        .await;
10706        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10707        let pending = forwarding
10708            .begin_route_bind_relay_for_test(
10709                client_ctx.connection_id,
10710                client_ctx.egress.clone(),
10711                2,
10712                "target",
10713            )
10714            .unwrap();
10715        forwarding
10716            .complete_pending_relay(
10717                target_ctx.connection_id,
10718                pending.corr,
10719                RouteBindRelayOutcome::Accepted,
10720            )
10721            .unwrap();
10722        let actual = query_live_roots(&handler, &target_ctx).await;
10723        let ModuleControlResponseToModule::LiveRoots {
10724            roots,
10725            unknown_root_bindings,
10726            total_bindings,
10727        } = actual
10728        else {
10729            panic!("expected live roots")
10730        };
10731        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10732        assert_eq!(
10733            unknown_root_bindings, 1,
10734            "unknown-root arm must not read as no bindings"
10735        );
10736        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10737        assert_eq!(
10738            total_bindings,
10739            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10740        );
10741    }
10742
10743    /// A module reads HELLO_ACK as its first frame and exits on anything else,
10744    /// so the ack has to be on its outbound queue before the module is
10745    /// routable. The connection loop writes a handler's replies only after the
10746    /// handler returns; this test stops in exactly that gap, runs a real
10747    /// route.open from another connection, and only then writes whatever the
10748    /// HELLO handler returned, the way the loop would. If the ack were still a
10749    /// reply, the route.bind request would reach the module first.
10750    #[tokio::test(start_paused = true)]
10751    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10752        let forwarding = Arc::new(ForwardingTable::default());
10753        let handler =
10754            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10755        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10756        let replies = handler
10757            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10758            .await
10759            .unwrap();
10760        let queued_by_hello = module_rx.len();
10761
10762        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10763        let open_handler = handler.clone();
10764        let open = tokio::spawn(async move {
10765            open_handler
10766                .handle_control_frame(
10767                    &client_ctx,
10768                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10769                )
10770                .await
10771                .unwrap()
10772        });
10773        // Let the route.open run until its route.bind is on the module's queue.
10774        let mut spins = 0;
10775        while module_rx.len() == queued_by_hello {
10776            spins += 1;
10777            assert!(spins < 10_000, "route.open never queued a route.bind");
10778            tokio::task::yield_now().await;
10779        }
10780
10781        // Now the connection loop's half: write the HELLO handler's replies.
10782        for reply in replies {
10783            module_ctx.egress.send(reply).await.unwrap();
10784        }
10785
10786        let first = module_rx.recv().await.unwrap().frame;
10787        assert_eq!(
10788            first.header.ty,
10789            FrameType::HelloAck,
10790            "the first frame a registering module reads must be its HELLO_ACK"
10791        );
10792        assert_eq!(first.header.corr, 7);
10793        let second = module_rx.recv().await.unwrap().frame;
10794        assert_eq!(second.header.ty, FrameType::Request);
10795        assert!(
10796            matches!(
10797                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10798                ModuleControlRequest::RouteBind { .. }
10799            ),
10800            "the route.bind follows the ack"
10801        );
10802        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10803
10804        handler
10805            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10806            .await
10807            .unwrap();
10808        assert!(open.await.unwrap().is_empty());
10809        let _ = client_rx.recv().await.unwrap();
10810    }
10811
10812    #[tokio::test(start_paused = true)]
10813    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10814        let handler = ControlHandler::with_forwarding(
10815            Arc::new(Registry::default()),
10816            Arc::new(ForwardingTable::default()),
10817        );
10818        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10819        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10820        hello_via_sink(
10821            &handler,
10822            &first_ctx,
10823            &mut first_rx,
10824            hello_frame("first", PROTOCOL_VERSION, 1),
10825        )
10826        .await;
10827        hello_via_sink(
10828            &handler,
10829            &second_ctx,
10830            &mut second_rx,
10831            hello_frame("second", PROTOCOL_VERSION, 2),
10832        )
10833        .await;
10834        let root = unique_project_root("second-only");
10835        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10836        let cloned = handler.clone();
10837        let open = tokio::spawn(async move {
10838            cloned
10839                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10840                .await
10841                .unwrap()
10842        });
10843        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10844            .await
10845            .unwrap()
10846            .unwrap();
10847        let first = query_live_roots(&handler, &first_ctx).await;
10848        let second = query_live_roots(&handler, &second_ctx).await;
10849        assert!(
10850            matches!(
10851                first,
10852                ModuleControlResponseToModule::LiveRoots {
10853                    total_bindings: 0,
10854                    ..
10855                }
10856            ),
10857            "cross-module scope must not expose another module's roots"
10858        );
10859        assert!(
10860            matches!(
10861                second,
10862                ModuleControlResponseToModule::LiveRoots {
10863                    total_bindings: 1,
10864                    ..
10865                }
10866            ),
10867            "second module must see its pending route"
10868        );
10869        handler
10870            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10871            .await
10872            .unwrap();
10873        assert!(open.await.unwrap().is_empty());
10874    }
10875
10876    #[tokio::test(start_paused = true)]
10877    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10878        let handler = ControlHandler::with_forwarding(
10879            Arc::new(Registry::default()),
10880            Arc::new(ForwardingTable::default()),
10881        );
10882        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10883        hello_via_sink(
10884            &handler,
10885            &target_ctx,
10886            &mut target_rx,
10887            hello_frame("target", PROTOCOL_VERSION, 1),
10888        )
10889        .await;
10890        let actual = query_live_roots(&handler, &target_ctx).await;
10891        let ModuleControlResponseToModule::LiveRoots {
10892            roots,
10893            unknown_root_bindings,
10894            total_bindings,
10895        } = actual
10896        else {
10897            panic!("expected live roots")
10898        };
10899        assert!(roots.is_empty());
10900        assert_eq!(unknown_root_bindings, 0);
10901        assert_eq!(total_bindings, 0);
10902        assert_eq!(
10903            total_bindings,
10904            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10905        );
10906    }
10907
10908    /// Read the vendored fed corpus rather than hand-building a package.
10909    ///
10910    /// A hand-built object encodes what the test author believed the carrier
10911    /// emits. These vectors are what it actually emits, and one of them exists
10912    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
10913    /// ignores additive unknown fields at the traversal emit terminus."
10914    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
10915        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10916            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
10917        let text = std::fs::read_to_string(&path)
10918            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
10919        let vectors: Vec<(String, Value)> = text
10920            .lines()
10921            .filter(|line| !line.trim().is_empty())
10922            .map(|line| {
10923                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
10924                let id = entry["corpus_id"]
10925                    .as_str()
10926                    .expect("every vector carries a corpus_id")
10927                    .to_string();
10928                (id, entry["package"].clone())
10929            })
10930            .collect();
10931        // Pin the count: a corpus that silently shrinks would take its coverage
10932        // with it, and a suite reading N-1 vectors reports the same clean pass
10933        // as one reading N.
10934        assert_eq!(
10935            vectors.len(),
10936            3,
10937            "vendored fed corpus changed size; re-sync from subc-federation"
10938        );
10939
10940        // Pin what makes the corpus DISCRIMINATING, not just present.
10941        //
10942        // The relay test below takes its expected value from the corpus, so the
10943        // corpus supplies the test's power to detect a lossy relay rather than
10944        // its correctness. A relay that dropped unrecognised fields would still
10945        // be caught -- but only by a package carrying fields it does not know.
10946        // Shrink every package to the handful of keys any implementation would
10947        // recognise and the test keeps passing over an input that can no longer
10948        // fail, which is the same clean green as a corpus that shrank away.
10949        //
10950        // So assert the precondition rather than duplicating the packages here:
10951        // at least one vector must carry a field beyond the small common set.
10952        // That is one claim to maintain instead of nine, and it fails loudly if
10953        // a re-sync ever flattens the corpus.
10954        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10955        let richest = vectors
10956            .iter()
10957            .filter_map(|(_, package)| package.as_object())
10958            .map(|object| {
10959                object
10960                    .keys()
10961                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10962                    .count()
10963            })
10964            .max()
10965            .unwrap_or(0);
10966        assert!(
10967            richest >= 2,
10968            "vendored corpus no longer carries a package with unmodelled fields, \
10969             so the relay test can no longer distinguish a verbatim relay from a lossy one"
10970        );
10971
10972        vectors
10973    }
10974
10975    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
10976    ///
10977    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
10978    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
10979    /// three-key object, so a relay that quietly dropped fields it did not
10980    /// recognise would satisfy it. These vectors carry nine keys including ones
10981    /// this crate has no type for, so a typed relay fails here and only here.
10982    #[tokio::test]
10983    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10984        for (corpus_id, package) in fed_admission_facts_vectors() {
10985            let registry = Arc::new(Registry::default());
10986            let forwarding = Arc::new(ForwardingTable::default());
10987            let supervisor = SupervisorHandle::new();
10988            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10989            let handler =
10990                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10991                    .with_supervisor(supervisor)
10992                    .with_admission_facts_config(
10993                        Some("fed".to_string()),
10994                        Some(vec!["target".to_string()]),
10995                    );
10996
10997            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10998            hello_via_sink(
10999                &handler,
11000                &target_ctx,
11001                &mut target_rx,
11002                hello_frame("target", PROTOCOL_VERSION, 1),
11003            )
11004            .await;
11005
11006            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11007            let route_handler = handler.clone();
11008            let expected = package.clone();
11009            let route_task = tokio::spawn(async move {
11010                route_handler
11011                    .handle_control_frame(
11012                        &client_ctx,
11013                        route_open_frame_with_admission_facts(
11014                            20,
11015                            "target",
11016                            unique_project_root("admission-facts"),
11017                            Some(subc_control::ConsumerIdentity {
11018                                module_id: "fed".to_string(),
11019                                launch_nonce: "fed-nonce".to_string(),
11020                            }),
11021                            Some(package),
11022                        ),
11023                    )
11024                    .await
11025                    .unwrap()
11026            });
11027
11028            let bind_frame = target_rx.recv().await.unwrap();
11029            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11030            let ModuleControlRequest::RouteBind {
11031                admission_facts, ..
11032            } = bind
11033            else {
11034                panic!("{corpus_id}: expected route.bind")
11035            };
11036            assert_eq!(
11037                admission_facts,
11038                Some(expected),
11039                "{corpus_id}: relay must not add, drop or reshape any field"
11040            );
11041
11042            handler
11043                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11044                .await
11045                .unwrap();
11046            route_task.await.unwrap();
11047        }
11048    }
11049
11050    #[tokio::test]
11051    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11052        let registry = Arc::new(Registry::default());
11053        let forwarding = Arc::new(ForwardingTable::default());
11054        let supervisor = SupervisorHandle::new();
11055        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11056        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11057        let handler =
11058            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11059                .with_supervisor(supervisor)
11060                .with_admission_facts_config(
11061                    Some("fed".to_string()),
11062                    Some(vec!["target".to_string()]),
11063                );
11064
11065        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11066        hello_via_sink(
11067            &handler,
11068            &target_ctx,
11069            &mut target_rx,
11070            hello_frame("target", PROTOCOL_VERSION, 1),
11071        )
11072        .await;
11073        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11074        hello_via_sink(
11075            &handler,
11076            &other_ctx,
11077            &mut other_rx,
11078            hello_frame("other", PROTOCOL_VERSION, 2),
11079        )
11080        .await;
11081
11082        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11083        let expected_facts = facts.clone();
11084        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11085        let route_handler = handler.clone();
11086        let route_task = tokio::spawn(async move {
11087            route_handler
11088                .handle_control_frame(
11089                    &client_ctx,
11090                    route_open_frame_with_admission_facts(
11091                        10,
11092                        "target",
11093                        unique_project_root("admission-facts"),
11094                        Some(subc_control::ConsumerIdentity {
11095                            module_id: "fed".to_string(),
11096                            launch_nonce: "fed-nonce".to_string(),
11097                        }),
11098                        Some(facts.clone()),
11099                    ),
11100                )
11101                .await
11102                .unwrap()
11103        });
11104        let bind_frame = target_rx.recv().await.unwrap();
11105        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11106        let ModuleControlRequest::RouteBind {
11107            admission_facts, ..
11108        } = bind
11109        else {
11110            panic!("expected route.bind")
11111        };
11112        assert_eq!(admission_facts, Some(expected_facts));
11113        handler
11114            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11115            .await
11116            .unwrap();
11117        assert!(route_task.await.unwrap().is_empty());
11118        assert!(matches!(
11119            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11120                .unwrap(),
11121            ClientControlResponse::RouteOpen { .. }
11122        ));
11123
11124        let direct = handler
11125            .handle_control_frame(
11126                &route_ctx(ConnectionId::new(73)).0,
11127                route_open_frame_with_admission_facts(
11128                    11,
11129                    "target",
11130                    unique_project_root("admission-facts"),
11131                    None,
11132                    Some(json!({"x": 1})),
11133                ),
11134            )
11135            .await
11136            .unwrap();
11137        assert_eq!(
11138            parse_error(&direct[0])["code"],
11139            "admission_facts_not_permitted"
11140        );
11141
11142        let different_reserved = handler
11143            .handle_control_frame(
11144                &route_ctx(ConnectionId::new(77)).0,
11145                route_open_frame_with_admission_facts(
11146                    15,
11147                    "target",
11148                    unique_project_root("admission-facts"),
11149                    Some(subc_control::ConsumerIdentity {
11150                        module_id: "other".to_string(),
11151                        launch_nonce: "other-nonce".to_string(),
11152                    }),
11153                    Some(json!({"x": 1})),
11154                ),
11155            )
11156            .await
11157            .unwrap();
11158        assert_eq!(
11159            parse_error(&different_reserved[0])["code"],
11160            "admission_facts_not_permitted"
11161        );
11162
11163        let other_target = handler
11164            .handle_control_frame(
11165                &route_ctx(ConnectionId::new(74)).0,
11166                route_open_frame_with_admission_facts(
11167                    12,
11168                    "other",
11169                    unique_project_root("admission-facts"),
11170                    Some(subc_control::ConsumerIdentity {
11171                        module_id: "fed".to_string(),
11172                        launch_nonce: "fed-nonce".to_string(),
11173                    }),
11174                    Some(json!({"x": 1})),
11175                ),
11176            )
11177            .await
11178            .unwrap();
11179        assert_eq!(
11180            parse_error(&other_target[0])["code"],
11181            "admission_facts_target_not_allowed"
11182        );
11183
11184        let nonexistent = handler
11185            .handle_control_frame(
11186                &route_ctx(ConnectionId::new(75)).0,
11187                route_open_frame_with_admission_facts(
11188                    13,
11189                    "missing",
11190                    unique_project_root("admission-facts"),
11191                    None,
11192                    Some(json!({"x": 1})),
11193                ),
11194            )
11195            .await
11196            .unwrap();
11197        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
11198
11199        let described = handler
11200            .handle_control_frame(
11201                &route_ctx(ConnectionId::new(76)).0,
11202                Frame::build(
11203                    FrameType::Request,
11204                    control_flags(),
11205                    0,
11206                    0,
11207                    14,
11208                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11209                )
11210                .unwrap(),
11211            )
11212            .await
11213            .unwrap();
11214        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11215            serde_json::from_slice(&described[0].body).unwrap()
11216        else {
11217            panic!("expected server.describe response")
11218        };
11219        assert!(capabilities
11220            .iter()
11221            .any(|cap| cap == "admission_facts_relay_v1"));
11222    }
11223
11224    #[tokio::test]
11225    async fn admission_facts_without_configured_carrier_are_rejected() {
11226        let registry = Arc::new(Registry::default());
11227        let forwarding = Arc::new(ForwardingTable::default());
11228        let handler = ControlHandler::with_forwarding(registry, forwarding);
11229        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
11230        hello_via_sink(
11231            &handler,
11232            &target_ctx,
11233            &mut target_rx,
11234            hello_frame("target", PROTOCOL_VERSION, 1),
11235        )
11236        .await;
11237
11238        let responses = handler
11239            .handle_control_frame(
11240                &route_ctx(ConnectionId::new(79)).0,
11241                route_open_frame_with_admission_facts(
11242                    16,
11243                    "target",
11244                    unique_project_root("admission-facts"),
11245                    None,
11246                    Some(json!({"x": 1})),
11247                ),
11248            )
11249            .await
11250            .unwrap();
11251        assert_eq!(
11252            parse_error(&responses[0])["code"],
11253            "admission_facts_not_permitted"
11254        );
11255    }
11256
11257    #[tokio::test]
11258    async fn route_open_relays_consumer_capabilities_verbatim() {
11259        let registry = Arc::new(Registry::default());
11260        let forwarding = Arc::new(ForwardingTable::default());
11261        let handler =
11262            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11263        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
11264        hello_via_sink(
11265            &handler,
11266            &module_ctx,
11267            &mut module_rx,
11268            hello_frame("aft", PROTOCOL_VERSION, 7),
11269        )
11270        .await;
11271
11272        let expected = vec!["elicitation".to_string(), "roots".to_string()];
11273        let expected_for_request = expected.clone();
11274        let project_root = unique_project_root("consumer-capabilities-present");
11275        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
11276        let route_handler = handler.clone();
11277        let route_task = tokio::spawn(async move {
11278            route_handler
11279                .handle_control_frame(
11280                    &client_ctx,
11281                    route_open_frame_with_consumer_capabilities(
11282                        401,
11283                        "aft",
11284                        project_root,
11285                        Some(expected_for_request),
11286                    ),
11287                )
11288                .await
11289                .unwrap()
11290        });
11291        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11292            .await
11293            .unwrap()
11294            .unwrap();
11295        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11296        let ModuleControlRequest::RouteBind {
11297            consumer_capabilities,
11298            ..
11299        } = bind
11300        else {
11301            panic!("expected route.bind request, got {bind:?}");
11302        };
11303        assert_eq!(consumer_capabilities, Some(expected.clone()));
11304
11305        handler
11306            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11307            .await
11308            .unwrap();
11309        let route_response = route_task.await.unwrap();
11310        assert!(route_response.is_empty());
11311        let published = client_rx.recv().await.unwrap();
11312        assert!(matches!(
11313            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11314            ClientControlResponse::RouteOpen { .. }
11315        ));
11316    }
11317
11318    #[tokio::test]
11319    async fn route_open_without_consumer_capabilities_relays_none() {
11320        let registry = Arc::new(Registry::default());
11321        let forwarding = Arc::new(ForwardingTable::default());
11322        let handler =
11323            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11324        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11325        hello_via_sink(
11326            &handler,
11327            &module_ctx,
11328            &mut module_rx,
11329            hello_frame("aft", PROTOCOL_VERSION, 7),
11330        )
11331        .await;
11332
11333        let project_root = unique_project_root("consumer-capabilities-absent");
11334        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11335        let route_handler = handler.clone();
11336        let route_task = tokio::spawn(async move {
11337            route_handler
11338                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11339                .await
11340                .unwrap()
11341        });
11342        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11343            .await
11344            .unwrap()
11345            .unwrap();
11346        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11347        let ModuleControlRequest::RouteBind {
11348            consumer_capabilities,
11349            ..
11350        } = bind
11351        else {
11352            panic!("expected route.bind request, got {bind:?}");
11353        };
11354        assert_eq!(consumer_capabilities, None);
11355
11356        handler
11357            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11358            .await
11359            .unwrap();
11360        let route_response = route_task.await.unwrap();
11361        assert!(route_response.is_empty());
11362        let published = client_rx.recv().await.unwrap();
11363        assert!(matches!(
11364            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11365            ClientControlResponse::RouteOpen { .. }
11366        ));
11367    }
11368
11369    /// Opens a route to a freshly registered `aft` with `sent` as the
11370    /// route.open's role_versions, acks the bind, and returns the role_versions
11371    /// the module's bind carried.
11372    async fn bind_role_versions_for(
11373        sent: Option<BTreeMap<String, String>>,
11374        connection: u64,
11375    ) -> Option<BTreeMap<String, String>> {
11376        let registry = Arc::new(Registry::default());
11377        let forwarding = Arc::new(ForwardingTable::default());
11378        let handler =
11379            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11380        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11381        hello_via_sink(
11382            &handler,
11383            &module_ctx,
11384            &mut module_rx,
11385            hello_frame("aft", PROTOCOL_VERSION, 7),
11386        )
11387        .await;
11388        let project_root = unique_project_root("role-versions");
11389        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11390        let route_handler = handler.clone();
11391        let route_task = tokio::spawn(async move {
11392            route_handler
11393                .handle_control_frame(
11394                    &client_ctx,
11395                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
11396                )
11397                .await
11398                .unwrap()
11399        });
11400        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11401            .await
11402            .expect("a well-formed route.open reaches the module as a bind")
11403            .unwrap();
11404        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11405        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11406            panic!("expected route.bind request, got {bind:?}");
11407        };
11408        handler
11409            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11410            .await
11411            .unwrap();
11412        assert!(route_task.await.unwrap().is_empty());
11413        let published = client_rx.recv().await.unwrap();
11414        assert!(matches!(
11415            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11416            ClientControlResponse::RouteOpen { .. }
11417        ));
11418        role_versions
11419    }
11420
11421    #[tokio::test]
11422    async fn route_open_relays_role_versions_verbatim() {
11423        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11424        assert_eq!(
11425            bind_role_versions_for(Some(sent.clone()), 141).await,
11426            Some(sent)
11427        );
11428    }
11429
11430    /// An empty map declares nothing, so the provider sees no field rather
11431    /// than an empty object it would have to treat as a second "none".
11432    #[tokio::test]
11433    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11434        assert_eq!(bind_role_versions_for(None, 143).await, None);
11435        assert_eq!(
11436            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11437            None
11438        );
11439    }
11440
11441    #[tokio::test]
11442    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11443        let registry = Arc::new(Registry::default());
11444        let forwarding = Arc::new(ForwardingTable::default());
11445        let handler =
11446            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11447        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11448        hello_via_sink(
11449            &handler,
11450            &module_ctx,
11451            &mut module_rx,
11452            hello_frame("aft", PROTOCOL_VERSION, 7),
11453        )
11454        .await;
11455
11456        let nine: BTreeMap<String, String> = (0..9)
11457            .map(|index| (format!("role-{index}"), "v1".to_string()))
11458            .collect();
11459        for (label, malformed) in [
11460            (
11461                "invalid role name",
11462                role_versions(&[("Tool_Provider", "v1")]),
11463            ),
11464            ("invalid version", role_versions(&[("tool-provider", "v0")])),
11465            ("nine entries", nine),
11466        ] {
11467            // A refused open answers at once; one that reached the module
11468            // would wait for its bind ack and trip this timeout.
11469            let responses = tokio::time::timeout(
11470                Duration::from_secs(1),
11471                handler.handle_control_frame(
11472                    &route_ctx(ConnectionId::new(148)).0,
11473                    route_open_frame_with_role_versions(
11474                        404,
11475                        "aft",
11476                        unique_project_root("role-versions-malformed"),
11477                        Some(malformed),
11478                    ),
11479                ),
11480            )
11481            .await
11482            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11483            .unwrap();
11484            assert_eq!(responses.len(), 1, "{label}");
11485            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11486            let error = parse_error(&responses[0]);
11487            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11488            assert_eq!(
11489                error["detail"]["field"], "role_versions",
11490                "{label}: {error}"
11491            );
11492            assert!(
11493                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11494                "{label}: a malformed declaration is terminal"
11495            );
11496            assert!(
11497                module_rx.try_recv().is_err(),
11498                "{label}: the module must never see a bind"
11499            );
11500        }
11501    }
11502
11503    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
11504    /// consumer can tell this daemon forwards the field from one that would
11505    /// drop it.
11506    #[tokio::test]
11507    async fn route_role_versions_capability_is_advertised() {
11508        let handler = ControlHandler::new(Arc::new(Registry::default()));
11509        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11510        let ack = hello_via_sink(
11511            &handler,
11512            &ctx,
11513            &mut rx,
11514            hello_frame("m", PROTOCOL_VERSION, 1),
11515        )
11516        .await;
11517        let ack = parse_ack(&ack);
11518        assert!(
11519            ack.subc_capabilities
11520                .iter()
11521                .any(|c| c == "route-role-versions/v1"),
11522            "{:?}",
11523            ack.subc_capabilities
11524        );
11525
11526        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11527        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11528        let reply = handler
11529            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11530            .await
11531            .unwrap()
11532            .pop()
11533            .unwrap();
11534        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11535            serde_json::from_slice(&reply.body).unwrap()
11536        else {
11537            panic!("not a server.describe reply");
11538        };
11539        assert!(
11540            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11541            "{capabilities:?}"
11542        );
11543    }
11544
11545    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
11546    /// catalog entry without `events` (the module declares none) from a daemon
11547    /// that never reports declared events.
11548    #[tokio::test]
11549    async fn catalog_events_capability_is_advertised() {
11550        let handler = ControlHandler::new(Arc::new(Registry::default()));
11551        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11552        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11553        let reply = handler
11554            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11555            .await
11556            .unwrap()
11557            .pop()
11558            .unwrap();
11559        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11560            serde_json::from_slice(&reply.body).unwrap()
11561        else {
11562            panic!("not a server.describe reply");
11563        };
11564        assert!(
11565            capabilities
11566                .iter()
11567                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11568            "{capabilities:?}"
11569        );
11570    }
11571
11572    #[tokio::test]
11573    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
11574        let handler = ControlHandler::new(Arc::new(Registry::default()));
11575        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
11576        let ack = hello_via_sink(
11577            &handler,
11578            &module,
11579            &mut module_rx,
11580            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
11581        )
11582        .await;
11583        let ack = parse_ack(&ack);
11584        assert!(
11585            ack.subc_capabilities
11586                .iter()
11587                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11588            "{:?}",
11589            ack.subc_capabilities
11590        );
11591        assert!(ack
11592            .subc_capabilities
11593            .iter()
11594            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11595
11596        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
11597        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11598        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11599        let reply = handler
11600            .handle_control_frame(&client, frame)
11601            .await
11602            .unwrap()
11603            .pop()
11604            .unwrap();
11605        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11606            serde_json::from_slice(&reply.body).unwrap()
11607        else {
11608            panic!("not a server.describe reply");
11609        };
11610        assert!(
11611            capabilities
11612                .iter()
11613                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11614            "{capabilities:?}"
11615        );
11616        assert!(capabilities
11617            .iter()
11618            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11619    }
11620
11621    #[tokio::test]
11622    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
11623        let handler = ControlHandler::new(Arc::new(Registry::default()));
11624        let (module, mut rx) = route_ctx(ConnectionId::new(156));
11625        let ack = hello_via_sink(
11626            &handler,
11627            &module,
11628            &mut rx,
11629            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
11630        )
11631        .await;
11632        assert!(parse_ack(&ack)
11633            .subc_capabilities
11634            .iter()
11635            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11636        assert!(parse_ack(&ack)
11637            .subc_capabilities
11638            .iter()
11639            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
11640        let (client, _rx) = route_ctx(ConnectionId::new(157));
11641        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11642        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11643        let reply = handler
11644            .handle_control_frame(&client, frame)
11645            .await
11646            .unwrap()
11647            .pop()
11648            .unwrap();
11649        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11650            serde_json::from_slice(&reply.body).unwrap()
11651        else {
11652            panic!("server.describe reply")
11653        };
11654        assert!(capabilities
11655            .iter()
11656            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11657        assert!(capabilities
11658            .iter()
11659            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
11660    }
11661
11662    #[tokio::test]
11663    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
11664        let handler = ControlHandler::new(Arc::new(Registry::default()));
11665        let (module, mut rx) = route_ctx(ConnectionId::new(154));
11666        let ack = hello_via_sink(
11667            &handler,
11668            &module,
11669            &mut rx,
11670            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
11671        )
11672        .await;
11673        assert!(parse_ack(&ack)
11674            .subc_capabilities
11675            .iter()
11676            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11677        let (client, _rx) = route_ctx(ConnectionId::new(155));
11678        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11679        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11680        let reply = handler
11681            .handle_control_frame(&client, frame)
11682            .await
11683            .unwrap()
11684            .pop()
11685            .unwrap();
11686        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11687            serde_json::from_slice(&reply.body).unwrap()
11688        else {
11689            panic!("not a server.describe reply")
11690        };
11691        assert!(capabilities
11692            .iter()
11693            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11694    }
11695
11696    #[tokio::test]
11697    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11698        let registry = Arc::new(Registry::default());
11699        let forwarding = Arc::new(ForwardingTable::default());
11700        let handler =
11701            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11702                .with_health_probe_timeout(Duration::from_secs(5));
11703        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11704        hello_via_sink(
11705            &handler,
11706            &module_ctx,
11707            &mut module_rx,
11708            non_routable_hello_frame_with_control_ops(
11709                "mcp",
11710                300,
11711                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11712            ),
11713        )
11714        .await;
11715        assert!(registry
11716            .get_module("mcp")
11717            .unwrap()
11718            .unwrap()
11719            .manifest
11720            .provides
11721            .is_empty());
11722
11723        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11724        let route_response = handler
11725            .handle_control_frame(
11726                &route_client_ctx,
11727                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11728            )
11729            .await
11730            .unwrap();
11731        assert_eq!(route_response[0].header.ty, FrameType::Error);
11732        assert_eq!(
11733            parse_error(&route_response[0])["code"],
11734            "target_unavailable"
11735        );
11736        assert!(parse_error(&route_response[0])["message"]
11737            .as_str()
11738            .unwrap()
11739            .contains("does not provide the requested target"));
11740        assert!(module_rx.try_recv().is_err());
11741
11742        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11743        let health_handler = handler.clone();
11744        let health_task = tokio::spawn(async move {
11745            health_handler
11746                .handle_control_frame(
11747                    &health_client_ctx,
11748                    supervisor_health_probe_frame(302, "mcp"),
11749                )
11750                .await
11751                .unwrap()
11752        });
11753        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11754            .await
11755            .unwrap()
11756            .unwrap();
11757        assert_eq!(
11758            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11759            ModuleControlRequest::HealthCheck {}
11760        );
11761        handler
11762            .handle_control_frame(
11763                &module_ctx,
11764                health_response(health_frame.header.corr, HealthStatus::Ok),
11765            )
11766            .await
11767            .unwrap();
11768        let health_response = health_task.await.unwrap();
11769        assert_eq!(health_response[0].header.ty, FrameType::Response);
11770        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11771            ClientControlResponse::SupervisorHealthProbe {
11772                module_id, status, ..
11773            } => {
11774                assert_eq!(module_id, "mcp");
11775                assert_eq!(status, HealthStatus::Ok);
11776            }
11777            other => panic!("unexpected health response: {other:?}"),
11778        }
11779
11780        // Exercise the forwarding cleanup path directly while leaving the registry
11781        // advertisement in place. If cleanup leaves a stale control sink behind,
11782        // the next probe will enqueue onto it and wait for the long probe timeout
11783        // instead of returning an immediate no-connection error.
11784        forwarding
11785            .cleanup_connection(module_ctx.connection_id)
11786            .unwrap();
11787        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11788        let cleanup_response = tokio::time::timeout(
11789            Duration::from_millis(200),
11790            handler.handle_control_frame(
11791                &cleanup_probe_ctx,
11792                supervisor_health_probe_frame(303, "mcp"),
11793            ),
11794        )
11795        .await
11796        .expect("probe should fail immediately when the control lane is gone")
11797        .unwrap();
11798        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11799        assert_eq!(
11800            parse_error(&cleanup_response[0])["code"],
11801            "target_unavailable"
11802        );
11803        assert!(parse_error(&cleanup_response[0])["message"]
11804            .as_str()
11805            .unwrap()
11806            .contains("no module connection"));
11807
11808        handler
11809            .cleanup_connection(module_ctx.connection_id)
11810            .unwrap();
11811    }
11812
11813    #[tokio::test]
11814    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11815        let registry = Arc::new(Registry::default());
11816        let supervisor_handle = SupervisorHandle::new();
11817        let supervisor =
11818            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
11819                .with_handle(supervisor_handle.clone())
11820                .with_connection_file_path(
11821                    std::env::temp_dir()
11822                        .join(format!("subc-route-open-warming-{}", std::process::id())),
11823                );
11824        let module = supervisor
11825            .supervise_configured(
11826                ModuleSpec {
11827                    module_id: "warming".to_string(),
11828                    program: fake_aft_stub_path(),
11829                    args: Vec::new(),
11830                    env: Vec::new(),
11831                    reserved: false,
11832                    reserved_prefixes: Vec::new(),
11833                    protocol: ModuleProtocol::Subc,
11834                    overlap: Default::default(),
11835                },
11836                true,
11837            )
11838            .unwrap();
11839        assert_eq!(module.state().unwrap(), ModuleState::Running);
11840
11841        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11842        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11843        let response = handler
11844            .handle_control_frame(
11845                &ctx,
11846                route_open_frame(304, "warming", unique_project_root("warming")),
11847            )
11848            .await
11849            .unwrap();
11850        module.stop().await.unwrap();
11851
11852        assert_eq!(response[0].header.ty, FrameType::Error);
11853        let error = parse_error(&response[0]);
11854        assert_eq!(error["code"], "module_warming");
11855        assert!(error["message"]
11856            .as_str()
11857            .unwrap()
11858            .contains("state=running, enabled=true, live=false"));
11859    }
11860
11861    #[test]
11862    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11863        let handler = ControlHandler::new(Arc::new(Registry::default()));
11864        let capture = EventCapture::default();
11865        let _subscriber =
11866            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11867        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11868        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11869        let pending = (0..limit).collect::<Vec<_>>();
11870        let response = handler
11871            .route_open_capacity_refusal(
11872                &ctx,
11873                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11874                "busy",
11875                pending.len(),
11876                limit,
11877            )
11878            .unwrap();
11879        assert_eq!(parse_error(&response)["code"], "target_unavailable");
11880        let event = capture
11881            .events()
11882            .into_iter()
11883            .find(|event| {
11884                event.target == "control"
11885                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11886            })
11887            .expect("connection admission refusal event");
11888        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11889        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11890    }
11891
11892    #[test]
11893    fn route_open_target_cap_logs_admission_reason_and_capacity() {
11894        let handler = ControlHandler::new(Arc::new(Registry::default()));
11895        let capture = EventCapture::default();
11896        let _subscriber =
11897            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11898        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
11899        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
11900        let guards = (0..limit)
11901            .map(|_| {
11902                handler
11903                    .route_bind_concurrency
11904                    .try_admit("busy", limit)
11905                    .unwrap()
11906            })
11907            .collect::<Vec<_>>();
11908        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
11909            Err(in_flight) => in_flight,
11910            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
11911        };
11912        let response = handler
11913            .route_open_target_capacity_refusal(
11914                &ctx,
11915                &route_open_frame(397, "busy", unique_project_root("target-cap")),
11916                "busy",
11917                in_flight,
11918            )
11919            .unwrap();
11920        assert_eq!(parse_error(&response)["code"], "target_unavailable");
11921        let event = capture
11922            .events()
11923            .into_iter()
11924            .find(|event| {
11925                event.target == "control"
11926                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
11927            })
11928            .expect("target admission refusal event");
11929        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11930        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11931        drop(guards);
11932    }
11933
11934    #[tokio::test(flavor = "current_thread")]
11935    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
11936        let handler = ControlHandler::new(Arc::new(Registry::default()));
11937        let capture = EventCapture::default();
11938        let _subscriber =
11939            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11940        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
11941        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11942
11943        let _ = handler
11944            .handle_client_control_request(
11945                &ctx,
11946                frame,
11947                ClientControlRequest::SupervisorSetEnabled {
11948                    module_id: "broca".to_string(),
11949                    enabled: false,
11950                },
11951                &mut None,
11952            )
11953            .await
11954            .unwrap();
11955
11956        let events = capture
11957            .events()
11958            .into_iter()
11959            .filter(|event| {
11960                event.target == "control"
11961                    && event.fields.get("message").map(String::as_str)
11962                        == Some("supervisor request received")
11963            })
11964            .collect::<Vec<_>>();
11965        assert_eq!(events.len(), 1, "one supervisor request log line");
11966        let event = &events[0];
11967        assert_eq!(event.level, tracing::Level::INFO);
11968        assert_eq!(
11969            event.fields.get("op"),
11970            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
11971        );
11972        assert_eq!(
11973            event.fields.get("module_id"),
11974            Some(&"\"broca\"".to_string())
11975        );
11976        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
11977        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
11978        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
11979    }
11980
11981    #[tokio::test(flavor = "current_thread")]
11982    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
11983        let registry = Arc::new(Registry::default());
11984        let module_connection = ConnectionId::new(303);
11985        registry
11986            .register_with_control_ops(
11987                manifest("aft", PROTOCOL_VERSION),
11988                PROTOCOL_VERSION,
11989                module_connection,
11990                module_baseline_control_ops(),
11991            )
11992            .unwrap();
11993        let handler = ControlHandler::new(registry);
11994        let capture = EventCapture::default();
11995        let _subscriber =
11996            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11997        let (ctx, _rx) = route_ctx(module_connection);
11998        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11999
12000        let _ = handler
12001            .handle_client_control_request(
12002                &ctx,
12003                frame,
12004                ClientControlRequest::SupervisorSetEnabled {
12005                    module_id: "broca".to_string(),
12006                    enabled: false,
12007                },
12008                &mut None,
12009            )
12010            .await
12011            .unwrap();
12012
12013        let callers = capture
12014            .events()
12015            .into_iter()
12016            .filter(|event| {
12017                event.fields.get("message").map(String::as_str)
12018                    == Some("supervisor request received")
12019            })
12020            .map(|event| event.fields.get("caller").cloned())
12021            .collect::<Vec<_>>();
12022        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12023    }
12024
12025    #[tokio::test(flavor = "current_thread")]
12026    async fn supervisor_list_does_not_log_request_received() {
12027        let handler = ControlHandler::new(Arc::new(Registry::default()));
12028        let capture = EventCapture::default();
12029        let _subscriber =
12030            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12031        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12032        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12033
12034        handler
12035            .handle_client_control_request(
12036                &ctx,
12037                frame,
12038                ClientControlRequest::SupervisorList {},
12039                &mut None,
12040            )
12041            .await
12042            .unwrap();
12043
12044        assert!(capture.events().into_iter().all(|event| {
12045            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12046        }));
12047    }
12048
12049    #[tokio::test(flavor = "current_thread")]
12050    async fn supervisor_rescan_preview_does_not_log_request_received() {
12051        let handler = ControlHandler::new(Arc::new(Registry::default()));
12052        let capture = EventCapture::default();
12053        let _subscriber =
12054            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12055        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
12056        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12057
12058        handler
12059            .handle_client_control_request(
12060                &ctx,
12061                frame,
12062                ClientControlRequest::SupervisorRescan { preview: true },
12063                &mut None,
12064            )
12065            .await
12066            .unwrap();
12067
12068        assert!(capture.events().into_iter().all(|event| {
12069            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12070        }));
12071    }
12072
12073    struct ApproveOperator;
12074    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
12075        fn prompt(
12076            &self,
12077            _: &str,
12078            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
12079        ) -> crate::operator_confirm::ProviderResult {
12080            crate::operator_confirm::ProviderResult::Approved
12081        }
12082    }
12083
12084    #[tokio::test]
12085    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
12086        assert_eq!(
12087            ControlHandler::default().privileged_operator_prompt,
12088            PrivilegedOperatorPrompt::Log
12089        );
12090        for (initial, changed, running) in [
12091            ("log", "enforce", PrivilegedOperatorPrompt::Log),
12092            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
12093        ] {
12094            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
12095            let path = dir.join("subc.jsonc");
12096            std::fs::write(
12097                &path,
12098                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
12099            )
12100            .unwrap();
12101            let config = crate::daemon_config::load(&path).unwrap().unwrap();
12102            let registry = Arc::new(Registry::default());
12103            let handle = SupervisorHandle::new();
12104            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
12105                .with_handle(handle.clone());
12106            let handler = ControlHandler::new(registry)
12107                .with_supervisor(handle)
12108                .with_privileged_operator_prompt(config.privileged_operator_prompt)
12109                .with_supervisor_rescan(supervisor, &path, config.port);
12110            handler.forwarding.operator_confirms().configure(
12111                Arc::new(ApproveOperator),
12112                crate::operator_confirm::OperatorLimits::default(),
12113            );
12114            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
12115
12116            for (disk_mode, needs_restart) in [
12117                (Some(initial), false),
12118                (Some(changed), true),
12119                (Some(changed), true),
12120                (None, running == PrivilegedOperatorPrompt::Enforce),
12121                (Some(initial), false),
12122            ] {
12123                let mut doc = json!({"version":1});
12124                if let Some(mode) = disk_mode {
12125                    doc["privileged_operator_prompt"] = json!(mode);
12126                }
12127                std::fs::write(&path, doc.to_string()).unwrap();
12128                for preview in [true, false] {
12129                    let request = ClientControlRequest::SupervisorRescan { preview };
12130                    let frame = Frame::build(
12131                        FrameType::Request,
12132                        control_flags(),
12133                        0,
12134                        0,
12135                        1,
12136                        serde_json::to_vec(&request).unwrap(),
12137                    )
12138                    .unwrap();
12139                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12140                    assert_eq!(frames.len(), 1);
12141                    let ClientControlResponse::SupervisorRescan { result } =
12142                        serde_json::from_slice(&frames[0].body).unwrap()
12143                    else {
12144                        panic!("expected a rescan response");
12145                    };
12146                    let expected = if needs_restart {
12147                        vec!["privileged_operator_prompt"]
12148                    } else {
12149                        Vec::new()
12150                    };
12151                    assert_eq!(result.restart_required, expected);
12152                    assert_eq!(handler.privileged_operator_prompt, running);
12153                    assert_eq!(
12154                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
12155                        running
12156                    );
12157                }
12158            }
12159        }
12160    }
12161
12162    #[tokio::test]
12163    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
12164        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
12165        let path = dir.join("subc.jsonc");
12166        let registry = Arc::new(Registry::default());
12167        let handle = SupervisorHandle::new();
12168        let supervisor =
12169            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
12170        let handler = ControlHandler::new(registry)
12171            .with_supervisor(handle)
12172            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
12173            .with_supervisor_rescan(supervisor, &path, None);
12174        handler.forwarding.operator_confirms().configure(
12175            Arc::new(ApproveOperator),
12176            crate::operator_confirm::OperatorLimits::default(),
12177        );
12178        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
12179        for preview in [true, false] {
12180            std::fs::write(
12181                &path,
12182                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
12183            )
12184            .unwrap();
12185            let request = ClientControlRequest::SupervisorRescan { preview };
12186            let frame = Frame::build(
12187                FrameType::Request,
12188                control_flags(),
12189                0,
12190                0,
12191                1,
12192                serde_json::to_vec(&request).unwrap(),
12193            )
12194            .unwrap();
12195            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12196            assert_eq!(frames.len(), 1);
12197            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
12198            assert_eq!(
12199                handler.privileged_operator_prompt,
12200                PrivilegedOperatorPrompt::Enforce
12201            );
12202        }
12203    }
12204
12205    /// One wire code has several senders, so the refusal line names the check
12206    /// that refused. This drives the shared refusal path for ordinary refusals
12207    /// with an unregistered
12208    /// target and requires the branch label on the event.
12209    #[tokio::test]
12210    async fn route_open_refusal_names_the_check_that_refused() {
12211        let handler = ControlHandler::new(Arc::new(Registry::default()));
12212        let capture = EventCapture::default();
12213        let _subscriber =
12214            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12215        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12216        let response = handler
12217            .handle_control_frame(
12218                &ctx,
12219                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
12220            )
12221            .await
12222            .unwrap();
12223
12224        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12225        let event = capture
12226            .events()
12227            .into_iter()
12228            .find(|event| {
12229                event.target == "control"
12230                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12231            })
12232            .expect("route.open refusal event");
12233        assert_eq!(
12234            event.fields.get("reason"),
12235            Some(&"\"not_registered\"".to_string())
12236        );
12237    }
12238
12239    #[tokio::test]
12240    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
12241        let registry = Arc::new(Registry::default());
12242        let supervisor_handle = SupervisorHandle::new();
12243        let supervisor =
12244            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12245                .with_handle(supervisor_handle.clone())
12246                .with_connection_file_path(std::env::temp_dir().join(format!(
12247                    "subc-route-open-refusal-info-{}",
12248                    std::process::id()
12249                )));
12250        let module = supervisor
12251            .supervise_configured(
12252                ModuleSpec {
12253                    module_id: "warming".to_string(),
12254                    program: fake_aft_stub_path(),
12255                    args: Vec::new(),
12256                    env: Vec::new(),
12257                    reserved: false,
12258                    reserved_prefixes: Vec::new(),
12259                    protocol: ModuleProtocol::Subc,
12260                    overlap: Default::default(),
12261                },
12262                true,
12263            )
12264            .unwrap();
12265        assert_eq!(module.state().unwrap(), ModuleState::Running);
12266
12267        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12268        assert!(handler
12269            .counters()
12270            .snapshot()
12271            .get("route_open_refused_by_code")
12272            .is_none());
12273        let capture = EventCapture::default();
12274        let _subscriber =
12275            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12276        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
12277        let response = handler
12278            .handle_control_frame(
12279                &ctx,
12280                route_open_frame(394, "warming", unique_project_root("refusal-info")),
12281            )
12282            .await
12283            .unwrap();
12284        module.stop().await.unwrap();
12285
12286        assert_eq!(parse_error(&response[0])["code"], "module_warming");
12287        let event = capture
12288            .events()
12289            .into_iter()
12290            .find(|event| {
12291                event.target == "control"
12292                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
12293            })
12294            .expect("route.open refusal event");
12295        assert_eq!(
12296            event.fields.get("module_id"),
12297            Some(&"\"warming\"".to_string())
12298        );
12299        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
12300        assert_eq!(
12301            event.fields.get("reason"),
12302            Some(&"\"supervised_not_registered\"".to_string())
12303        );
12304        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
12305        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
12306        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
12307        assert_eq!(
12308            handler.counters().snapshot()["route_open_refused_by_code"],
12309            json!({ "module_warming": 1 })
12310        );
12311    }
12312
12313    const OUTAGE_START: &str = "route.open refusing module: not serving";
12314    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
12315
12316    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
12317        capture
12318            .events()
12319            .into_iter()
12320            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
12321            .collect()
12322    }
12323
12324    fn supervise_stub(
12325        registry: &Arc<Registry>,
12326        module_id: &str,
12327        enabled: bool,
12328    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
12329        let supervisor_handle = SupervisorHandle::new();
12330        let supervisor =
12331            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
12332                .with_handle(supervisor_handle.clone())
12333                .with_connection_file_path(std::env::temp_dir().join(format!(
12334                    "subc-route-outage-{module_id}-{}",
12335                    std::process::id()
12336                )));
12337        let module = supervisor
12338            .supervise_configured(
12339                ModuleSpec {
12340                    module_id: module_id.to_string(),
12341                    program: fake_aft_stub_path(),
12342                    args: Vec::new(),
12343                    env: Vec::new(),
12344                    reserved: false,
12345                    reserved_prefixes: Vec::new(),
12346                    protocol: ModuleProtocol::Subc,
12347                    overlap: Default::default(),
12348                },
12349                enabled,
12350            )
12351            .unwrap();
12352        (supervisor_handle, module)
12353    }
12354
12355    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
12356        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
12357            module_id: module_id.to_string(),
12358            drain_timeout_ms: Some(50),
12359        })
12360        .unwrap();
12361        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
12362    }
12363
12364    /// Two handlers built over one forwarding table must share one outage
12365    /// tracker; separate trackers would each log their own opening line for
12366    /// the same outage.
12367    #[test]
12368    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
12369        let registry = Arc::new(Registry::default());
12370        let forwarding = Arc::new(ForwardingTable::default());
12371        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12372        let second = ControlHandler::with_forwarding(registry, forwarding);
12373        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
12374    }
12375
12376    /// A client can name any module id it likes. Refusing an unknown one,
12377    /// however often, must not create outage state or outage lines, or the
12378    /// tracker would be a memory sink any client could fill.
12379    #[tokio::test(flavor = "current_thread")]
12380    async fn route_open_unknown_module_refusals_add_no_outage_state() {
12381        let handler = ControlHandler::new(Arc::new(Registry::default()));
12382        let capture = EventCapture::default();
12383        let _subscriber =
12384            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12385        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12386        for corr in 0..8 {
12387            let response = handler
12388                .handle_control_frame(
12389                    &ctx,
12390                    route_open_frame(
12391                        380 + corr,
12392                        &format!("nobody-{corr}"),
12393                        unique_project_root("outage-unknown"),
12394                    ),
12395                )
12396                .await
12397                .unwrap();
12398            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12399        }
12400
12401        assert_eq!(handler.route_outages.tracked_module_count(), 0);
12402        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12403        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12404    }
12405
12406    /// Drives the refusal path end to end: a supervised module that served
12407    /// before and stopped being registered with no instruction to stop is a
12408    /// WARN, and the same module refused after an operator `supervisor.restart`
12409    /// is an INFO.
12410    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12411    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12412        let registry = Arc::new(Registry::default());
12413        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
12414        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12415        let capture = EventCapture::default();
12416        let _subscriber =
12417            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12418        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12419        // The stub never registers, so pretend it served once: otherwise every
12420        // refusal would fall in its startup window.
12421        handler.route_outages.record_accepted("outage-restart");
12422
12423        let response = handler
12424            .handle_control_frame(
12425                &ctx,
12426                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12427            )
12428            .await
12429            .unwrap();
12430        assert_eq!(response[0].header.ty, FrameType::Error);
12431        let starts = outage_lines(&capture, OUTAGE_START);
12432        assert_eq!(starts.len(), 1, "{starts:?}");
12433        assert_eq!(starts[0].level, tracing::Level::WARN);
12434        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12435        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12436        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12437        handler.route_outages.record_accepted("outage-restart");
12438        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12439
12440        let restart = handler
12441            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12442            .await
12443            .unwrap();
12444        assert_eq!(
12445            restart[0].header.ty,
12446            FrameType::Response,
12447            "{:?}",
12448            parse_error(&restart[0])
12449        );
12450        handler
12451            .handle_control_frame(
12452                &ctx,
12453                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
12454            )
12455            .await
12456            .unwrap();
12457        module.stop().await.unwrap();
12458
12459        let starts = outage_lines(&capture, OUTAGE_START);
12460        assert_eq!(starts.len(), 2, "{starts:?}");
12461        assert_eq!(starts[1].level, tracing::Level::INFO);
12462        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
12463    }
12464
12465    /// A restart refused before it touched the module (here: the module is
12466    /// disabled) must clear its operator mark, so the next real outage is
12467    /// still reported as a warning.
12468    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12469    async fn failed_operator_restart_leaves_no_operator_mark() {
12470        let registry = Arc::new(Registry::default());
12471        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
12472        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12473        let capture = EventCapture::default();
12474        let _subscriber =
12475            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12476        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
12477        handler.route_outages.record_accepted("outage-disabled");
12478
12479        let restart = handler
12480            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
12481            .await
12482            .unwrap();
12483        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
12484        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
12485
12486        handler
12487            .handle_control_frame(
12488                &ctx,
12489                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
12490            )
12491            .await
12492            .unwrap();
12493        let starts = outage_lines(&capture, OUTAGE_START);
12494        assert_eq!(starts.len(), 1, "{starts:?}");
12495        assert_eq!(starts[0].level, tracing::Level::WARN);
12496    }
12497
12498    #[tokio::test(flavor = "current_thread")]
12499    async fn route_open_unknown_module_escapes_target_module_id() {
12500        let handler = ControlHandler::new(Arc::new(Registry::default()));
12501        let capture = EventCapture::default();
12502        let _subscriber =
12503            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12504        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
12505        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12506        let response = handler
12507            .handle_control_frame(
12508                &ctx,
12509                route_open_frame(
12510                    395,
12511                    hostile_module_id,
12512                    unique_project_root("hostile-target-module-id"),
12513                ),
12514            )
12515            .await
12516            .unwrap();
12517
12518        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12519        let event = capture
12520            .events()
12521            .into_iter()
12522            .find(|event| {
12523                event.target == "control"
12524                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12525            })
12526            .expect("route.open unknown-module refusal event");
12527        let logged = event.fields.get("module_id").expect("module_id field");
12528        assert!(!logged.bytes().any(|byte| byte < 0x20));
12529        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12530    }
12531
12532    #[tokio::test(flavor = "current_thread")]
12533    async fn route_open_module_rejection_uses_daemon_counter_key() {
12534        let registry = Arc::new(Registry::default());
12535        let forwarding = Arc::new(ForwardingTable::default());
12536        let handler =
12537            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12538        let module_connection = ConnectionId::new(95);
12539        let (module_ctx, mut module_rx) = route_ctx(module_connection);
12540        hello_via_sink(
12541            &handler,
12542            &module_ctx,
12543            &mut module_rx,
12544            hello_frame("aft", PROTOCOL_VERSION, 395),
12545        )
12546        .await;
12547
12548        let client_connection = ConnectionId::new(96);
12549        let (client_ctx, _client_rx) = route_ctx(client_connection);
12550        let capture = EventCapture::default();
12551        let _subscriber =
12552            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12553        let (route_task, bind) = relay_route_open(
12554            &handler,
12555            client_connection,
12556            &client_ctx.egress,
12557            &mut module_rx,
12558            396,
12559            "aft",
12560            "hostile-module-code",
12561        )
12562        .await;
12563        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
12564        let rejection = Frame::build(
12565            FrameType::Error,
12566            control_flags(),
12567            0,
12568            0,
12569            bind.header.corr,
12570            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
12571        )
12572        .unwrap();
12573        handler
12574            .handle_control_frame(&module_ctx, rejection)
12575            .await
12576            .unwrap();
12577
12578        let response = route_task.await.unwrap();
12579        assert_eq!(parse_error(&response[0])["code"], hostile_code);
12580        let counters = handler.counters().snapshot();
12581        assert_eq!(
12582            counters["route_open_refused_by_code"],
12583            json!({ "module_rejected": 1 })
12584        );
12585        assert!(counters["route_open_refused_by_code"]
12586            .get(hostile_code)
12587            .is_none());
12588
12589        let event = capture
12590            .events()
12591            .into_iter()
12592            .find(|event| {
12593                event.target == "control"
12594                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
12595            })
12596            .expect("route.open module-rejection refusal event");
12597        let logged = event.fields.get("module_code").expect("module_code field");
12598        assert!(!logged.bytes().any(|byte| byte < 0x20));
12599        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12600    }
12601
12602    #[tokio::test]
12603    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12604        let registry = Arc::new(Registry::default());
12605        let supervisor_handle = SupervisorHandle::new();
12606        let missing_program = std::env::temp_dir().join(format!(
12607            "subc-route-open-missing-program-{}",
12608            std::process::id()
12609        ));
12610        let supervisor =
12611            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12612                .with_handle(supervisor_handle.clone());
12613        let module = supervisor
12614            .supervise_configured(
12615                ModuleSpec {
12616                    module_id: "failed".to_string(),
12617                    program: missing_program,
12618                    args: Vec::new(),
12619                    env: Vec::new(),
12620                    reserved: false,
12621                    reserved_prefixes: Vec::new(),
12622                    protocol: ModuleProtocol::Subc,
12623                    overlap: Default::default(),
12624                },
12625                true,
12626            )
12627            .unwrap();
12628        assert_eq!(module.state().unwrap(), ModuleState::Failed);
12629
12630        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12631        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12632        let response = handler
12633            .handle_control_frame(
12634                &ctx,
12635                route_open_frame(305, "failed", unique_project_root("failed")),
12636            )
12637            .await
12638            .unwrap();
12639
12640        assert_eq!(response[0].header.ty, FrameType::Error);
12641        let error = parse_error(&response[0]);
12642        assert_eq!(error["code"], "target_unavailable");
12643        assert!(error["message"]
12644            .as_str()
12645            .unwrap()
12646            .contains("state=failed, enabled=true, live=false"));
12647    }
12648
12649    #[tokio::test]
12650    async fn route_open_role_mismatch_remains_target_unavailable() {
12651        let registry = Arc::new(Registry::default());
12652        let handler = ControlHandler::new(Arc::clone(&registry));
12653        handler
12654            .handle_control(
12655                ConnectionId::new(41),
12656                non_routable_hello_frame_with_control_ops("health-only", 306, None),
12657            )
12658            .unwrap();
12659
12660        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12661        let response = handler
12662            .handle_control_frame(
12663                &ctx,
12664                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12665            )
12666            .await
12667            .unwrap();
12668
12669        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12670        assert!(parse_error(&response[0])["message"]
12671            .as_str()
12672            .unwrap()
12673            .contains("does not provide the requested target"));
12674    }
12675
12676    #[tokio::test]
12677    async fn route_open_inactive_registration_remains_target_unavailable() {
12678        let registry = Arc::new(Registry::default());
12679        let handler = ControlHandler::new(Arc::clone(&registry));
12680        handler
12681            .handle_control(
12682                ConnectionId::new(43),
12683                hello_frame("inactive", PROTOCOL_VERSION, 308),
12684            )
12685            .unwrap();
12686        assert!(registry
12687            .set_module_state_for_test("inactive", ChannelState::Closed)
12688            .unwrap());
12689
12690        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12691        let response = handler
12692            .handle_control_frame(
12693                &ctx,
12694                route_open_frame(309, "inactive", unique_project_root("inactive")),
12695            )
12696            .await
12697            .unwrap();
12698
12699        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12700        assert!(parse_error(&response[0])["message"]
12701            .as_str()
12702            .unwrap()
12703            .contains("is not active"));
12704    }
12705
12706    #[tokio::test]
12707    async fn late_health_reply_is_recorded_through_the_module_response_path() {
12708        let registry = Arc::new(Registry::default());
12709        let forwarding = Arc::new(ForwardingTable::default());
12710        let supervisor_handle = SupervisorHandle::new();
12711        let supervisor =
12712            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
12713                .with_forwarding(Arc::clone(&forwarding))
12714                .with_handle(supervisor_handle.clone());
12715        let module = supervisor
12716            .supervise_configured(
12717                crate::ModuleSpec {
12718                    module_id: "late-health-response".to_string(),
12719                    program: PathBuf::from("disabled-module"),
12720                    args: Vec::new(),
12721                    env: Vec::new(),
12722                    reserved: false,
12723                    reserved_prefixes: Vec::new(),
12724                    protocol: ModuleProtocol::Subc,
12725                    overlap: Default::default(),
12726                },
12727                false,
12728            )
12729            .unwrap();
12730        let handler =
12731            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12732                .with_supervisor(supervisor_handle);
12733        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12734        handler
12735            .handle_control_frame(
12736                &module_ctx,
12737                hello_frame_with_control_ops(
12738                    "late-health-response",
12739                    PROTOCOL_VERSION,
12740                    7,
12741                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12742                ),
12743            )
12744            .await
12745            .unwrap();
12746        let probe_started_at = Instant::now() - Duration::from_millis(80);
12747        let pending = forwarding
12748            .begin_health_probe_rpc_for(
12749                "late-health-response",
12750                MODULE_CONTROL_OP_HEALTH_CHECK,
12751                probe_started_at,
12752                Instant::now() - Duration::from_millis(1),
12753            )
12754            .unwrap();
12755        assert!(forwarding
12756            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12757            .unwrap());
12758
12759        let responses = handler
12760            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12761            .await
12762            .unwrap();
12763
12764        assert!(responses.is_empty());
12765        let health = module.status().unwrap().health;
12766        assert_eq!(health.late_answer_count, 1);
12767        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12768    }
12769
12770    #[tokio::test]
12771    async fn health_probe_timeout_and_module_death_are_typed() {
12772        let registry = Arc::new(Registry::default());
12773        let forwarding = Arc::new(ForwardingTable::default());
12774        let handler =
12775            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12776                .with_health_probe_timeout(Duration::from_millis(50));
12777        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12778        hello_via_sink(
12779            &handler,
12780            &module_ctx,
12781            &mut module_rx,
12782            hello_frame_with_control_ops(
12783                "aft",
12784                PROTOCOL_VERSION,
12785                7,
12786                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12787            ),
12788        )
12789        .await;
12790
12791        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12792        let responses = handler
12793            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12794            .await
12795            .unwrap();
12796        assert_eq!(responses[0].header.ty, FrameType::Error);
12797        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12798        let _ = module_rx.try_recv();
12799
12800        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12801        let health_handler = handler.clone();
12802        let death_task = tokio::spawn(async move {
12803            health_handler
12804                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12805                .await
12806                .unwrap()
12807        });
12808        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12809            .await
12810            .unwrap()
12811            .unwrap();
12812        handler
12813            .cleanup_connection(module_ctx.connection_id)
12814            .unwrap();
12815        let responses = death_task.await.unwrap();
12816        assert_eq!(responses[0].header.ty, FrameType::Error);
12817        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12818    }
12819
12820    #[test]
12821    fn hello_requires_exact_protocol_version() {
12822        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12823            let registry = Arc::new(Registry::default());
12824            let handler = ControlHandler::new(Arc::clone(&registry));
12825            let responses = handler
12826                .handle_control(
12827                    ConnectionId::new(connection),
12828                    hello_frame("aft", offered, 9),
12829                )
12830                .unwrap();
12831
12832            assert_eq!(responses.len(), 1);
12833            assert_eq!(responses[0].header.ty, FrameType::Error);
12834            let error = parse_error(&responses[0]);
12835            assert_eq!(error["code"], "version_unsupported");
12836            assert!(registry.get_module("aft").unwrap().is_none());
12837            assert_eq!(registry.active_registration_count().unwrap(), 0);
12838        }
12839    }
12840
12841    #[test]
12842    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12843        let registry = Arc::new(Registry::default());
12844        let forwarding = Arc::new(ForwardingTable::default());
12845        let handler =
12846            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12847        let module_connection = ConnectionId::new(301);
12848        let registration = registry
12849            .register_with_control_ops(
12850                manifest("aft-push", PROTOCOL_VERSION),
12851                PROTOCOL_VERSION,
12852                module_connection,
12853                module_baseline_control_ops(),
12854            )
12855            .unwrap();
12856        let (module_tx, _module_rx) = mpsc::channel(8);
12857        let endpoint = forwarding
12858            .register_module_connection(
12859                module_connection,
12860                "aft-push".to_string(),
12861                PROTOCOL_VERSION,
12862                manifest_concurrency(&registration.manifest),
12863                FrameSink::new(module_tx),
12864            )
12865            .unwrap();
12866
12867        // A push op this version does not know is ignored (forward-compat), not errored.
12868        let unknown = Frame::build(
12869            FrameType::Push,
12870            control_flags(),
12871            0,
12872            0,
12873            5,
12874            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12875        )
12876        .unwrap();
12877        let out = handler.handle_status_update(endpoint, unknown).unwrap();
12878        assert!(
12879            out.is_empty(),
12880            "unknown push op must be ignored, got {out:?}"
12881        );
12882
12883        // A malformed body for a KNOWN op is a real error worth surfacing.
12884        let malformed = Frame::build(
12885            FrameType::Push,
12886            control_flags(),
12887            0,
12888            0,
12889            6,
12890            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
12891        )
12892        .unwrap();
12893        let out = handler.handle_status_update(endpoint, malformed).unwrap();
12894        assert_eq!(out.len(), 1);
12895        assert_eq!(out[0].header.ty, FrameType::Error);
12896        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
12897    }
12898
12899    #[test]
12900    fn hello_rejected_when_connection_already_owns_client_routes() {
12901        let registry = Arc::new(Registry::default());
12902        let forwarding = Arc::new(ForwardingTable::default());
12903        let handler =
12904            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12905        // Commits a client route on connection 202 (bound to a module on conn 101).
12906        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
12907        let client_connection = ConnectionId::new(202);
12908
12909        // That same connection now tries to register as a module: rejected, so one
12910        // connection never holds both client-route and module-endpoint state.
12911        let responses = handler
12912            .handle_control(
12913                client_connection,
12914                hello_frame("aft-second", PROTOCOL_VERSION, 9),
12915            )
12916            .unwrap();
12917        assert_eq!(responses[0].header.ty, FrameType::Error);
12918        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
12919        assert!(registry.get_module("aft-second").unwrap().is_none());
12920    }
12921
12922    #[tokio::test]
12923    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
12924        let registry = Arc::new(Registry::default());
12925        let forwarding = Arc::new(ForwardingTable::default());
12926        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
12927        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
12928        hello_via_sink(
12929            &handler,
12930            &module_ctx,
12931            &mut module_rx,
12932            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
12933        )
12934        .await;
12935        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
12936        let pending = forwarding
12937            .begin_route_bind_relay_for_test(
12938                client_ctx.connection_id,
12939                client_ctx.egress.clone(),
12940                2,
12941                "alpha",
12942            )
12943            .unwrap();
12944        forwarding
12945            .complete_pending_relay(
12946                module_ctx.connection_id,
12947                pending.corr,
12948                RouteBindRelayOutcome::Accepted,
12949            )
12950            .unwrap();
12951        client_rx.try_recv().unwrap();
12952        for module_id in ["beta", "alpha"] {
12953            let replies = handler
12954                .handle_control_frame(
12955                    &module_ctx,
12956                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
12957                )
12958                .await
12959                .unwrap();
12960            assert_eq!(replies.len(), 1, "second HELLO must be refused");
12961            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
12962        }
12963        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
12964        assert!(registry.get_module("beta").unwrap().is_none());
12965        assert!(matches!(
12966            forwarding
12967                .lookup_data_route(
12968                    client_ctx.connection_id,
12969                    pending.client_channel,
12970                    pending.client_epoch,
12971                )
12972                .unwrap(),
12973            DataRoute::Client(DataRouteState::Bound(_))
12974        ));
12975        assert!(handler
12976            .hello_launch_nonces
12977            .lock()
12978            .unwrap()
12979            .presented(module_ctx.connection_id, Some("alpha-nonce")));
12980        assert!(module_rx.try_recv().is_err());
12981    }
12982
12983    #[test]
12984    fn reserved_module_hello_requires_matching_launch_nonce() {
12985        let registry = Arc::new(Registry::default());
12986        let supervisor = SupervisorHandle::new();
12987        // The supervisor recorded the nonce it injected when it spawned the reserved
12988        // module; the HELLO verifier checks against the same shared handle.
12989        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12990        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
12991
12992        // A HELLO with NO nonce is rejected.
12993        let no_nonce = handler
12994            .handle_control(
12995                ConnectionId::new(1),
12996                hello_frame("vault", PROTOCOL_VERSION, 1),
12997            )
12998            .unwrap();
12999        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13000        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13001        assert!(registry.get_module("vault").unwrap().is_none());
13002
13003        // A HELLO with the WRONG nonce is rejected.
13004        let wrong = handler
13005            .handle_control(
13006                ConnectionId::new(2),
13007                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13008            )
13009            .unwrap();
13010        assert_eq!(wrong[0].header.ty, FrameType::Error);
13011        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13012        assert!(registry.get_module("vault").unwrap().is_none());
13013
13014        // A HELLO with the CORRECT nonce registers.
13015        let ok = handler
13016            .handle_control(
13017                ConnectionId::new(3),
13018                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13019            )
13020            .unwrap();
13021        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13022        assert!(registry.get_module("vault").unwrap().is_some());
13023    }
13024
13025    #[test]
13026    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13027        let registry = Arc::new(Registry::default());
13028        let supervisor = SupervisorHandle::new();
13029        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13030        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13031        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13032
13033        let squat = handler
13034            .handle_control(
13035                ConnectionId::new(1),
13036                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
13037            )
13038            .unwrap();
13039        assert_eq!(squat[0].header.ty, FrameType::Error);
13040        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
13041        assert!(parse_error(&squat[0])["message"]
13042            .as_str()
13043            .unwrap()
13044            .contains("fed:"));
13045
13046        let accepted_peer = handler
13047            .handle_control(
13048                ConnectionId::new(2),
13049                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
13050            )
13051            .unwrap();
13052        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
13053
13054        let accepted_short = handler
13055            .handle_control(
13056                ConnectionId::new(3),
13057                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
13058            )
13059            .unwrap();
13060        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
13061
13062        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
13063            let response = handler
13064                .handle_control(
13065                    ConnectionId::new(conn),
13066                    hello_frame(module_id, PROTOCOL_VERSION, conn),
13067                )
13068                .unwrap();
13069            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
13070        }
13071    }
13072
13073    #[test]
13074    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
13075        let registry = Arc::new(Registry::default());
13076        let supervisor = SupervisorHandle::new();
13077        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13078        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13079        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
13080        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
13081
13082        let owner_nonce = handler
13083            .handle_control(
13084                ConnectionId::new(1),
13085                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
13086            )
13087            .unwrap();
13088        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
13089        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
13090        assert!(registry.get_module("fed:special").unwrap().is_none());
13091
13092        let exact_nonce = handler
13093            .handle_control(
13094                ConnectionId::new(2),
13095                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
13096            )
13097            .unwrap();
13098        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
13099        assert!(registry.get_module("fed:special").unwrap().is_some());
13100    }
13101
13102    #[test]
13103    fn non_reserved_module_ignores_launch_nonce() {
13104        let registry = Arc::new(Registry::default());
13105        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
13106        // registration succeeds whether a spawned process echoes a nonce or not.
13107        let handler = ControlHandler::new(Arc::clone(&registry));
13108        let no_nonce = handler
13109            .handle_control(
13110                ConnectionId::new(1),
13111                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
13112            )
13113            .unwrap();
13114        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
13115        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
13116
13117        let echoed_nonce = handler
13118            .handle_control(
13119                ConnectionId::new(2),
13120                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
13121            )
13122            .unwrap();
13123        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
13124        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
13125    }
13126
13127    #[test]
13128    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
13129        let handler = ControlHandler::default();
13130        let conn = ConnectionId::new(1);
13131        let malformed = Frame::build(
13132            FrameType::Hello,
13133            control_flags(),
13134            0,
13135            0,
13136            3,
13137            b"{not json".to_vec(),
13138        )
13139        .unwrap();
13140
13141        let error = handler.handle_control(conn, malformed).unwrap();
13142        assert_eq!(error[0].header.ty, FrameType::Error);
13143        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
13144
13145        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13146        let pong = handler.handle_control(conn, ping).unwrap();
13147        assert_eq!(pong[0].header.ty, FrameType::Pong);
13148        assert_eq!(pong[0].header.corr, 4);
13149    }
13150
13151    #[test]
13152    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
13153        let registry = Arc::new(Registry::default());
13154        let handler = ControlHandler::new(Arc::clone(&registry));
13155
13156        handler
13157            .handle_control(
13158                ConnectionId::new(1),
13159                hello_frame("aft", PROTOCOL_VERSION, 1),
13160            )
13161            .unwrap();
13162        let duplicate = handler
13163            .handle_control(
13164                ConnectionId::new(2),
13165                hello_frame("aft", PROTOCOL_VERSION, 2),
13166            )
13167            .unwrap();
13168
13169        assert_eq!(duplicate[0].header.ty, FrameType::Error);
13170        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13171        let registration = registry.get_module("aft").unwrap().unwrap();
13172        assert_eq!(registration.connection_id, ConnectionId::new(1));
13173    }
13174
13175    #[test]
13176    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
13177        let registry = Arc::new(Registry::default());
13178        let forwarding = Arc::new(ForwardingTable::default());
13179        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
13180        let handler =
13181            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13182                .with_process_liveness(process_liveness);
13183        let (ctx, route_channel, route_epoch) =
13184            bind_liveness_route(&registry, &forwarding, "aft-dead");
13185        let responses = handler
13186            .handle_route_poll(
13187                &ctx,
13188                route_poll_frame(41, PollKind::Liveness, route_channel),
13189                route_channel,
13190                route_epoch,
13191                PollKind::Liveness,
13192            )
13193            .unwrap();
13194
13195        assert_eq!(responses.len(), 1);
13196        assert_eq!(responses[0].header.ty, FrameType::Response);
13197        assert_route_poll_liveness(&responses[0], false);
13198    }
13199
13200    #[test]
13201    fn liveness_poll_without_process_source_uses_bound_route() {
13202        let registry = Arc::new(Registry::default());
13203        let forwarding = Arc::new(ForwardingTable::default());
13204        let handler =
13205            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13206        let (ctx, route_channel, route_epoch) =
13207            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
13208        let responses = handler
13209            .handle_route_poll(
13210                &ctx,
13211                route_poll_frame(42, PollKind::Liveness, route_channel),
13212                route_channel,
13213                route_epoch,
13214                PollKind::Liveness,
13215            )
13216            .unwrap();
13217
13218        assert_route_poll_liveness(&responses[0], true);
13219    }
13220
13221    #[test]
13222    fn liveness_poll_untracked_process_source_uses_bound_route() {
13223        let registry = Arc::new(Registry::default());
13224        let forwarding = Arc::new(ForwardingTable::default());
13225        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
13226        let handler =
13227            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13228                .with_process_liveness(process_liveness);
13229        let (ctx, route_channel, route_epoch) =
13230            bind_liveness_route(&registry, &forwarding, "aft-untracked");
13231        let responses = handler
13232            .handle_route_poll(
13233                &ctx,
13234                route_poll_frame(43, PollKind::Liveness, route_channel),
13235                route_channel,
13236                route_epoch,
13237                PollKind::Liveness,
13238            )
13239            .unwrap();
13240
13241        assert_route_poll_liveness(&responses[0], true);
13242    }
13243
13244    #[tokio::test]
13245    async fn unknown_op_returns_unknown_control_op() {
13246        let handler = ControlHandler::default();
13247        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
13248        let request = Frame::build(
13249            FrameType::Request,
13250            control_flags(),
13251            0,
13252            0,
13253            55,
13254            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
13255        )
13256        .unwrap();
13257
13258        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13259
13260        assert_eq!(response.len(), 1);
13261        assert_eq!(response[0].header.ty, FrameType::Error);
13262        assert_eq!(response[0].header.corr, 55);
13263        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
13264    }
13265
13266    #[tokio::test]
13267    async fn supervisor_provenance_rejects_unknown_exact_module() {
13268        let handler = ControlHandler::default();
13269        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
13270        let request = Frame::build(
13271            FrameType::Request,
13272            control_flags(),
13273            0,
13274            0,
13275            57,
13276            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
13277        )
13278        .unwrap();
13279
13280        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13281
13282        assert_eq!(response.len(), 1);
13283        assert_eq!(response[0].header.ty, FrameType::Error);
13284        assert_eq!(response[0].header.corr, 57);
13285        let error = parse_error(&response[0]);
13286        assert_eq!(error["code"], "unknown_module");
13287        assert_eq!(error["message"], "module_id 'missing' is not supervised");
13288    }
13289
13290    #[test]
13291    fn provenance_probe_override_keeps_handler_tests_deterministic() {
13292        let expected = subc_control::RunningImageAgreement::Unavailable {
13293            reason: subc_control::RunningImageUnavailableReason::HashFailed,
13294        };
13295        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
13296        assert_eq!(handler.provenance_probe_override, Some(expected));
13297    }
13298
13299    #[test]
13300    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
13301        let verdict = reload_verdict(
13302            std::path::Path::new("/bin/new"),
13303            Some(std::path::Path::new("/bin/old")),
13304            subc_control::RunningImageAgreement::Unavailable {
13305                reason: subc_control::RunningImageUnavailableReason::HashFailed,
13306            },
13307        );
13308        assert!(matches!(
13309            verdict.path,
13310            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
13311                if configured == std::path::Path::new("/bin/new")
13312                    && spawned_from == std::path::Path::new("/bin/old")
13313        ));
13314    }
13315
13316    #[test]
13317    fn launch_spec_without_running_digest_is_unavailable_not_match() {
13318        use subc_control::{
13319            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
13320        };
13321        let configured = crate::supervise::ModuleSpec {
13322            module_id: "legacy".into(),
13323            program: "/module".into(),
13324            args: Vec::new(),
13325            env: Vec::new(),
13326            reserved: false,
13327            reserved_prefixes: Vec::new(),
13328            protocol: subc_control::ModuleProtocol::Subc,
13329            overlap: Default::default(),
13330        };
13331        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
13332        assert_eq!(
13333            launch_spec_agreement(&configured, false, Some(&digest)),
13334            Agreement::Unavailable {
13335                reason: Reason::NotRunning
13336            }
13337        );
13338        assert_eq!(
13339            launch_spec_agreement(&configured, true, None),
13340            Agreement::Unavailable {
13341                reason: Reason::SpawnedSpecUnavailable
13342            }
13343        );
13344        assert_eq!(
13345            launch_spec_agreement(&configured, true, Some(&digest)),
13346            Agreement::Match
13347        );
13348    }
13349
13350    #[cfg(windows)]
13351    #[test]
13352    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
13353        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
13354        let configured = dir.join("module.exe");
13355        std::fs::write(&configured, b"executable path fixture").unwrap();
13356        let spawned = std::fs::canonicalize(&configured).unwrap();
13357        assert_ne!(
13358            configured, spawned,
13359            "fixture must exercise different path spellings"
13360        );
13361        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
13362        let lower_case = configured.to_string_lossy().to_lowercase();
13363        for spelling in [
13364            &configured,
13365            Path::new(&forward_slashes),
13366            Path::new(&lower_case),
13367        ] {
13368            let verdict = reload_verdict(
13369                spelling,
13370                Some(&spawned),
13371                subc_control::RunningImageAgreement::Unavailable {
13372                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13373                },
13374            );
13375            assert_eq!(
13376                verdict.path,
13377                ReloadPathAgreement::Match,
13378                "configured {spelling:?}, spawned {spawned:?}"
13379            );
13380        }
13381    }
13382
13383    #[cfg(windows)]
13384    #[test]
13385    fn windows_reload_path_keeps_different_existing_program_pending() {
13386        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
13387        let old = dir.join("old.exe");
13388        let new = dir.join("new.exe");
13389        std::fs::write(&old, b"same executable content").unwrap();
13390        std::fs::copy(&old, &new).unwrap();
13391        let spawned = std::fs::canonicalize(&old).unwrap();
13392        let image = subc_control::RunningImageAgreement::Unavailable {
13393            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13394        };
13395        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
13396        assert!(
13397            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
13398            if configured == new && spawned_from == spawned)
13399        );
13400        let missing = dir.join("missing.exe");
13401        assert!(matches!(
13402            reload_verdict(&missing, Some(&spawned), image).path,
13403            ReloadPathAgreement::Mismatch { .. }
13404        ));
13405    }
13406
13407    #[test]
13408    fn reload_verdict_detects_replaced_image_at_same_path() {
13409        let image = subc_control::RunningImageAgreement::Mismatch {
13410            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13411                digest: "old".into(),
13412            },
13413            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13414                digest: "new".into(),
13415            },
13416        };
13417        let verdict = reload_verdict(
13418            std::path::Path::new("/bin/same"),
13419            Some(std::path::Path::new("/bin/same")),
13420            image.clone(),
13421        );
13422        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13423        assert_eq!(verdict.image, image);
13424    }
13425
13426    #[test]
13427    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13428        let image = subc_control::RunningImageAgreement::Unavailable {
13429            reason: subc_control::RunningImageUnavailableReason::NotRunning,
13430        };
13431        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13432        assert_eq!(
13433            verdict.path,
13434            subc_control::ReloadPathAgreement::Unavailable {
13435                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13436            }
13437        );
13438        assert_eq!(verdict.image, image);
13439
13440        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13441            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13442        };
13443        let verdict = reload_verdict(
13444            std::path::Path::new("/bin/same"),
13445            Some(std::path::Path::new("/bin/same")),
13446            unconfirmed.clone(),
13447        );
13448        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13449        assert_eq!(verdict.image, unconfirmed);
13450    }
13451
13452    #[test]
13453    fn reload_verdict_preserves_each_image_unavailability_reason() {
13454        use subc_control::RunningImageUnavailableReason as Reason;
13455
13456        for reason in [
13457            Reason::NotRunning,
13458            Reason::UnsupportedPlatform,
13459            Reason::RunningExecutableUnreadable,
13460            Reason::SpawnedPathUnreadable,
13461            Reason::HashFailed,
13462            Reason::ProcessIdentityUnconfirmed,
13463            Reason::Unknown("future_probe_reason".to_string()),
13464        ] {
13465            let image = subc_control::RunningImageAgreement::Unavailable {
13466                reason: reason.clone(),
13467            };
13468            let verdict = reload_verdict(
13469                std::path::Path::new("/bin/same"),
13470                Some(std::path::Path::new("/bin/same")),
13471                image.clone(),
13472            );
13473            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13474            assert_eq!(verdict.image, image, "{reason:?}");
13475        }
13476    }
13477
13478    #[tokio::test]
13479    async fn malformed_control_bodies_return_invalid_control_body() {
13480        let handler = ControlHandler::default();
13481        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
13482
13483        for (corr, body) in [
13484            (56, br#"{"route_channel":1}"#.as_slice()),
13485            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
13486            (
13487                58,
13488                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
13489            ),
13490        ] {
13491            let request = Frame::build(
13492                FrameType::Request,
13493                control_flags(),
13494                0,
13495                0,
13496                corr,
13497                body.to_vec(),
13498            )
13499            .unwrap();
13500            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13501
13502            assert_eq!(response.len(), 1);
13503            assert_eq!(response[0].header.ty, FrameType::Error);
13504            assert_eq!(response[0].header.corr, corr);
13505            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
13506        }
13507    }
13508
13509    #[tokio::test]
13510    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
13511        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13512        let registry = Arc::new(Registry::default());
13513        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
13514        let router = Router::with_control_handler(Arc::clone(&control));
13515        let connection = router.begin_connection();
13516        let (ctx, mut rx) = route_ctx(connection.id());
13517
13518        router
13519            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
13520            .await
13521            .unwrap();
13522        let response = rx.recv().await.unwrap();
13523        let ack = parse_ack(&response);
13524        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13525        let channel = 1;
13526
13527        let goodbye =
13528            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
13529        router.route_for_connection(&ctx, goodbye).await.unwrap();
13530        assert!(rx.try_recv().is_err());
13531        assert!(registry.get_module("aft").unwrap().is_none());
13532
13533        router
13534            .route_for_connection(&ctx, channel_request(channel, 13))
13535            .await
13536            .unwrap();
13537        let error_frame = rx.recv().await.unwrap();
13538        assert_eq!(error_frame.header.ty, FrameType::Error);
13539        assert_eq!(error_frame.header.channel, channel);
13540        let captured = crate::router::test_log::captured_logs(&logs);
13541        assert_eq!(
13542            captured
13543                .lines()
13544                .filter(|line| {
13545                    line.contains("module_id=aft")
13546                        && line.contains("reason=explicit_goodbye")
13547                        && line.contains("module registration ended")
13548                })
13549                .count(),
13550            1,
13551            "unexpected GOODBYE registry log: {captured}"
13552        );
13553    }
13554
13555    #[tokio::test]
13556    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
13557        let registry = Arc::new(Registry::default());
13558        let handler = ControlHandler::new(registry).with_capability_config(
13559            [("prov".to_string(), true), ("cons".to_string(), true)],
13560            BTreeMap::new(),
13561        );
13562        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
13563        register_capability_manifest(
13564            &handler,
13565            &provider_ctx,
13566            &mut provider_rx,
13567            capability_manifest("prov", &["thing/v1"], &[]),
13568            1,
13569        )
13570        .await;
13571        let mut consumer = capability_manifest("cons", &[], &[]);
13572        consumer.capabilities.as_mut().unwrap().requires.push(
13573            subc_protocol::manifest::CapabilityRequirement {
13574                capability: "thing/v1".to_string(),
13575                need: subc_protocol::manifest::CapabilityNeed::Required,
13576            },
13577        );
13578        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
13579        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
13580        assert_eq!(
13581            handler.capability_evaluator.verdict("cons", "thing/v1"),
13582            Some(CapabilityVerdict::Provided)
13583        );
13584        let (mut client_rx, _) = open_route_for_capability_test(
13585            &handler,
13586            &provider_ctx,
13587            &mut provider_rx,
13588            703,
13589            3,
13590            "prov",
13591            None,
13592        )
13593        .await;
13594        let goodbye =
13595            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13596        handler
13597            .handle_control_frame(&provider_ctx, goodbye)
13598            .await
13599            .unwrap();
13600        assert_eq!(
13601            handler.capability_evaluator.verdict("cons", "thing/v1"),
13602            Some(CapabilityVerdict::NeverProvided)
13603        );
13604        let closed = client_rx
13605            .try_recv()
13606            .expect("GOODBYE pushes route.closed before route GOODBYE");
13607        assert!(
13608            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
13609            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
13610        );
13611        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
13612        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
13613    }
13614
13615    struct RouteClosedFixture {
13616        handler: ControlHandler,
13617        #[cfg(unix)]
13618        supervisor: Supervisor,
13619        module_ctx: RouteCtx,
13620        #[cfg(unix)]
13621        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13622        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13623        route: Arc<crate::forwarding::RouteBinding>,
13624    }
13625
13626    async fn route_closed_fixture() -> RouteClosedFixture {
13627        let registry = Arc::new(Registry::default());
13628        let handler = ControlHandler::new(Arc::clone(&registry));
13629        #[cfg(unix)]
13630        let supervisor = Supervisor::new(registry, RestartPolicy::default())
13631            .with_forwarding(Arc::clone(&handler.forwarding));
13632        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13633        register_capability_manifest(
13634            &handler,
13635            &module_ctx,
13636            &mut module_rx,
13637            capability_manifest("prov", &[], &[]),
13638            1,
13639        )
13640        .await;
13641        let (client_rx, bind) = open_route_for_capability_test(
13642            &handler,
13643            &module_ctx,
13644            &mut module_rx,
13645            702,
13646            2,
13647            "prov",
13648            None,
13649        )
13650        .await;
13651        let ModuleControlRequest::RouteBind {
13652            route_channel: channel,
13653            epoch: route_epoch,
13654            ..
13655        } = bind
13656        else {
13657            panic!("expected route.bind");
13658        };
13659        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13660            handler
13661                .forwarding
13662                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13663                .unwrap()
13664        else {
13665            panic!("the acknowledged route must be bound");
13666        };
13667        RouteClosedFixture {
13668            handler,
13669            #[cfg(unix)]
13670            supervisor,
13671            module_ctx,
13672            #[cfg(unix)]
13673            module_rx,
13674            client_rx,
13675            route,
13676        }
13677    }
13678
13679    impl RouteClosedFixture {
13680        #[cfg(unix)]
13681        async fn cut(&mut self) {
13682            let notices = async {
13683                let mut draining = self.module_rx.recv().await.unwrap();
13684                assert!(matches!(
13685                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13686                        &draining.body
13687                    )
13688                    .unwrap(),
13689                    subc_protocol::session::ModuleControlCommand::Draining {
13690                        reason: RouteCloseReason::Restart,
13691                        ..
13692                    }
13693                ));
13694                draining.flushed.take().unwrap().send(()).unwrap();
13695                let mut closing = self.client_rx.recv().await.unwrap();
13696                assert!(matches!(
13697                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13698                    ClientControlPush::RouteClosing {
13699                        reason: RouteCloseReason::Restart,
13700                        ..
13701                    }
13702                ));
13703                closing.flushed.take().unwrap().send(()).unwrap();
13704            };
13705            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13706            result.unwrap();
13707        }
13708
13709        fn disconnect(&mut self) -> ClientControlPush {
13710            self.handler
13711                .cleanup_connection(self.module_ctx.connection_id)
13712                .unwrap();
13713            let closed = self.client_rx.try_recv().unwrap();
13714            assert_eq!(closed.header.ty, FrameType::Push);
13715            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13716            assert!(matches!(
13717                &push,
13718                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13719                    if channels == &[self.route.client_channel]
13720            ));
13721            assert_eq!(
13722                self.client_rx.try_recv().unwrap().header.ty,
13723                FrameType::Goodbye
13724            );
13725            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13726            push
13727        }
13728    }
13729
13730    #[cfg(unix)]
13731    #[tokio::test(start_paused = true)]
13732    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13733        let mut fixture = route_closed_fixture().await;
13734        fixture.cut().await;
13735        assert!(matches!(
13736            fixture.disconnect(),
13737            ClientControlPush::RouteClosed {
13738                reason: RouteCloseReason::Restart,
13739                drained: true,
13740                ..
13741            }
13742        ));
13743    }
13744
13745    #[cfg(unix)]
13746    #[tokio::test(start_paused = true)]
13747    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13748        let mut fixture = route_closed_fixture().await;
13749        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13750        let started = tokio::time::Instant::now();
13751        fixture.cut().await;
13752        assert!(started.elapsed() >= Duration::from_secs(2));
13753        assert_eq!(fixture.route.flow.in_flight(), 1);
13754        assert!(matches!(
13755            fixture.disconnect(),
13756            ClientControlPush::RouteClosed {
13757                reason: RouteCloseReason::Restart,
13758                drained: false,
13759                ..
13760            }
13761        ));
13762    }
13763
13764    #[tokio::test]
13765    async fn module_connection_crash_reports_crash_and_not_drained() {
13766        let mut fixture = route_closed_fixture().await;
13767        // The server calls cleanup_connection on module EOF. Without a planned
13768        // daemon shutdown, that disconnect must remain a crash with no prior
13769        // route.closing notice, even when the route has no in-flight calls.
13770        assert!(matches!(
13771            fixture.disconnect(),
13772            ClientControlPush::RouteClosed {
13773                reason: RouteCloseReason::Crash,
13774                drained: false,
13775                ..
13776            }
13777        ));
13778    }
13779
13780    #[tokio::test]
13781    async fn dropping_router_connection_releases_registration() {
13782        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13783        let registry = Arc::new(Registry::default());
13784        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
13785        let router = Router::with_control_handler(Arc::clone(&control));
13786        let connection = router.begin_connection();
13787        let connection_id = connection.id();
13788        let (ctx, mut rx) = route_ctx(connection_id);
13789
13790        router
13791            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13792            .await
13793            .unwrap();
13794        let response = rx.recv().await.unwrap();
13795        let ack = parse_ack(&response);
13796        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13797        assert!(registry.get_module("aft").unwrap().is_some());
13798
13799        drop(connection);
13800
13801        assert!(registry.get_module("aft").unwrap().is_none());
13802        assert_eq!(registry.active_registration_count().unwrap(), 0);
13803
13804        control
13805            .cleanup_connection(ConnectionId::new(u64::MAX))
13806            .unwrap();
13807        let captured = crate::router::test_log::captured_logs(&logs);
13808        println!("captured registry lifecycle logs:\n{captured}");
13809        let events: Vec<_> = captured
13810            .lines()
13811            .filter(|line| {
13812                line.contains("module registered module_id=aft ")
13813                    || line.contains("module registration ended")
13814            })
13815            .collect();
13816        assert_eq!(
13817            events.len(),
13818            2,
13819            "unexpected registry lifecycle logs: {captured}"
13820        );
13821        assert!(events[0].contains("module registered module_id=aft "));
13822        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13823        assert!(events[1].contains(&format!(
13824            "module_id=aft connection_id={}",
13825            connection_id.get()
13826        )));
13827        assert!(events[1].contains("reason=connection_closed"));
13828        assert!(events[1].contains("module registration ended"));
13829        assert_eq!(
13830            captured
13831                .lines()
13832                .filter(|line| line.contains("module registered module_id=aft "))
13833                .count(),
13834            1,
13835            "legacy registration admission line must appear once: {captured}"
13836        );
13837    }
13838
13839    #[tokio::test]
13840    async fn hello_registration_keeps_legacy_module_registered_line_once() {
13841        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13842        let registry = Arc::new(Registry::default());
13843        let control = ControlHandler::new(Arc::clone(&registry));
13844        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13845        hello_via_sink(
13846            &control,
13847            &ctx,
13848            &mut rx,
13849            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13850        )
13851        .await;
13852
13853        let captured = crate::router::test_log::captured_logs(&logs);
13854        let admissions: Vec<_> = captured
13855            .lines()
13856            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13857            .collect();
13858        assert_eq!(
13859            admissions.len(),
13860            1,
13861            "legacy admission line must remain exactly once: {captured}"
13862        );
13863        assert!(admissions[0].contains("routable_provider=true"));
13864        assert!(admissions[0].contains("connection_id=777"));
13865    }
13866
13867    fn capability_manifest(
13868        module_id: &str,
13869        provides: &[&str],
13870        must_never_reach: &[&str],
13871    ) -> ModuleManifest {
13872        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13873        manifest.capabilities = Some(CapabilityDeclarations {
13874            provides: provides
13875                .iter()
13876                .map(|capability| (*capability).to_string())
13877                .collect(),
13878            requires: Vec::new(),
13879            must_never_reach: must_never_reach
13880                .iter()
13881                .map(|capability| (*capability).to_string())
13882                .collect(),
13883        });
13884        manifest
13885    }
13886
13887    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
13888        Frame::build(
13889            FrameType::Hello,
13890            control_flags(),
13891            0,
13892            0,
13893            corr,
13894            serde_json::to_vec(&ModuleHelloBody {
13895                protocol_ver: manifest.protocol_ver,
13896                manifest,
13897                control_ops: None,
13898                launch_nonce: None,
13899            })
13900            .expect("capability test HELLO serializes"),
13901        )
13902        .expect("capability test HELLO frame builds")
13903    }
13904
13905    fn catalog_update_with_capabilities_frame(
13906        corr: u64,
13907        capabilities: CapabilityDeclarations,
13908    ) -> Frame {
13909        Frame::build(
13910            FrameType::Request,
13911            control_flags(),
13912            0,
13913            0,
13914            corr,
13915            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13916                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
13917                capabilities: Some(capabilities),
13918                ready: None,
13919            })
13920            .expect("capability catalog.update serializes"),
13921        )
13922        .expect("capability catalog.update frame builds")
13923    }
13924
13925    async fn register_capability_manifest(
13926        handler: &ControlHandler,
13927        ctx: &RouteCtx,
13928        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13929        manifest: ModuleManifest,
13930        corr: u64,
13931    ) {
13932        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
13933    }
13934
13935    async fn open_route_for_capability_test(
13936        handler: &ControlHandler,
13937        target_ctx: &RouteCtx,
13938        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13939        client_connection_id: u64,
13940        corr: u64,
13941        target_module_id: &str,
13942        consumer_identity: Option<ConsumerIdentity>,
13943    ) -> (
13944        mpsc::Receiver<crate::router::OutboundFrame>,
13945        ModuleControlRequest,
13946    ) {
13947        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
13948        let route_handler = handler.clone();
13949        let target_module_id = target_module_id.to_string();
13950        let route_task = tokio::spawn(async move {
13951            route_handler
13952                .handle_control_frame(
13953                    &client_ctx,
13954                    route_open_frame_with_admission_facts(
13955                        corr,
13956                        &target_module_id,
13957                        unique_project_root("admission-facts"),
13958                        consumer_identity,
13959                        None,
13960                    ),
13961                )
13962                .await
13963                .expect("capability test route.open succeeds")
13964        });
13965        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
13966            .await
13967            .expect("capability test route.open must reach route.bind")
13968            .expect("target control receiver stays open");
13969        let bind_request: ModuleControlRequest =
13970            serde_json::from_slice(&bind.body).expect("route.bind decodes");
13971        handler
13972            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
13973            .await
13974            .expect("capability test route.bind ACK succeeds");
13975        assert!(route_task.await.expect("route.open task joins").is_empty());
13976        let opened = client_rx
13977            .recv()
13978            .await
13979            .expect("successful route.open publishes a response");
13980        assert!(matches!(
13981            serde_json::from_slice::<ClientControlResponse>(&opened.body),
13982            Ok(ClientControlResponse::RouteOpen { .. })
13983        ));
13984        (client_rx, bind_request)
13985    }
13986
13987    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
13988        assert_eq!(frame.header.ty, FrameType::Push);
13989        assert_eq!(frame.header.channel, 0);
13990        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
13991            .expect("route.closed control push decodes");
13992        let ClientControlPush::RouteClosed { channels, .. } = &push else {
13993            panic!("expected route.closed");
13994        };
13995        assert_eq!(channels.len(), 1, "exactly one violating route closed");
13996        let channels = channels.clone();
13997        assert_eq!(
13998            push,
13999            ClientControlPush::RouteClosed {
14000                module_id: target_module_id.to_string(),
14001                channels,
14002                reason: RouteCloseReason::CapabilityDenied,
14003                drained: false,
14004                abandoned: 0,
14005                excluded_subscriptions: 0,
14006                terminal: Some(false),
14007            }
14008        );
14009    }
14010
14011    #[tokio::test]
14012    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14013        let registry = Arc::new(Registry::default());
14014        let forwarding = Arc::new(ForwardingTable::default());
14015        let supervisor = SupervisorHandle::new();
14016        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14017        let handler =
14018            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14019                .with_supervisor(supervisor);
14020        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14021        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14022        register_capability_manifest(
14023            &handler,
14024            &target_ctx,
14025            &mut target_rx,
14026            capability_manifest("target", &["credentials-provider/v1"], &[]),
14027            1,
14028        )
14029        .await;
14030        register_capability_manifest(
14031            &handler,
14032            &opener_ctx,
14033            &mut opener_rx,
14034            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14035            2,
14036        )
14037        .await;
14038
14039        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
14040        let replies = handler
14041            .handle_control_frame(
14042                &client_ctx,
14043                route_open_frame_with_admission_facts(
14044                    3,
14045                    "target",
14046                    unique_project_root("admission-facts"),
14047                    Some(ConsumerIdentity {
14048                        module_id: "opener".to_string(),
14049                        launch_nonce: "opener-nonce".to_string(),
14050                    }),
14051                    None,
14052                ),
14053            )
14054            .await
14055            .expect("denied route.open returns a typed frame");
14056        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14057        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14058        assert!(
14059            target_rx.try_recv().is_err(),
14060            "forbidden route.open must not relay route.bind"
14061        );
14062    }
14063
14064    #[tokio::test]
14065    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
14066        let registry = Arc::new(Registry::default());
14067        let forwarding = Arc::new(ForwardingTable::default());
14068        let supervisor = SupervisorHandle::new();
14069        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14070        let handler =
14071            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14072                .with_supervisor(supervisor);
14073        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
14074        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
14075        register_capability_manifest(
14076            &handler,
14077            &target_ctx,
14078            &mut target_rx,
14079            capability_manifest("target", &["credentials-provider/v1"], &[]),
14080            1,
14081        )
14082        .await;
14083        register_capability_manifest(
14084            &handler,
14085            &old_opener_ctx,
14086            &mut old_opener_rx,
14087            capability_manifest("opener", &[], &[]),
14088            2,
14089        )
14090        .await;
14091        let (mut client_rx, _) = open_route_for_capability_test(
14092            &handler,
14093            &target_ctx,
14094            &mut target_rx,
14095            712,
14096            3,
14097            "target",
14098            Some(ConsumerIdentity {
14099                module_id: "opener".to_string(),
14100                launch_nonce: "opener-nonce".to_string(),
14101            }),
14102        )
14103        .await;
14104        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14105
14106        handler
14107            .cleanup_connection(old_opener_ctx.connection_id)
14108            .expect("old opener registration cleans up");
14109        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
14110        register_capability_manifest(
14111            &handler,
14112            &new_opener_ctx,
14113            &mut new_opener_rx,
14114            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14115            4,
14116        )
14117        .await;
14118
14119        assert_capability_denied_push(
14120            client_rx
14121                .try_recv()
14122                .expect("HELLO deny addition must emit route.closed")
14123                .frame,
14124            "target",
14125        );
14126        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14127        assert!(matches!(
14128            target_rx.try_recv(),
14129            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14130        ));
14131    }
14132
14133    #[tokio::test]
14134    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
14135        let registry = Arc::new(Registry::default());
14136        let forwarding = Arc::new(ForwardingTable::default());
14137        let supervisor = SupervisorHandle::new();
14138        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14139        let handler =
14140            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14141                .with_supervisor(supervisor);
14142        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
14143        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
14144        register_capability_manifest(
14145            &handler,
14146            &target_ctx,
14147            &mut target_rx,
14148            capability_manifest("target", &[], &[]),
14149            1,
14150        )
14151        .await;
14152        register_capability_manifest(
14153            &handler,
14154            &opener_ctx,
14155            &mut opener_rx,
14156            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14157            2,
14158        )
14159        .await;
14160        let (mut client_rx, _) = open_route_for_capability_test(
14161            &handler,
14162            &target_ctx,
14163            &mut target_rx,
14164            722,
14165            3,
14166            "target",
14167            Some(ConsumerIdentity {
14168                module_id: "opener".to_string(),
14169                launch_nonce: "opener-nonce".to_string(),
14170            }),
14171        )
14172        .await;
14173        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14174
14175        let replies = handler
14176            .handle_control_frame(
14177                &target_ctx,
14178                catalog_update_with_capabilities_frame(
14179                    4,
14180                    CapabilityDeclarations {
14181                        provides: vec!["credentials-provider/v1".to_string()],
14182                        requires: Vec::new(),
14183                        must_never_reach: Vec::new(),
14184                    },
14185                ),
14186            )
14187            .await
14188            .expect("claim catalog.update succeeds");
14189        assert!(matches!(
14190            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
14191            Ok(ModuleControlResponseToModule::CatalogUpdate {})
14192        ));
14193        assert_capability_denied_push(
14194            client_rx
14195                .try_recv()
14196                .expect("claim addition must emit route.closed")
14197                .frame,
14198            "target",
14199        );
14200        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14201        assert!(matches!(
14202            target_rx.try_recv(),
14203            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14204        ));
14205    }
14206
14207    #[tokio::test]
14208    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
14209        let registry = Arc::new(Registry::default());
14210        let forwarding = Arc::new(ForwardingTable::default());
14211        let supervisor = SupervisorHandle::new();
14212        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14213        let handler =
14214            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14215                .with_supervisor(supervisor);
14216        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
14217        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
14218        register_capability_manifest(
14219            &handler,
14220            &target_ctx,
14221            &mut target_rx,
14222            capability_manifest("target", &["credentials-provider/v1"], &[]),
14223            1,
14224        )
14225        .await;
14226        register_capability_manifest(
14227            &handler,
14228            &opener_ctx,
14229            &mut opener_rx,
14230            capability_manifest("opener", &[], &[]),
14231            2,
14232        )
14233        .await;
14234        let (mut client_rx, _) = open_route_for_capability_test(
14235            &handler,
14236            &target_ctx,
14237            &mut target_rx,
14238            732,
14239            3,
14240            "target",
14241            Some(ConsumerIdentity {
14242                module_id: "opener".to_string(),
14243                launch_nonce: "opener-nonce".to_string(),
14244            }),
14245        )
14246        .await;
14247        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14248
14249        handler
14250            .handle_control_frame(
14251                &target_ctx,
14252                catalog_update_with_capabilities_frame(
14253                    4,
14254                    CapabilityDeclarations {
14255                        provides: Vec::new(),
14256                        requires: Vec::new(),
14257                        must_never_reach: Vec::new(),
14258                    },
14259                ),
14260            )
14261            .await
14262            .expect("claim removal catalog.update succeeds");
14263        assert_eq!(
14264            forwarding.active_binding_count().unwrap(),
14265            1,
14266            "removing an attested target claim must leave the route census unchanged"
14267        );
14268        assert!(
14269            client_rx.try_recv().is_err(),
14270            "claim removal must not emit route.closed capability_denied"
14271        );
14272        assert!(
14273            target_rx.try_recv().is_err(),
14274            "claim removal must not send the target a route GOODBYE"
14275        );
14276    }
14277
14278    /// A direct client may open a route to a denied capability provider; this
14279    /// policy applies only to attested supervised module origins, not to direct clients.
14280    #[tokio::test]
14281    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
14282        let registry = Arc::new(Registry::default());
14283        let forwarding = Arc::new(ForwardingTable::default());
14284        let supervisor = SupervisorHandle::new();
14285        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14286        let handler =
14287            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14288                .with_supervisor(supervisor);
14289        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
14290        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
14291        register_capability_manifest(
14292            &handler,
14293            &target_ctx,
14294            &mut target_rx,
14295            capability_manifest("target", &["credentials-provider/v1"], &[]),
14296            1,
14297        )
14298        .await;
14299        register_capability_manifest(
14300            &handler,
14301            &opener_ctx,
14302            &mut opener_rx,
14303            capability_manifest("opener", &[], &["credentials-provider/v1"]),
14304            2,
14305        )
14306        .await;
14307
14308        let (_client_rx, bind) = open_route_for_capability_test(
14309            &handler,
14310            &target_ctx,
14311            &mut target_rx,
14312            742,
14313            3,
14314            "target",
14315            None,
14316        )
14317        .await;
14318        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14319            panic!("direct scope-honesty route must bind");
14320        };
14321        assert_eq!(principal, Some(Principal::Direct));
14322        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14323    }
14324
14325    /// A module that denies a capability receives no self-route exemption when it
14326    /// also attestedly provides that capability.
14327    #[tokio::test]
14328    async fn must_never_reach_self_route_is_capability_forbidden() {
14329        let registry = Arc::new(Registry::default());
14330        let forwarding = Arc::new(ForwardingTable::default());
14331        let supervisor = SupervisorHandle::new();
14332        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
14333        let handler =
14334            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14335                .with_supervisor(supervisor);
14336        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
14337        register_capability_manifest(
14338            &handler,
14339            &self_ctx,
14340            &mut self_rx,
14341            capability_manifest(
14342                "self-provider",
14343                &["credentials-provider/v1"],
14344                &["credentials-provider/v1"],
14345            ),
14346            1,
14347        )
14348        .await;
14349
14350        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
14351        let replies = handler
14352            .handle_control_frame(
14353                &client_ctx,
14354                route_open_frame_with_admission_facts(
14355                    2,
14356                    "self-provider",
14357                    unique_project_root("admission-facts"),
14358                    Some(ConsumerIdentity {
14359                        module_id: "self-provider".to_string(),
14360                        launch_nonce: "self-nonce".to_string(),
14361                    }),
14362                    None,
14363                ),
14364            )
14365            .await
14366            .expect("self-route refusal returns a typed frame");
14367        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14368        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14369        assert!(
14370            self_rx.try_recv().is_err(),
14371            "self denial must not relay route.bind"
14372        );
14373    }
14374
14375    #[test]
14376    fn unsupported_channel_zero_frame_returns_error() {
14377        let handler = ControlHandler::default();
14378        let request = Frame::build(
14379            FrameType::Request,
14380            control_flags(),
14381            0,
14382            0,
14383            21,
14384            b"opaque".to_vec(),
14385        )
14386        .unwrap();
14387
14388        let response = handler
14389            .handle_control(ConnectionId::new(1), request)
14390            .unwrap();
14391
14392        assert_eq!(response[0].header.ty, FrameType::Error);
14393        assert_eq!(
14394            parse_error(&response[0])["code"],
14395            "unsupported_control_frame"
14396        );
14397    }
14398
14399    /// Blue/green swap at the control-plane boundary. The supervisor that opens
14400    /// a swap is not wired yet, so the candidate is registered here directly
14401    /// into the registry and forwarding candidate slots, the way the swap's
14402    /// HELLO admission will.
14403    mod swap {
14404        use super::*;
14405
14406        const INCUMBENT: ConnectionId = ConnectionId::new(30);
14407        const CANDIDATE: ConnectionId = ConnectionId::new(40);
14408
14409        struct Swap {
14410            registry: Arc<Registry>,
14411            forwarding: Arc<ForwardingTable>,
14412            handler: ControlHandler,
14413            incumbent_ctx: RouteCtx,
14414            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14415            candidate_ctx: RouteCtx,
14416            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14417        }
14418
14419        async fn swap_with_incumbent() -> Swap {
14420            let registry = Arc::new(Registry::default());
14421            let forwarding = Arc::new(ForwardingTable::default());
14422            let handler =
14423                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14424            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14425            hello_via_sink(
14426                &handler,
14427                &incumbent_ctx,
14428                &mut incumbent_rx,
14429                hello_frame("aft", PROTOCOL_VERSION, 7),
14430            )
14431            .await;
14432            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14433            Swap {
14434                registry,
14435                forwarding,
14436                handler,
14437                incumbent_ctx,
14438                incumbent_rx,
14439                candidate_ctx,
14440                candidate_rx,
14441            }
14442        }
14443
14444        fn register_candidate(swap: &Swap, ready: Option<bool>) {
14445            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14446            candidate_manifest.ready = ready;
14447            let registration = swap
14448                .registry
14449                .register_candidate_with_control_ops(
14450                    candidate_manifest,
14451                    PROTOCOL_VERSION,
14452                    CANDIDATE,
14453                    module_baseline_control_ops(),
14454                )
14455                .unwrap();
14456            swap.forwarding
14457                .register_candidate_module_connection(
14458                    CANDIDATE,
14459                    "aft".to_string(),
14460                    PROTOCOL_VERSION,
14461                    manifest_concurrency(&registration.manifest),
14462                    swap.candidate_ctx.egress.clone(),
14463                )
14464                .unwrap();
14465        }
14466
14467        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
14468            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
14469            swap.registry.promote_candidate("aft").unwrap().unwrap();
14470            cutover.incumbent.unwrap()
14471        }
14472
14473        /// A swap candidate's HELLO reserves its candidate registry entry
14474        /// before installing its candidate forwarding endpoint. The swap
14475        /// supervisor promotes a ready candidate as soon as it reads it as
14476        /// registered, and promotion cuts forwarding over to the candidate
14477        /// endpoint, so the lookup the supervisor reads
14478        /// (`Registry::published_registration`) must not return the candidate
14479        /// until that endpoint exists. Otherwise the cutover finds no candidate
14480        /// endpoint and the swap fails as if the candidate had disconnected.
14481        #[tokio::test]
14482        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
14483            let mut swap = swap_with_incumbent().await;
14484            let hold = Arc::new(std::sync::Barrier::new(2));
14485            let handler = swap
14486                .handler
14487                .clone()
14488                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14489            handler
14490                .supervisor
14491                .open_swap("aft", "candidate-nonce".to_string());
14492            let hello = {
14493                let handler = handler.clone();
14494                let candidate_ctx = swap.candidate_ctx.clone();
14495                std::thread::spawn(move || {
14496                    handler
14497                        .handle_hello(
14498                            candidate_ctx.connection_id,
14499                            Some(candidate_ctx.egress.clone()),
14500                            hello_frame_with_nonce(
14501                                "aft",
14502                                PROTOCOL_VERSION,
14503                                8,
14504                                Some("candidate-nonce"),
14505                            ),
14506                        )
14507                        .unwrap()
14508                })
14509            };
14510
14511            hold.wait();
14512            // Parked: the candidate slot is claimed but not published, and
14513            // forwarding has no candidate endpoint.
14514            assert!(swap
14515                .registry
14516                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
14517                .unwrap()
14518                .is_some());
14519            assert!(swap
14520                .registry
14521                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14522                .unwrap()
14523                .is_none());
14524            assert!(swap
14525                .forwarding
14526                .module_endpoint_for_connection(CANDIDATE)
14527                .unwrap()
14528                .is_none());
14529
14530            hold.wait();
14531            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
14532            assert_eq!(
14533                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
14534                FrameType::HelloAck
14535            );
14536            let published = swap
14537                .registry
14538                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14539                .unwrap()
14540                .expect("candidate is published once its HELLO completes");
14541            assert_eq!(published.connection_id, CANDIDATE);
14542            assert!(swap
14543                .forwarding
14544                .module_endpoint_for_connection(CANDIDATE)
14545                .unwrap()
14546                .is_some());
14547        }
14548
14549        fn keyed_total(counters: &Value, key: &str) -> u64 {
14550            counters[key]
14551                .as_object()
14552                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
14553                .unwrap_or(0)
14554        }
14555
14556        #[tokio::test]
14557        async fn replacement_logs_old_end_and_new_admission_once() {
14558            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14559            let mut swap = swap_with_incumbent().await;
14560            swap.handler
14561                .supervisor
14562                .open_swap("aft", "candidate-nonce".to_string());
14563            hello_via_sink(
14564                &swap.handler,
14565                &swap.candidate_ctx,
14566                &mut swap.candidate_rx,
14567                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
14568            )
14569            .await;
14570            cutover(&swap);
14571            swap.handler.cleanup_connection(INCUMBENT).unwrap();
14572
14573            let captured = crate::router::test_log::captured_logs(&logs);
14574            println!("captured replacement registry logs:\n{captured}");
14575            let new_admissions: Vec<_> = captured
14576                .lines()
14577                .filter(|line| {
14578                    line.contains("connection_id=40")
14579                        && line.contains("swap candidate registered; not routable until cutover")
14580                })
14581                .collect();
14582            assert_eq!(
14583                new_admissions.len(),
14584                1,
14585                "unexpected admission logs: {captured}"
14586            );
14587            assert!(new_admissions[0].contains("module_id=aft"));
14588            assert!(new_admissions[0].contains("connection_id=40"));
14589            assert!(
14590                new_admissions[0].contains("swap candidate registered; not routable until cutover")
14591            );
14592            assert!(new_admissions[0].contains("ready=true"));
14593
14594            let old_admissions: Vec<_> = captured
14595                .lines()
14596                .filter(|line| {
14597                    line.contains("module registered module_id=aft ")
14598                        && line.contains("connection_id=30")
14599                })
14600                .collect();
14601            assert_eq!(
14602                old_admissions.len(),
14603                1,
14604                "unexpected incumbent admission: {captured}"
14605            );
14606
14607            let old_ends: Vec<_> = captured
14608                .lines()
14609                .filter(|line| {
14610                    line.contains("connection_id=30") && line.contains("module registration ended")
14611                })
14612                .collect();
14613            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
14614            assert!(old_ends[0].contains("module_id=aft"));
14615            assert!(old_ends[0].contains("connection_id=30"));
14616            assert!(old_ends[0].contains("reason=replaced"));
14617            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
14618            assert_eq!(
14619                captured
14620                    .lines()
14621                    .filter(|line| line.contains("module registration promoted"))
14622                    .count(),
14623                1,
14624                "unexpected promotion logs: {captured}"
14625            );
14626        }
14627
14628        /// An ack from the incumbent for a bind it was sent before cutover,
14629        /// arriving before the incumbent is drained. The incumbent is the live
14630        /// connection carrying every other client's routes, so the ack must
14631        /// not end it: the waiting client is told to retry, the reservation is
14632        /// given back, and the incumbent is told to drop just that binding.
14633        #[tokio::test]
14634        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14635            let mut swap = swap_with_incumbent().await;
14636            let handler = swap.handler.clone();
14637
14638            // A co-tenant route, bound on the incumbent before the swap.
14639            let cotenant = ConnectionId::new(31);
14640            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14641            let (cotenant_task, cotenant_bind) = relay_route_open(
14642                &handler,
14643                cotenant,
14644                &cotenant_ctx.egress,
14645                &mut swap.incumbent_rx,
14646                100,
14647                "aft",
14648                "swap-cotenant",
14649            )
14650            .await;
14651            handler
14652                .handle_control_frame(
14653                    &swap.incumbent_ctx,
14654                    route_bind_ack(cotenant_bind.header.corr),
14655                )
14656                .await
14657                .unwrap();
14658            assert!(cotenant_task.await.unwrap().is_empty());
14659            let (cotenant_channel, cotenant_epoch) =
14660                published_route(&cotenant_rx.recv().await.unwrap());
14661
14662            // A second route.open, relayed to the incumbent and not yet acked.
14663            let caller = ConnectionId::new(32);
14664            let (caller_ctx, mut caller_rx) = route_ctx(caller);
14665            let (caller_task, caller_bind) = relay_route_open(
14666                &handler,
14667                caller,
14668                &caller_ctx.egress,
14669                &mut swap.incumbent_rx,
14670                101,
14671                "aft",
14672                "swap-caller",
14673            )
14674            .await;
14675            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14676
14677            register_candidate(&swap, None);
14678            cutover(&swap);
14679
14680            // The incumbent acks after promotion and before any drain.
14681            let ack = handler
14682                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14683                .await;
14684            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14685            if module_loop_error.is_some() {
14686                // What the connection loop does with an untranslated router
14687                // error: end the connection, releasing every route on it.
14688                handler.cleanup_connection(INCUMBENT).unwrap();
14689            }
14690
14691            // 1. The incumbent's other routes survive.
14692            assert!(
14693                cotenant_rx.try_recv().is_err(),
14694                "the co-tenant route on the incumbent was torn down by one late ack: \
14695                 {module_loop_error:?}"
14696            );
14697            assert!(matches!(
14698                swap.forwarding
14699                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14700                    .unwrap(),
14701                DataRoute::Client(DataRouteState::Bound(_))
14702            ));
14703            assert_eq!(module_loop_error, None);
14704            assert!(swap
14705                .registry
14706                .get_module_by_connection(INCUMBENT)
14707                .unwrap()
14708                .is_some());
14709
14710            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
14711            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14712                .await
14713                .expect("the incumbent is told to drop the abandoned binding")
14714                .unwrap()
14715                .frame;
14716            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14717            assert_eq!(goodbye.header.channel, abandoned_channel);
14718            assert_eq!(goodbye.header.epoch, abandoned_epoch);
14719            assert!(swap.incumbent_rx.try_recv().is_err());
14720
14721            // 3. The waiting client gets a retryable refusal and no route.
14722            let response = caller_task.await.unwrap();
14723            assert_eq!(response.len(), 1);
14724            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14725            assert!(caller_rx.try_recv().is_err());
14726
14727            // 4. The reservation pair is given back, and the pending bind
14728            //    settled exactly once: one accepted open (the co-tenant) and one
14729            //    refused open (the caller), nothing counted twice.
14730            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14731            let counters = handler.counters().snapshot();
14732            assert_eq!(
14733                keyed_total(&counters, "route_open_accepted_by_principal"),
14734                1
14735            );
14736            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14737            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14738        }
14739
14740        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
14741        /// id would resolve to the promoted candidate and every new route.open
14742        /// would be refused as reloading, leaving neither process routable.
14743        #[tokio::test]
14744        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14745            let mut swap = swap_with_incumbent().await;
14746            register_candidate(&swap, None);
14747            let incumbent = cutover(&swap);
14748            swap.forwarding
14749                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14750                .unwrap()
14751                .expect("the incumbent is still registered");
14752
14753            let client = ConnectionId::new(33);
14754            let (client_ctx, mut client_rx) = route_ctx(client);
14755            let route_handler = swap.handler.clone();
14756            let open_ctx = RouteCtx {
14757                connection_id: client,
14758                egress: client_ctx.egress.clone(),
14759            };
14760            let mut route_task = tokio::spawn(async move {
14761                route_handler
14762                    .handle_control_frame(
14763                        &open_ctx,
14764                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14765                    )
14766                    .await
14767                    .unwrap()
14768            });
14769            let bind = tokio::select! {
14770                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14771                response = &mut route_task => {
14772                    let response = response.unwrap();
14773                    panic!(
14774                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
14775                        parse_error(&response[0])["code"]
14776                    );
14777                }
14778            };
14779            swap.handler
14780                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14781                .await
14782                .unwrap();
14783            assert!(route_task.await.unwrap().is_empty());
14784            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14785            match swap
14786                .forwarding
14787                .lookup_data_route(client, channel, epoch)
14788                .unwrap()
14789            {
14790                DataRoute::Client(DataRouteState::Bound(route)) => {
14791                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14792                }
14793                other => panic!("expected a bound route on the candidate, got {other:?}"),
14794            }
14795            assert!(swap.incumbent_rx.try_recv().is_err());
14796        }
14797
14798        /// A candidate declares itself ready with `catalog.update` on its own
14799        /// connection. If the connection-keyed registry lookups searched only the
14800        /// active slot, this would answer `not_registered` and the candidate
14801        /// would never become ready.
14802        #[tokio::test]
14803        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14804            let swap = swap_with_incumbent().await;
14805            register_candidate(&swap, Some(false));
14806            let update = Frame::build(
14807                FrameType::Request,
14808                control_flags(),
14809                0,
14810                0,
14811                55,
14812                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14813                    provides: manifest("aft", PROTOCOL_VERSION).provides,
14814                    capabilities: None,
14815                    ready: Some(true),
14816                })
14817                .unwrap(),
14818            )
14819            .unwrap();
14820
14821            let replies = swap
14822                .handler
14823                .handle_control_frame(&swap.candidate_ctx, update)
14824                .await
14825                .unwrap();
14826
14827            assert_eq!(replies.len(), 1);
14828            assert_eq!(
14829                replies[0].header.ty,
14830                FrameType::Response,
14831                "candidate catalog.update was refused: {:?}",
14832                serde_json::from_slice::<Value>(&replies[0].body).ok()
14833            );
14834            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14835            assert_eq!(
14836                swap.registry
14837                    .get_module("aft")
14838                    .unwrap()
14839                    .unwrap()
14840                    .connection_id,
14841                INCUMBENT
14842            );
14843        }
14844    }
14845
14846    /// The HELLO gate while the supervisor has a swap open: only the nonce it
14847    /// minted for the candidate admits a second process, into the candidate
14848    /// slot, and that check runs ahead of the reserved-module gate.
14849    mod swap_admission {
14850        use super::*;
14851
14852        const INCUMBENT_NONCE: &str = "incumbent-nonce";
14853        const CANDIDATE_NONCE: &str = "candidate-nonce";
14854
14855        fn handler_with_incumbent(
14856            module_id: &str,
14857            reserved: bool,
14858        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14859            let registry = Arc::new(Registry::default());
14860            let supervisor = SupervisorHandle::new();
14861            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14862            if reserved {
14863                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14864            }
14865            let handler =
14866                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
14867            let incumbent = handler
14868                .handle_control(
14869                    ConnectionId::new(1),
14870                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14871                )
14872                .unwrap();
14873            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14874            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14875            (registry, supervisor, handler)
14876        }
14877
14878        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
14879        /// admits every nonce, so while a swap is open the swap gate is the only
14880        /// thing between a key-holder and the candidate slot. A nonce the
14881        /// supervisor did not mint, or none at all, is refused, and neither the
14882        /// incumbent's registration nor the candidate slot moves.
14883        #[test]
14884        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
14885            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
14886
14887            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
14888                let replies = handler
14889                    .handle_control(
14890                        ConnectionId::new(connection),
14891                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
14892                    )
14893                    .unwrap();
14894                assert_eq!(replies[0].header.ty, FrameType::Error);
14895                assert_eq!(
14896                    parse_error(&replies[0])["code"],
14897                    "swap_token_invalid",
14898                    "nonce {nonce:?}"
14899                );
14900            }
14901            assert!(registry.get_candidate("aft").unwrap().is_none());
14902            assert_eq!(
14903                registry.get_module("aft").unwrap().unwrap().connection_id,
14904                ConnectionId::new(1)
14905            );
14906
14907            // Control: the minted token is admitted, into the candidate slot,
14908            // and only once.
14909            let admitted = handler
14910                .handle_control(
14911                    ConnectionId::new(4),
14912                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
14913                )
14914                .unwrap();
14915            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
14916            assert_eq!(
14917                registry
14918                    .get_candidate("aft")
14919                    .unwrap()
14920                    .unwrap()
14921                    .connection_id,
14922                ConnectionId::new(4)
14923            );
14924            assert_eq!(
14925                registry.get_module("aft").unwrap().unwrap().connection_id,
14926                ConnectionId::new(1),
14927                "the candidate must not take the active slot"
14928            );
14929            let replayed = handler
14930                .handle_control(
14931                    ConnectionId::new(5),
14932                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
14933                )
14934                .unwrap();
14935            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
14936
14937            // The case only this gate covers: the incumbent has died mid-swap,
14938            // so its duplicate refusal is gone too, and without the gate a
14939            // key-holder would take the id's ACTIVE slot.
14940            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
14941            let squatter = handler
14942                .handle_control(
14943                    ConnectionId::new(6),
14944                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
14945                )
14946                .unwrap();
14947            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
14948            assert!(
14949                registry.get_module("aft").unwrap().is_none(),
14950                "a squatter took the active slot of an id being swapped"
14951            );
14952        }
14953
14954        /// Design mutation arm (iii). A reserved module's candidate presents a
14955        /// nonce the reserved gate has never seen (that gate holds the
14956        /// incumbent's), so the swap gate must run first or the candidate is
14957        /// refused `reserved_module` and a reserved module can never be swapped.
14958        #[test]
14959        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
14960            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
14961
14962            let replies = handler
14963                .handle_control(
14964                    ConnectionId::new(2),
14965                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14966                )
14967                .unwrap();
14968
14969            assert_eq!(
14970                replies[0].header.ty,
14971                FrameType::HelloAck,
14972                "reserved candidate refused: {:?}",
14973                serde_json::from_slice::<Value>(&replies[0].body).ok()
14974            );
14975            assert_eq!(
14976                registry
14977                    .get_candidate("vault")
14978                    .unwrap()
14979                    .unwrap()
14980                    .connection_id,
14981                ConnectionId::new(2)
14982            );
14983        }
14984
14985        /// With no swap open the gate is inert: the incumbent's reserved gate
14986        /// and duplicate refusal behave exactly as before.
14987        #[test]
14988        fn without_an_open_swap_the_ordinary_gates_decide() {
14989            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
14990            supervisor.close_swap("vault");
14991
14992            let candidate = handler
14993                .handle_control(
14994                    ConnectionId::new(2),
14995                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14996                )
14997                .unwrap();
14998            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
14999            let duplicate = handler
15000                .handle_control(
15001                    ConnectionId::new(3),
15002                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15003                )
15004                .unwrap();
15005            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15006            assert!(registry.get_candidate("vault").unwrap().is_none());
15007        }
15008    }
15009
15010    /// `scope.sync` and `scope.describe` through the real control handler: who
15011    /// may sync is decided by the registration and launch nonce of the module
15012    /// connection, never by the request body.
15013    mod scopes {
15014        mod idle_tests {
15015            use super::*;
15016            include!("scope_idle_tests.rs");
15017        }
15018        mod route_open_timings {
15019            use super::*;
15020            include!("route_open_timing_tests.rs");
15021        }
15022        mod activity {
15023            include!("activity_scope_tests.rs");
15024        }
15025        mod delta {
15026            include!("scope_handler_delta_tests.rs");
15027        }
15028        mod call_modes {
15029            include!("scope_call_mode_tests.rs");
15030        }
15031        #[tokio::test]
15032        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
15033            let rig = rig().await;
15034            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
15035                "s".into(),
15036                1,
15037                "tool".into(),
15038            ))
15039            .unwrap();
15040            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
15041            let (direct, _rx) = wide_ctx(999);
15042            for (ctx, expected) in [
15043                (&rig.owner, FrameType::Response),
15044                (&direct, FrameType::Error),
15045            ] {
15046                let frame =
15047                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
15048                        .unwrap();
15049                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
15050                assert_eq!(replies[0].header.ty, expected);
15051                if expected == FrameType::Response {
15052                    let reply: subc_control::ScopeServedByReply =
15053                        serde_json::from_slice(&replies[0].body).unwrap();
15054                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
15055                } else {
15056                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
15057                }
15058            }
15059            let reply = call(
15060                &rig.handler,
15061                &rig.owner,
15062                &ModuleControlRequestFromModule::ScopeDescribe {
15063                    owner: Principal::Reserved {
15064                        module_id: OWNER.into(),
15065                    },
15066                    scope_ref: "s".into(),
15067                },
15068            )
15069            .await;
15070            assert!(matches!(
15071                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
15072                ModuleControlResponseToModule::ScopeDescribe { .. }
15073            ));
15074        }
15075
15076        use subc_protocol::scope::{
15077            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
15078            ScopeStatus,
15079        };
15080
15081        use super::*;
15082
15083        const OWNER: &str = "prefrontal-core";
15084
15085        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
15086            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
15087        }
15088
15089        async fn call(
15090            handler: &ControlHandler,
15091            ctx: &RouteCtx,
15092            request: &ModuleControlRequestFromModule,
15093        ) -> Frame {
15094            let body = serde_json::to_vec(request).unwrap();
15095            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
15096            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
15097            assert_eq!(replies.len(), 1, "{replies:?}");
15098            replies.pop().unwrap()
15099        }
15100
15101        async fn sync(
15102            handler: &ControlHandler,
15103            ctx: &RouteCtx,
15104            generation: u64,
15105            scopes: Vec<ScopeRecord>,
15106        ) -> Result<ModuleControlResponseToModule, String> {
15107            let reply = call(
15108                handler,
15109                ctx,
15110                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
15111            )
15112            .await;
15113            match reply.header.ty {
15114                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
15115                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
15116            }
15117        }
15118
15119        async fn describe(
15120            handler: &ControlHandler,
15121            ctx: &RouteCtx,
15122            owner: &str,
15123            scope_ref: &str,
15124        ) -> ModuleControlResponseToModule {
15125            let reply = call(
15126                handler,
15127                ctx,
15128                &ModuleControlRequestFromModule::ScopeDescribe {
15129                    owner: Principal::Reserved {
15130                        module_id: owner.to_string(),
15131                    },
15132                    scope_ref: scope_ref.to_string(),
15133                },
15134            )
15135            .await;
15136            assert_eq!(
15137                reply.header.ty,
15138                FrameType::Response,
15139                "{:?}",
15140                parse_error(&reply)
15141            );
15142            serde_json::from_slice(&reply.body).unwrap()
15143        }
15144
15145        #[tokio::test]
15146        async fn scope_describe_never_echoes_bind_proof_or_hash() {
15147            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15148            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15149            let mut rig = rig().await;
15150            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15151                .await;
15152            let _bound = rig
15153                .bound(
15154                    Some(AFT),
15155                    PLEXUS,
15156                    Some(
15157                        rig.selector("s", Some(1))
15158                            .with_bind_proof(Some(PROOF.into())),
15159                    ),
15160                )
15161                .await;
15162            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
15163            let output = serde_json::to_string(&reply).unwrap();
15164            assert!(output.contains("\"status\":\"live\""));
15165            assert!(!output.contains(PROOF));
15166            assert!(!output.contains(HASH));
15167        }
15168
15169        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
15170        async fn module(
15171            handler: &ControlHandler,
15172            connection: u64,
15173            module_id: &str,
15174            nonce: Option<&str>,
15175        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15176            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
15177            hello_via_sink(
15178                handler,
15179                &ctx,
15180                &mut rx,
15181                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
15182            )
15183            .await;
15184            (ctx, rx)
15185        }
15186
15187        /// `direct` and every other client connection has no registration, so
15188        /// it can neither sync nor own a scope.
15189        #[tokio::test]
15190        async fn a_client_connection_cannot_sync_or_describe() {
15191            let handler = ControlHandler::new(Arc::new(Registry::default()));
15192            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
15193            for request in [
15194                ModuleControlRequestFromModule::ScopeSync {
15195                    generation: 1,
15196                    scopes: vec![head("s", 1)],
15197                },
15198                ModuleControlRequestFromModule::ScopeDescribe {
15199                    owner: Principal::Direct,
15200                    scope_ref: "s".to_string(),
15201                },
15202            ] {
15203                let reply = call(&handler, &ctx, &request).await;
15204                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
15205            }
15206            assert!(
15207                !handler
15208                    .scopes
15209                    .read()
15210                    .unwrap()
15211                    .describe(
15212                        &Principal::Reserved {
15213                            module_id: OWNER.to_string()
15214                        },
15215                        "s"
15216                    )
15217                    .owner_synced
15218            );
15219        }
15220
15221        /// A module the supervisor did not spawn registers without a launch
15222        /// nonce, so it is never an owner's current launch.
15223        #[tokio::test]
15224        async fn a_module_without_a_supervised_launch_cannot_sync() {
15225            let handler = ControlHandler::new(Arc::new(Registry::default()));
15226            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
15227            assert_eq!(
15228                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
15229                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15230            );
15231        }
15232
15233        #[tokio::test]
15234        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
15235            let supervisor = SupervisorHandle::new();
15236            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15237            let handler = ControlHandler::new(Arc::new(Registry::default()))
15238                .with_supervisor(supervisor.clone());
15239            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15240            sync(&handler, &incumbent, 1, vec![head("s", 1)])
15241                .await
15242                .expect("the current launch syncs");
15243
15244            // A swap candidate registers with the swap token and is refused
15245            // while the incumbent keeps syncing.
15246            supervisor.open_swap(OWNER, "n2".to_string());
15247            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
15248            assert_eq!(
15249                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15250                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15251            );
15252            sync(&handler, &incumbent, 2, vec![head("s", 1)])
15253                .await
15254                .expect("the serving owner syncs during the swap");
15255
15256            // The swap fails and is rolled back. The candidate never held sync
15257            // authority, and still cannot sync.
15258            supervisor.close_swap(OWNER);
15259            assert_eq!(
15260                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15261                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15262            );
15263            sync(&handler, &incumbent, 3, vec![head("s", 1)])
15264                .await
15265                .expect("the serving owner syncs after the rollback");
15266            handler.cleanup_connection(candidate.connection_id).unwrap();
15267
15268            // A swap that cuts over. Promotion records the candidate's nonce as
15269            // the module's spawn nonce, which is what `set_spawn_nonce` does
15270            // here; the promoted connection then takes authority at any
15271            // generation and the superseded incumbent is refused.
15272            supervisor.open_swap(OWNER, "n3".to_string());
15273            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
15274            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
15275            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
15276                .await
15277                .expect("the promoted launch takes authority");
15278            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15279                panic!("unexpected reply {reply:?}");
15280            };
15281            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
15282            assert_eq!(
15283                sync(&handler, &incumbent, 4, Vec::new()).await,
15284                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15285            );
15286        }
15287
15288        /// Authority dies with its connection: the cleanup path releases it,
15289        /// so the owner's next connection takes it at any generation.
15290        #[tokio::test]
15291        async fn closing_the_authority_connection_frees_sync_authority() {
15292            let supervisor = SupervisorHandle::new();
15293            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15294            let handler = ControlHandler::new(Arc::new(Registry::default()))
15295                .with_supervisor(supervisor.clone());
15296            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15297            sync(&handler, &first, 10, vec![head("s", 1)])
15298                .await
15299                .unwrap();
15300            handler.cleanup_connection(first.connection_id).unwrap();
15301
15302            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
15303            sync(&handler, &second, 1, vec![head("s", 1)])
15304                .await
15305                .expect("the next connection takes the released authority");
15306        }
15307
15308        #[tokio::test]
15309        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
15310            let supervisor = SupervisorHandle::new();
15311            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15312            let handler =
15313                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
15314            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
15315            sync(&handler, &first, 10, vec![head("s", 1)])
15316                .await
15317                .unwrap();
15318            handler
15319                .handle_control_frame(
15320                    &first,
15321                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
15322                )
15323                .await
15324                .unwrap();
15325            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
15326            sync(&handler, &second, 1, vec![head("s", 1)])
15327                .await
15328                .expect("GOODBYE releases authority even if the old socket remains open");
15329        }
15330
15331        #[tokio::test]
15332        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
15333            let registry = Arc::new(Registry::default());
15334            let supervisor_handle = SupervisorHandle::new();
15335            let supervisor =
15336                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
15337                    .with_handle(supervisor_handle.clone())
15338                    .with_daemon_incarnation("incarnation-7".to_string());
15339            // Configured with enabled: false, so the supervisor lists the
15340            // module without spawning a process for it.
15341            supervisor
15342                .supervise_configured(
15343                    ModuleSpec {
15344                        module_id: OWNER.to_string(),
15345                        program: PathBuf::from("/nonexistent/prefrontal-core"),
15346                        args: Vec::new(),
15347                        env: Vec::new(),
15348                        reserved: false,
15349                        reserved_prefixes: Vec::new(),
15350                        protocol: ModuleProtocol::Subc,
15351                        overlap: Default::default(),
15352                    },
15353                    false,
15354                )
15355                .unwrap();
15356            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
15357            let handler =
15358                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
15359            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
15360
15361            // Configured but not yet synced: a reader waits for the owner.
15362            let ModuleControlResponseToModule::ScopeDescribe {
15363                status,
15364                daemon_incarnation,
15365                owner_synced,
15366                owner_configured,
15367                scope,
15368                ..
15369            } = describe(&handler, &reader, OWNER, "s").await
15370            else {
15371                panic!("not a describe reply");
15372            };
15373            assert_eq!(status, ScopeStatus::NotLive);
15374            assert_eq!(daemon_incarnation, "incarnation-7");
15375            assert!(!owner_synced);
15376            assert!(owner_configured);
15377            assert!(scope.is_none());
15378
15379            // Not a supervised module: the owner will never sync, and a reader
15380            // refuses rather than waits.
15381            let ModuleControlResponseToModule::ScopeDescribe {
15382                status,
15383                owner_configured,
15384                ..
15385            } = describe(&handler, &reader, "ghost", "s").await
15386            else {
15387                panic!("not a describe reply");
15388            };
15389            assert_eq!(status, ScopeStatus::NotLive);
15390            assert!(!owner_configured);
15391
15392            // Live, with the stamp fields and the computed owner_authorized.
15393            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
15394            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
15395            let ModuleControlResponseToModule::ScopeDescribe {
15396                status,
15397                scope_epoch,
15398                owner_synced,
15399                scope,
15400                ..
15401            } = describe(&handler, &reader, OWNER, "s").await
15402            else {
15403                panic!("not a describe reply");
15404            };
15405            assert_eq!(status, ScopeStatus::Live);
15406            assert_eq!(scope_epoch, Some(4));
15407            assert!(owner_synced);
15408            let stamp = scope.expect("a live scope carries its stamp");
15409            assert!(
15410                stamp.owner_authorized,
15411                "prefrontal-core is the default authority"
15412            );
15413            assert_eq!(stamp.kind, ScopeKind::Head);
15414        }
15415
15416        #[tokio::test]
15417        async fn scope_authority_owners_decides_owner_authorized() {
15418            let supervisor = SupervisorHandle::new();
15419            supervisor.set_spawn_nonce("broca", "b1".to_string());
15420            let handler = ControlHandler::new(Arc::new(Registry::default()))
15421                .with_supervisor(supervisor)
15422                .with_scope_authority_owners(vec!["broca".to_string()]);
15423            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
15424            let mut gated = head("s", 1);
15425            gated.attributes.agent_id = Some("agent".to_string());
15426            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
15427            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
15428                describe(&handler, &broca, "broca", "s").await
15429            else {
15430                panic!("not a describe reply");
15431            };
15432            assert!(scope.unwrap().owner_authorized);
15433        }
15434
15435        /// With route admission, the stamp, the commit re-check and drains in
15436        /// place, the feature is advertised: the module ops in HELLO_ACK, and
15437        /// `scopes/v1` in HELLO_ACK and `server.describe`.
15438        #[tokio::test]
15439        async fn scope_ops_and_the_scopes_capability_are_advertised() {
15440            let handler = ControlHandler::new(Arc::new(Registry::default()));
15441            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
15442            let ack = hello_via_sink(
15443                &handler,
15444                &ctx,
15445                &mut rx,
15446                hello_frame("m", PROTOCOL_VERSION, 1),
15447            )
15448            .await;
15449            let ack = parse_ack(&ack);
15450            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
15451                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
15452            }
15453            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
15454
15455            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
15456            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15457            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15458            let reply = handler
15459                .handle_control_frame(&client, frame)
15460                .await
15461                .unwrap()
15462                .pop()
15463                .unwrap();
15464            let ClientControlResponse::ServerDescribe { capabilities, .. } =
15465                serde_json::from_slice(&reply.body).unwrap()
15466            else {
15467                panic!("not a server.describe reply");
15468            };
15469            assert!(
15470                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
15471                "{capabilities:?}"
15472            );
15473        }
15474
15475        // ---- route admission, stamps, commit re-check and drains ----------
15476
15477        const PLEXUS: &str = "plexus";
15478        const OTHER: &str = "other";
15479        const AFT: &str = "aft";
15480        const BROCA: &str = "broca";
15481        const MAGIC: &str = "magic-context";
15482
15483        fn nonce(module_id: &str) -> String {
15484            format!("nonce-{module_id}")
15485        }
15486
15487        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15488            let (tx, rx) = mpsc::channel(64);
15489            (
15490                RouteCtx {
15491                    connection_id: ConnectionId::new(connection),
15492                    egress: FrameSink::new(tx),
15493                },
15494                rx,
15495            )
15496        }
15497
15498        /// A daemon with a configured owner (prefrontal-core) registered on its
15499        /// own module connection, two routable targets (plexus, other), and
15500        /// launch nonces minted for the modules that open routes as carriers.
15501        struct Rig {
15502            handler: ControlHandler,
15503            forwarding: Arc<ForwardingTable>,
15504            owner: RouteCtx,
15505            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15506            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
15507            generation: u64,
15508            next_connection: u64,
15509            _supervisor: Supervisor,
15510        }
15511
15512        async fn rig() -> Rig {
15513            rig_with_flow_support(true).await
15514        }
15515
15516        async fn rig_with_flow_support(flow_support: bool) -> Rig {
15517            rig_with_scope_capabilities(flow_support, false).await
15518        }
15519
15520        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
15521            let mut provides = Vec::new();
15522            if flow_support {
15523                provides.push("flow-scopes/v1");
15524            }
15525            if run_support {
15526                provides.push("agent-run-scopes/v1");
15527            }
15528            rig_with_target_capabilities(&provides).await
15529        }
15530
15531        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
15532            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
15533            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15534            // Decoder versions alone cannot admit scoped calls: only a capability
15535            // declaration promises the required run or flow behaviour.
15536            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15537            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
15538            Frame::build(
15539                FrameType::Hello,
15540                control_flags(),
15541                0,
15542                0,
15543                connection,
15544                serde_json::to_vec(&body).unwrap(),
15545            )
15546            .unwrap()
15547        }
15548
15549        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
15550            let registry = Arc::new(Registry::default());
15551            let forwarding = Arc::new(ForwardingTable::default());
15552            let supervisor_handle = SupervisorHandle::new();
15553            let supervisor =
15554                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
15555                    .with_handle(supervisor_handle.clone());
15556            supervisor
15557                .supervise_configured(
15558                    ModuleSpec {
15559                        module_id: OWNER.to_string(),
15560                        program: PathBuf::from("/nonexistent/prefrontal-core"),
15561                        args: Vec::new(),
15562                        env: Vec::new(),
15563                        reserved: false,
15564                        reserved_prefixes: Vec::new(),
15565                        protocol: ModuleProtocol::Subc,
15566                        overlap: Default::default(),
15567                    },
15568                    false,
15569                )
15570                .unwrap();
15571            for module_id in [OWNER, AFT, BROCA, MAGIC] {
15572                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
15573            }
15574            let handler =
15575                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15576                    .with_supervisor(supervisor_handle);
15577            let (owner, mut owner_rx) = wide_ctx(1);
15578            hello_via_sink(
15579                &handler,
15580                &owner,
15581                &mut owner_rx,
15582                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
15583            )
15584            .await;
15585            let mut modules = BTreeMap::new();
15586            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
15587                let (ctx, mut rx) = wide_ctx(connection);
15588                let hello = hello_with_capabilities(module_id, connection, provides);
15589                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
15590                modules.insert(module_id.to_string(), (ctx, rx));
15591            }
15592            Rig {
15593                handler,
15594                forwarding,
15595                owner,
15596                _owner_rx: owner_rx,
15597                modules,
15598                generation: 0,
15599                next_connection: 100,
15600                _supervisor: supervisor,
15601            }
15602        }
15603
15604        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
15605            ScopeCarrier::new(Principal::Reserved {
15606                module_id: module_id.to_string(),
15607            })
15608            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
15609        }
15610
15611        /// The scope most tests open under: aft carries to any module, broca
15612        /// only to plexus and other, and the owner delegates as agent-1.
15613        fn session(scope_epoch: u64) -> ScopeRecord {
15614            let mut record = head("s", scope_epoch);
15615            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
15616            record.attributes.agent_id = Some("agent-1".to_string());
15617            record.attributes.delegates = true;
15618            record
15619        }
15620
15621        impl Rig {
15622            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
15623                self.generation += 1;
15624                sync(&self.handler, &self.owner, self.generation, scopes)
15625                    .await
15626                    .expect("the owner's sync is accepted");
15627            }
15628
15629            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15630                ScopeSelector::new(
15631                    Principal::Reserved {
15632                        module_id: OWNER.to_string(),
15633                    },
15634                    scope_ref.to_string(),
15635                    scope_epoch,
15636                )
15637            }
15638
15639            fn open_frame(
15640                &mut self,
15641                opener: Option<&str>,
15642                target: &str,
15643                scope: Option<ScopeSelector>,
15644            ) -> (
15645                RouteCtx,
15646                mpsc::Receiver<crate::router::OutboundFrame>,
15647                Frame,
15648            ) {
15649                self.next_connection += 1;
15650                let (ctx, rx) = wide_ctx(self.next_connection);
15651                let root = unique_project_root("scoped-open");
15652                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15653                    target: RouteTarget::ToolProvider {
15654                        module_id: target.to_string(),
15655                    },
15656                    identity: BindIdentity::new(
15657                        root.path().to_path_buf(),
15658                        "unit".to_string(),
15659                        "session".to_string(),
15660                    ),
15661                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
15662                        module_id: module_id.to_string(),
15663                        launch_nonce: nonce(module_id),
15664                    }),
15665                    consumer_capabilities: None,
15666                    role_versions: None,
15667                    admission_facts: None,
15668                    scope,
15669                })
15670                .unwrap();
15671                let frame = Frame::build(
15672                    FrameType::Request,
15673                    control_flags(),
15674                    0,
15675                    0,
15676                    self.next_connection,
15677                    body,
15678                )
15679                .unwrap();
15680                (ctx, rx, frame)
15681            }
15682
15683            /// Open and expect a refusal before anything is relayed.
15684            async fn refused(
15685                &mut self,
15686                opener: Option<&str>,
15687                target: &str,
15688                scope: Option<ScopeSelector>,
15689            ) -> String {
15690                self.refusal_body(opener, target, scope).await["code"]
15691                    .as_str()
15692                    .unwrap()
15693                    .to_string()
15694            }
15695
15696            async fn refusal_body(
15697                &mut self,
15698                opener: Option<&str>,
15699                target: &str,
15700                scope: Option<ScopeSelector>,
15701            ) -> Value {
15702                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15703                let replies = tokio::time::timeout(
15704                    Duration::from_secs(2),
15705                    self.handler.handle_control_frame(&ctx, frame),
15706                )
15707                .await
15708                .expect("the open must be refused before waiting for a bind ack")
15709                .unwrap();
15710                assert_eq!(replies.len(), 1, "{replies:?}");
15711                assert_eq!(replies[0].header.ty, FrameType::Error);
15712                let (_, module_rx) = self.modules.get_mut(target).unwrap();
15713                assert!(
15714                    module_rx.try_recv().is_err(),
15715                    "a refused open relays nothing"
15716                );
15717                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15718                parse_error(&replies[0])
15719            }
15720
15721            /// Start an open and return its task and the bind the target got.
15722            async fn relayed(
15723                &mut self,
15724                opener: Option<&str>,
15725                target: &str,
15726                scope: Option<ScopeSelector>,
15727            ) -> Relayed {
15728                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15729                let handler = self.handler.clone();
15730                let task_ctx = ctx.clone();
15731                let task = tokio::spawn(async move {
15732                    handler
15733                        .handle_control_frame(&task_ctx, frame)
15734                        .await
15735                        .unwrap()
15736                });
15737                let (_, module_rx) = self.modules.get_mut(target).unwrap();
15738                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15739                    .await
15740                    .expect("the target receives the relayed route.bind")
15741                    .unwrap()
15742                    .frame;
15743                Relayed {
15744                    target: target.to_string(),
15745                    client: ctx,
15746                    client_rx: rx,
15747                    task,
15748                    bind,
15749                }
15750            }
15751
15752            async fn ack(&self, relayed: &Relayed) {
15753                let (module, _) = &self.modules[&relayed.target];
15754                self.handler
15755                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15756                    .await
15757                    .unwrap();
15758            }
15759
15760            /// Open, ack and return the bound route.
15761            async fn bound(
15762                &mut self,
15763                opener: Option<&str>,
15764                target: &str,
15765                scope: Option<ScopeSelector>,
15766            ) -> Bound {
15767                let relayed = self.relayed(opener, target, scope).await;
15768                self.ack(&relayed).await;
15769                let Relayed {
15770                    target,
15771                    client,
15772                    mut client_rx,
15773                    task,
15774                    bind,
15775                } = relayed;
15776                assert!(
15777                    task.await.unwrap().is_empty(),
15778                    "the open is answered by commit"
15779                );
15780                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15781                Bound {
15782                    target,
15783                    client,
15784                    client_rx,
15785                    channel,
15786                    epoch,
15787                    bind,
15788                }
15789            }
15790
15791            fn live(&self, route: &Bound) -> bool {
15792                matches!(
15793                    self.forwarding
15794                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15795                        .unwrap(),
15796                    DataRoute::Client(DataRouteState::Bound(_))
15797                )
15798            }
15799        }
15800
15801        struct Relayed {
15802            target: String,
15803            client: RouteCtx,
15804            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15805            task: tokio::task::JoinHandle<Vec<Frame>>,
15806            bind: Frame,
15807        }
15808
15809        struct Bound {
15810            target: String,
15811            client: RouteCtx,
15812            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15813            channel: u16,
15814            epoch: u32,
15815            bind: Frame,
15816        }
15817
15818        impl Bound {
15819            /// The reason of the `route.closed` this client was sent, after
15820            /// checking it also got a GOODBYE on exactly this route.
15821            fn closed_reason(&mut self) -> RouteCloseReason {
15822                let mut reason = None;
15823                let mut goodbye = false;
15824                while let Ok(outbound) = self.client_rx.try_recv() {
15825                    let frame = outbound.frame;
15826                    match frame.header.ty {
15827                        FrameType::Goodbye => {
15828                            assert_eq!(
15829                                (frame.header.channel, frame.header.epoch),
15830                                (self.channel, self.epoch)
15831                            );
15832                            goodbye = true;
15833                        }
15834                        FrameType::Push => {
15835                            let ClientControlPush::RouteClosed {
15836                                reason: r,
15837                                module_id,
15838                                ..
15839                            } = serde_json::from_slice(&frame.body).unwrap()
15840                            else {
15841                                panic!("unexpected push");
15842                            };
15843                            assert_eq!(module_id, self.target);
15844                            reason = Some(r);
15845                        }
15846                        other => panic!("unexpected frame {other:?}"),
15847                    }
15848                }
15849                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15850                reason.expect("the client is told why the route closed")
15851            }
15852
15853            fn untouched(&mut self) -> bool {
15854                self.client_rx.try_recv().is_err()
15855            }
15856
15857            fn stamp(&self) -> Option<ScopeStamp> {
15858                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15859                    ModuleControlRequest::RouteBind { scope, .. } => scope,
15860                    other => panic!("expected a route.bind, got {other:?}"),
15861                }
15862            }
15863        }
15864
15865        #[tokio::test]
15866        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15867        ) {
15868            let mut rig = rig().await;
15869            let mut record = session(1);
15870            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15871            record.child_owners = vec![Principal::Reserved {
15872                module_id: MAGIC.to_string(),
15873            }];
15874            rig.sync(vec![record]).await;
15875            let scope = || Some(rig_selector("s", Some(1)));
15876
15877            // Admitted: the owner, a bare carrier to any module, a targeted
15878            // carrier to its listed module.
15879            rig.bound(Some(OWNER), PLEXUS, scope()).await;
15880            rig.bound(Some(AFT), OTHER, scope()).await;
15881            rig.bound(Some(BROCA), PLEXUS, scope()).await;
15882
15883            // Refused scope_not_carrier: a targeted carrier to an unlisted
15884            // module, a module that is not listed at all (a child owner is not
15885            // a carrier), and a direct key-holder.
15886            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
15887                assert_eq!(
15888                    rig.refused(opener, target, scope()).await,
15889                    error_codes::SCOPE_NOT_CARRIER,
15890                    "{opener:?} -> {target}"
15891                );
15892            }
15893        }
15894
15895        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15896            ScopeSelector::new(
15897                Principal::Reserved {
15898                    module_id: OWNER.to_string(),
15899                },
15900                scope_ref.to_string(),
15901                scope_epoch,
15902            )
15903        }
15904
15905        #[tokio::test]
15906        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
15907            let mut rig = rig().await;
15908            rig.sync(vec![session(1)]).await;
15909            for opener in [OWNER, AFT] {
15910                assert_eq!(
15911                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
15912                        .await,
15913                    error_codes::SCOPE_EPOCH_REQUIRED,
15914                    "{opener}"
15915                );
15916            }
15917        }
15918
15919        #[tokio::test]
15920        async fn admission_separates_not_synced_not_live_and_ended() {
15921            let mut rig = rig().await;
15922            // Before the configured owner's first sync: retryable.
15923            let code = rig
15924                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15925                .await;
15926            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
15927            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
15928
15929            // An owner that is not configured will never sync: terminal.
15930            let ghost = ScopeSelector::new(
15931                Principal::Reserved {
15932                    module_id: "ghost".to_string(),
15933                },
15934                "s".to_string(),
15935                Some(1),
15936            );
15937            assert_eq!(
15938                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
15939                error_codes::SCOPE_NOT_LIVE
15940            );
15941
15942            rig.sync(vec![session(2)]).await;
15943            assert_eq!(
15944                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
15945                    .await,
15946                error_codes::SCOPE_NOT_LIVE
15947            );
15948            for epoch in [1, 3] {
15949                assert_eq!(
15950                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
15951                        .await,
15952                    error_codes::SCOPE_ENDED,
15953                    "epoch {epoch}"
15954                );
15955            }
15956            // Control: the live epoch is admitted.
15957            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
15958                .await;
15959        }
15960
15961        #[tokio::test]
15962        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
15963            let mut rig = rig().await;
15964            rig.sync(vec![session(1)]).await;
15965            let route = rig
15966                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15967                .await;
15968            let stamp = route.stamp().expect("a scoped bind carries the stamp");
15969            assert_eq!(stamp.scope_ref, "s");
15970            assert_eq!(stamp.scope_epoch, 1);
15971            assert_eq!(stamp.kind, ScopeKind::Head);
15972            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
15973            assert!(stamp.attributes.delegates);
15974            assert!(stamp.owner_authorized);
15975            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15976            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
15977
15978            // broca owns a scope of its own on its own module connection; it is
15979            // not in scope_authority_owners, so its stamp is not authorized.
15980            let (broca, mut broca_rx) = wide_ctx(50);
15981            hello_via_sink(
15982                &rig.handler,
15983                &broca,
15984                &mut broca_rx,
15985                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
15986            )
15987            .await;
15988            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
15989                .await
15990                .unwrap();
15991            let own = ScopeSelector::new(
15992                Principal::Reserved {
15993                    module_id: BROCA.to_string(),
15994                },
15995                "b".to_string(),
15996                Some(1),
15997            );
15998            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
15999            assert!(!route.stamp().unwrap().owner_authorized);
16000        }
16001
16002        #[tokio::test]
16003        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16004            let mut rig = rig().await;
16005            let mut record = head("s", 1);
16006            record.carriers = vec![carrier(AFT, None)];
16007            let flow_id = "Flow:run-7/step_2!~";
16008            record.attributes.flow_id = Some(flow_id.to_string());
16009            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16010                .await
16011                .unwrap();
16012            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16013                panic!("not a sync reply");
16014            };
16015            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
16016            let route = rig
16017                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16018                .await;
16019            assert!(rig.live(&route), "the stamped bind committed");
16020            let stamp = route.stamp().expect("a flow scope carries a stamp");
16021            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
16022            assert_eq!(stamp.attributes.agent_id, None);
16023            assert!(!stamp.attributes.delegates);
16024            assert!(stamp.owner_authorized);
16025            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16026            assert_eq!(unscoped.stamp(), None);
16027        }
16028
16029        #[tokio::test]
16030        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
16031            let mut rig = rig_with_flow_support(false).await;
16032            let mut record = session(1);
16033            record.attributes.flow_id = Some("flow:7".to_string());
16034            rig.sync(vec![record]).await;
16035            for opener in [OWNER, AFT] {
16036                let body = rig
16037                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16038                    .await;
16039                assert_eq!(body["code"], "target_flow_unsupported");
16040                let message = body["message"].as_str().unwrap();
16041                for required in [PLEXUS, "flow-scopes/v1"] {
16042                    assert!(message.contains(required), "{message}");
16043                }
16044            }
16045        }
16046
16047        #[tokio::test]
16048        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
16049            let mut rig = rig_with_flow_support(true).await;
16050            let mut record = session(1);
16051            record.attributes.flow_id = Some("flow:7".to_string());
16052            rig.sync(vec![record]).await;
16053            for opener in [OWNER, AFT] {
16054                let route = rig
16055                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16056                    .await;
16057                assert!(rig.live(&route));
16058                assert_eq!(
16059                    route.stamp().unwrap().attributes.flow_id.as_deref(),
16060                    Some("flow:7")
16061                );
16062            }
16063        }
16064
16065        #[tokio::test]
16066        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
16067            use std::future::Future;
16068
16069            let mut rig = rig_with_flow_support(true).await;
16070            let mut record = session(1);
16071            record.attributes.flow_id = Some("flow:7".to_string());
16072            rig.sync(vec![record]).await;
16073            let (client, mut client_rx, frame) =
16074                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
16075            // Hold the route-open response permit so admission sees the first
16076            // target but relay reservation cannot capture an endpoint yet.
16077            for _ in 0..64 {
16078                client.egress.try_send(route_bind_ack(1)).unwrap();
16079            }
16080            let handler = rig.handler.clone();
16081            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16082            std::future::poll_fn(|cx| {
16083                assert!(open.as_mut().poll(cx).is_pending());
16084                std::task::Poll::Ready(())
16085            })
16086            .await;
16087            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16088
16089            let old_connection = rig.modules[PLEXUS].0.connection_id;
16090            rig.handler.cleanup_connection(old_connection).unwrap();
16091            let (replacement, mut replacement_rx) = wide_ctx(200);
16092            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
16093            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16094            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16095            let hello = Frame::build(
16096                FrameType::Hello,
16097                control_flags(),
16098                0,
16099                0,
16100                200,
16101                serde_json::to_vec(&body).unwrap(),
16102            )
16103            .unwrap();
16104            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
16105
16106            client_rx.try_recv().unwrap();
16107            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16108                .await
16109                .expect("the replacement is refused without waiting for a bind ack")
16110                .unwrap();
16111            assert_eq!(replies.len(), 1);
16112            let body = parse_error(&replies[0]);
16113            assert_eq!(body["code"], "target_flow_unsupported");
16114            for required in [PLEXUS, "flow-scopes/v1"] {
16115                assert!(body["message"].as_str().unwrap().contains(required));
16116            }
16117            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
16118            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16119            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16120            assert!(rig
16121                .handler
16122                .registry
16123                .get_module_by_connection(replacement.connection_id)
16124                .unwrap()
16125                .is_some());
16126        }
16127
16128        #[tokio::test]
16129        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
16130            use std::future::Future;
16131            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16132            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16133            const NEXT_HASH: &str =
16134                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16135            let mut rig = rig().await;
16136            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
16137            record.carriers.push(ScopeCarrier::new(Principal::Direct));
16138            rig.sync(vec![record.clone()]).await;
16139            let (client, mut rx, frame) = rig.open_frame(
16140                None,
16141                PLEXUS,
16142                Some(
16143                    rig.selector("s", Some(1))
16144                        .with_bind_proof(Some(PROOF.into())),
16145                ),
16146            );
16147            // Hold egress reservation while the scope owner changes its bind-proof
16148            // hash, so a proof valid at admission must be checked again at relay.
16149            for _ in 0..64 {
16150                client.egress.try_send(route_bind_ack(1)).unwrap();
16151            }
16152            let handler = rig.handler.clone();
16153            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16154            std::future::poll_fn(|cx| {
16155                assert!(open.as_mut().poll(cx).is_pending());
16156                std::task::Poll::Ready(())
16157            })
16158            .await;
16159            record.bind_proof_sha256 = Some(NEXT_HASH.into());
16160            rig.sync(vec![record]).await;
16161            rx.try_recv().unwrap();
16162            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16163                .await
16164                .unwrap()
16165                .unwrap();
16166            assert_eq!(
16167                parse_error(&replies[0])["code"],
16168                error_codes::SCOPE_BIND_PROOF_MISMATCH
16169            );
16170            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
16171            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16172            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16173        }
16174
16175        #[tokio::test]
16176        async fn scope_bind_proof_reserved_openers_ignore_proof() {
16177            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16178            let mut rig = rig().await;
16179            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16180                .await;
16181            for opener in [OWNER, AFT] {
16182                for proof in [None, Some("malformed+proof=")] {
16183                    let route = rig
16184                        .bound(
16185                            Some(opener),
16186                            PLEXUS,
16187                            Some(
16188                                rig.selector("s", Some(1))
16189                                    .with_bind_proof(proof.map(str::to_string)),
16190                            ),
16191                        )
16192                        .await;
16193                    assert!(rig.live(&route));
16194                }
16195            }
16196        }
16197
16198        #[tokio::test]
16199        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
16200            use std::future::Future;
16201            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16202            const NEXT_HASH: &str =
16203                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16204            let mut rig = rig().await;
16205            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16206                .await;
16207            let (client, mut rx, frame) =
16208                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
16209            for _ in 0..64 {
16210                client.egress.try_send(route_bind_ack(1)).unwrap();
16211            }
16212            let handler = rig.handler.clone();
16213            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16214            std::future::poll_fn(|cx| {
16215                assert!(open.as_mut().poll(cx).is_pending());
16216                std::task::Poll::Ready(())
16217            })
16218            .await;
16219            rig.sync(vec![
16220                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
16221            ])
16222            .await;
16223            rx.try_recv().unwrap();
16224            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16225            let bind = std::future::poll_fn(|cx| {
16226                assert!(open.as_mut().poll(cx).is_pending());
16227                module_rx.poll_recv(cx)
16228            })
16229            .await
16230            .unwrap()
16231            .frame;
16232            handler
16233                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
16234                .await
16235                .unwrap();
16236            let replies = tokio::time::timeout(Duration::from_secs(2), open)
16237                .await
16238                .unwrap()
16239                .unwrap();
16240            assert_eq!(replies.len(), 1);
16241            // Proof rechecking allowed the relay. Independent tag settlement
16242            // still asks the opener to retry after any same-epoch content change.
16243            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
16244            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16245            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
16246            let route = rig
16247                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
16248                .await;
16249            assert!(rig.live(&route));
16250        }
16251
16252        #[tokio::test]
16253        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
16254        {
16255            let mut rig = rig_with_flow_support(false).await;
16256            rig.sync(vec![session(1)]).await;
16257            let route = rig
16258                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16259                .await;
16260            assert!(rig.live(&route));
16261            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
16262            let mut record = session(1);
16263            record.attributes.flow_id = Some("flow:7".to_string());
16264            rig.sync(vec![record]).await;
16265            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
16266            assert!(rig.live(&unscoped));
16267            assert_eq!(unscoped.stamp(), None);
16268        }
16269
16270        #[tokio::test]
16271        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
16272            let mut rig = rig().await;
16273            let mut record = session(1);
16274            record.attributes.flow_id = Some("flow:7".to_string());
16275            rig.sync(vec![record.clone()]).await;
16276            let before = rig.forwarding.published_scope_tag(OWNER, "s");
16277            let mut owner_route = rig
16278                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16279                .await;
16280            let mut carrier_route = rig
16281                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
16282                .await;
16283            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16284            rig.sync(vec![record.clone()]).await;
16285            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16286            assert!(rig.live(&owner_route) && owner_route.untouched());
16287            assert!(rig.live(&carrier_route) && carrier_route.untouched());
16288
16289            record.attributes.flow_id = Some("flow:8".to_string());
16290            rig.sync(vec![record]).await;
16291            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
16292            let before = before.unwrap();
16293            assert_eq!(after.scope_epoch, before.scope_epoch);
16294            assert!(after.version > before.version);
16295            for route in [&mut owner_route, &mut carrier_route] {
16296                assert!(!rig.live(route));
16297                assert_eq!(
16298                    route.closed_reason(),
16299                    RouteCloseReason::ScopeDelegationChanged
16300                );
16301            }
16302            assert!(rig.live(&unscoped) && unscoped.untouched());
16303            // Each provider also receives a GOODBYE for its drained route;
16304            // consume it before expecting the next route.bind on that sink.
16305            for target in [PLEXUS, OTHER] {
16306                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
16307                let goodbye = module_rx
16308                    .try_recv()
16309                    .expect("the provider sees the drain")
16310                    .frame;
16311                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16312                assert!(module_rx.try_recv().is_err());
16313            }
16314            let rebound = rig
16315                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16316                .await;
16317            assert_eq!(
16318                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
16319                Some("flow:8")
16320            );
16321        }
16322
16323        /// The owner's sync lands between admission and the module's ack. The
16324        /// open is refused by name, the module's other routes stay up, and the
16325        /// reserved pair is released. Changed content is retryable; an ended
16326        /// scope is not.
16327        #[tokio::test]
16328        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
16329            let mut rig = rig().await;
16330            rig.sync(vec![session(1)]).await;
16331            let mut cotenant = rig
16332                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16333                .await;
16334
16335            let mut changed = session(1);
16336            changed.child_owners.push(Principal::Reserved {
16337                module_id: MAGIC.to_string(),
16338            });
16339            let mut ended = None;
16340            for (code, next) in [
16341                (error_codes::SCOPE_CHANGED, vec![changed]),
16342                (error_codes::SCOPE_ENDED, Vec::new()),
16343            ] {
16344                let relayed = rig
16345                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16346                    .await;
16347                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
16348                ended = Some(next.is_empty());
16349                rig.sync(next).await;
16350                rig.ack(&relayed).await;
16351                let replies = relayed.task.await.unwrap();
16352                assert_eq!(replies.len(), 1, "{replies:?}");
16353                assert_eq!(parse_error(&replies[0])["code"], code);
16354                assert_eq!(
16355                    subc_protocol::error_codes::is_retryable_route_open(code),
16356                    code == error_codes::SCOPE_CHANGED
16357                );
16358                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16359                // The module is told to drop just the binding it created.
16360                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16361                // Collected, because ending the scope also closes the co-tenant
16362                // route, whose GOODBYE comes first.
16363                let mut goodbyes = Vec::new();
16364                while let Ok(outbound) = plexus_rx.try_recv() {
16365                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
16366                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
16367                }
16368                assert!(
16369                    goodbyes.contains(&(bind_channel, bind_epoch)),
16370                    "{goodbyes:?}"
16371                );
16372                assert!(rig
16373                    .handler
16374                    .registry
16375                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
16376                    .unwrap()
16377                    .is_some());
16378            }
16379            assert_eq!(ended, Some(true));
16380            // The co-tenant stayed up through the change, and closed only when
16381            // the scope ended, by the drain rule rather than by the commit.
16382            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
16383        }
16384
16385        /// Each row of the drain table on one set of routes: the owner's, a
16386        /// bare carrier's, and a targeted carrier's to each of its targets.
16387        #[tokio::test]
16388        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
16389            struct Case {
16390                name: &'static str,
16391                change: fn(&mut ScopeRecord),
16392                /// Closed routes by index: owner->plexus, aft->plexus,
16393                /// broca->plexus, broca->other.
16394                closed: [Option<RouteCloseReason>; 4],
16395            }
16396            use RouteCloseReason::*;
16397            let cases = [
16398                Case {
16399                    name: "a carrier entry removed",
16400                    change: |r| {
16401                        r.carriers.retain(|c| {
16402                            c.principal
16403                                != Principal::Reserved {
16404                                    module_id: AFT.to_string(),
16405                                }
16406                        })
16407                    },
16408                    closed: [None, Some(ScopeCarrierRemoved), None, None],
16409                },
16410                Case {
16411                    name: "a target removed from a carrier",
16412                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
16413                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
16414                },
16415                Case {
16416                    name: "a bare carrier narrowed to targets",
16417                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
16418                    closed: [None, Some(ScopeCarrierRemoved), None, None],
16419                },
16420                Case {
16421                    name: "delegates turned off",
16422                    change: |r| r.attributes.delegates = false,
16423                    closed: [Some(ScopeDelegationChanged); 4],
16424                },
16425                Case {
16426                    name: "agent_id changed",
16427                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
16428                    closed: [Some(ScopeDelegationChanged); 4],
16429                },
16430                Case {
16431                    name: "a carrier added, child owners changed, the record re-sent",
16432                    change: |r| {
16433                        r.carriers.push(carrier(MAGIC, None));
16434                        r.child_owners.push(Principal::Reserved {
16435                            module_id: MAGIC.to_string(),
16436                        });
16437                    },
16438                    closed: [None; 4],
16439                },
16440                Case {
16441                    name: "a target added",
16442                    change: |r| {
16443                        r.carriers[1]
16444                            .targets
16445                            .as_mut()
16446                            .unwrap()
16447                            .push("third".to_string())
16448                    },
16449                    closed: [None; 4],
16450                },
16451                Case {
16452                    name: "delegates turned on",
16453                    change: |r| r.attributes.delegates = true,
16454                    closed: [None; 4],
16455                },
16456            ];
16457            for case in cases {
16458                let mut rig = rig().await;
16459                rig.sync(vec![session(1)]).await;
16460                let scope = || Some(rig_selector("s", Some(1)));
16461                let mut routes = [
16462                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
16463                    rig.bound(Some(AFT), PLEXUS, scope()).await,
16464                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
16465                    rig.bound(Some(BROCA), OTHER, scope()).await,
16466                ];
16467                let mut record = session(1);
16468                (case.change)(&mut record);
16469                rig.sync(vec![record]).await;
16470                for (index, expected) in case.closed.iter().enumerate() {
16471                    let route = &mut routes[index];
16472                    match expected {
16473                        Some(reason) => {
16474                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
16475                            assert_eq!(
16476                                route.closed_reason(),
16477                                *reason,
16478                                "{}: route {index}",
16479                                case.name
16480                            );
16481                        }
16482                        None => {
16483                            assert!(rig.live(route), "{}: route {index} closed", case.name);
16484                            assert!(
16485                                route.untouched(),
16486                                "{}: route {index} was told something",
16487                                case.name
16488                            );
16489                        }
16490                    }
16491                }
16492            }
16493        }
16494
16495        #[tokio::test]
16496        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
16497            // Removed, and replaced by a higher epoch.
16498            for next in [Vec::new(), vec![session(2)]] {
16499                let mut rig = rig().await;
16500                rig.sync(vec![session(1)]).await;
16501                let mut route = rig
16502                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16503                    .await;
16504                rig.sync(next).await;
16505                assert!(!rig.live(&route));
16506                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
16507            }
16508
16509            // A child whose parent ends: its routes close as parent-ended, the
16510            // child stays live, and routes under the parent close as ended.
16511            let mut rig = rig().await;
16512            let mut child = session(1);
16513            child.scope_ref = "child".to_string();
16514            child.kind = ScopeKind::Worker;
16515            child.parent = Some(ScopeParent::new(
16516                Principal::Reserved {
16517                    module_id: OWNER.to_string(),
16518                },
16519                "s".to_string(),
16520                1,
16521            ));
16522            rig.sync(vec![session(1), child.clone()]).await;
16523            let mut child_route = rig
16524                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
16525                .await;
16526            assert_eq!(
16527                child_route.stamp().unwrap().parent_state,
16528                Some(ParentState::Linked)
16529            );
16530            rig.sync(vec![child]).await;
16531            assert!(!rig.live(&child_route));
16532            assert_eq!(
16533                child_route.closed_reason(),
16534                RouteCloseReason::ScopeParentEnded
16535            );
16536        }
16537
16538        #[tokio::test]
16539        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
16540        ) {
16541            let mut rig = rig().await;
16542            rig.sync(vec![session(1)]).await;
16543            let mut route = rig
16544                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16545                .await;
16546            let before = rig.forwarding.published_scope_tag(OWNER, "s");
16547            rig.sync(vec![session(1)]).await;
16548            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16549            assert!(rig.live(&route) && route.untouched());
16550
16551            // A call in flight on the route when another carrier is added. A
16552            // forwarded REQUEST holds one credit on the route's flow until the
16553            // module answers; the router takes it exactly like this.
16554            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
16555                .forwarding
16556                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16557                .unwrap()
16558            else {
16559                panic!("the route is bound");
16560            };
16561            binding.flow.acquire_tagged(9, false).await.unwrap();
16562            let mut widened = session(1);
16563            widened.carriers.push(carrier(MAGIC, None));
16564            rig.sync(vec![widened]).await;
16565            assert!(rig.live(&route) && route.untouched());
16566            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16567            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
16568            // The call's credit is still held on an open flow, so its answer
16569            // will be delivered: closing the route would have closed the flow.
16570            assert_eq!(binding.flow.in_flight(), 1);
16571            binding
16572                .flow
16573                .acquire_tagged(10, false)
16574                .await
16575                .expect("the flow is still open");
16576        }
16577
16578        /// A swap's superseded endpoint keeps its routes until drained; ending
16579        /// the scope closes them there too.
16580        #[tokio::test]
16581        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
16582            let mut rig = rig().await;
16583            rig.sync(vec![session(1)]).await;
16584            let mut on_incumbent = rig
16585                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16586                .await;
16587
16588            // Swap plexus: register a candidate and cut over, leaving the
16589            // incumbent superseded with the route still on it.
16590            let (candidate, _candidate_rx) = wide_ctx(9);
16591            let registration = rig
16592                .handler
16593                .registry
16594                .register_candidate_with_control_ops(
16595                    manifest(PLEXUS, PROTOCOL_VERSION),
16596                    PROTOCOL_VERSION,
16597                    candidate.connection_id,
16598                    module_baseline_control_ops(),
16599                )
16600                .unwrap();
16601            rig.forwarding
16602                .register_candidate_module_connection(
16603                    candidate.connection_id,
16604                    PLEXUS.to_string(),
16605                    PROTOCOL_VERSION,
16606                    manifest_concurrency(&registration.manifest),
16607                    candidate.egress.clone(),
16608                )
16609                .unwrap();
16610            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
16611            rig.handler
16612                .registry
16613                .promote_candidate(PLEXUS)
16614                .unwrap()
16615                .unwrap();
16616            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
16617
16618            rig.sync(Vec::new()).await;
16619            assert!(!rig.live(&on_incumbent));
16620            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
16621            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16622            let goodbye = incumbent_rx
16623                .try_recv()
16624                .expect("the superseded endpoint is told")
16625                .frame;
16626            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16627        }
16628    }
16629}
16630
16631#[cfg(test)]
16632mod concurrency_default_exposure_tests {
16633    use super::*;
16634
16635    fn hello_body(role_json: &str) -> Vec<u8> {
16636        format!(
16637            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
16638        )
16639        .into_bytes()
16640    }
16641
16642    fn manifest_from(body: &[u8]) -> ModuleManifest {
16643        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
16644        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
16645            .expect("manifest parses")
16646    }
16647
16648    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
16649
16650    #[test]
16651    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
16652        let body = hello_body(&format!(
16653            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
16654        ));
16655        let manifest = manifest_from(&body);
16656        // Precondition: serde really resolved it to the default, so the typed
16657        // manifest alone cannot answer the question this probe exists for.
16658        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16659        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
16660    }
16661
16662    #[test]
16663    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
16664        let body = hello_body(&format!(
16665            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
16666        ));
16667        let manifest = manifest_from(&body);
16668        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16669        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16670    }
16671
16672    #[test]
16673    fn non_management_roles_are_never_reported() {
16674        let body = hello_body(
16675            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
16676        );
16677        let manifest = manifest_from(&body);
16678        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16679    }
16680}