Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_event_declarations,
25        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27    },
28    scope::{
29        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31        SCOPE_SYNC_OP,
32    },
33    session::{
34        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37        ROLE_VERSIONS_FIELD,
38    },
39    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40    Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46    caller_identity::CallerIdentity,
47    capability_requirements::{
48        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50        RuntimeModule,
51    },
52    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
53    forwarding::{
54        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
55        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
56        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
57    },
58    observability::{
59        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
60    },
61    privileged_prompt::{prompt_text, PrivilegedOperation},
62    provenance::{
63        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
64    },
65    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
66    router::{RouteCtx, RouterError},
67    scopes::{HelloLaunchNonces, ScopeTable},
68    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
69    stderr_tail::{CaptureState, TailEntry},
70    supervise::{
71        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
72        SupervisorHandle, SwapHelloAdmission,
73    },
74    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
75};
76
77/// Lowest envelope version this subc build will negotiate.
78///
79/// Module HELLO negotiation is exact: peers must use the daemon's locked
80/// protocol version. Older and newer peers receive `version_unsupported` and
81/// are not registered.
82pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
83
84const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
85const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
86const CAP_PING_PONG: &str = "ping_pong_v1";
87const CAP_SESSION_ATTACH: &str = "session_attach_v1";
88const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
89
90const SUBC_CONTROL_OPS: &[&str] = &[
91    ops::ACTIVITY_RECORD,
92    ops::ACTIVITY_QUERY,
93    ops::ACTIVITY_SUBSCRIBE,
94    ops::SERVER_DESCRIBE,
95    ops::CATALOG_LIST,
96    ops::ROUTE_OPEN,
97    ops::ROUTE_POLL,
98    ops::ROUTE_CLOSING,
99    ops::ROUTE_CLOSED,
100    ops::SUPERVISOR_LIST,
101    ops::SUPERVISOR_RESTART,
102    ops::SUPERVISOR_SWAP,
103    ops::SUPERVISOR_RELOAD,
104    ops::SUPERVISOR_RESCAN,
105    ops::SUPERVISOR_RELEASE_RESERVED,
106    ops::SUPERVISOR_SET_ENABLED,
107    ops::SUPERVISOR_HEALTH_PROBE,
108    ops::SUPERVISOR_HEALTH,
109    ops::SUPERVISOR_STDERR_TAIL,
110    ops::SUPERVISOR_TERMINALS,
111    ops::SUPERVISOR_ROUTES,
112    ops::SUPERVISOR_PROVENANCE,
113    ops::SUPERVISOR_SPAWN_SNAPSHOT,
114    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
115];
116
117const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
118    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
119    "supervisor.live_roots",
120    SCOPE_SYNC_OP,
121    SCOPE_APPLY_OP,
122    SCOPE_DESCRIBE_OP,
123    "operator.confirm",
124];
125
126/// Module-originated ops the daemon answers but does not advertise in
127/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
128/// belongs to is incomplete, so no module is told it works before it does.
129const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
130
131const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
132
133/// How long subc waits for a module to ack a relayed route.bind before returning
134/// `module_timeout`. The ack waits on the module's own configure, which for AFT
135/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
136/// and DB-open work — on a cold page cache or a large repo that legitimately
137/// exceeds a couple of seconds. The default is generous because rejecting a VALID
138/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
139/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
140pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
141
142/// How many CONSECUTIVE full-budget relay timeouts against one target module
143/// open that module's bind-relay breaker.
144///
145/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
146/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
147/// so three consecutive full-budget timeouts take ~36s to observe: every client
148/// whose open contributed to opening the breaker had already given up on its
149/// own. That is what makes opening the breaker unable to turn a call that would
150/// have succeeded into a refusal — it can only make an already-failing module
151/// fail faster.
152///
153/// Two would be reachable inside one default deadline. One would convict a
154/// module on a single cold-cache bind, which is exactly the valid-but-slow case
155/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
156pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
157
158/// How long a module's bind-relay breaker stays open before exactly one
159/// `route.open` is let through as a probe.
160///
161/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
162/// re-pays a full-budget stall almost continuously, and the breaker stops being
163/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
164/// deadline: a client that starts retrying after the module recovers has to get
165/// a probe opportunity inside its own deadline, or the breaker converts a
166/// recovered module into a failed call — the failure it exists to prevent,
167/// pointed the other way.
168///
169/// 20s sits between those with room on both sides, and it caps what a wedged
170/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
171/// rather than one per `route.open` per connection. The stall that motivated
172/// this, with its measurements, is written up in
173/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
174/// waited the whole budget out.
175pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
176
177const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
178const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
179
180fn reload_verdict(
181    configured: &Path,
182    spawned_from: Option<&Path>,
183    image: subc_control::RunningImageAgreement,
184) -> PendingReloadVerdict {
185    let path = match spawned_from {
186        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
187            ReloadPathAgreement::Match
188        }
189        Some(spawned_from) => ReloadPathAgreement::Mismatch {
190            configured: configured.to_path_buf(),
191            spawned_from: spawned_from.to_path_buf(),
192        },
193        None => ReloadPathAgreement::Unavailable {
194            reason: if matches!(
195                image,
196                subc_control::RunningImageAgreement::Unavailable {
197                    reason: subc_control::RunningImageUnavailableReason::NotRunning
198                }
199            ) {
200                ReloadPathUnavailableReason::NotRunning
201            } else {
202                ReloadPathUnavailableReason::SpawnedPathUnavailable
203            },
204        },
205    };
206    PendingReloadVerdict {
207        path,
208        image,
209        spec: None,
210    }
211}
212
213fn launch_spec_agreement(
214    configured: &crate::supervise::ModuleSpec,
215    running: bool,
216    spawned: Option<&crate::supervise::LaunchSpecDigest>,
217) -> subc_control::LaunchSpecAgreement {
218    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
219    if !running {
220        return Agreement::Unavailable {
221            reason: Reason::NotRunning,
222        };
223    }
224    match spawned {
225        None => Agreement::Unavailable {
226            reason: Reason::SpawnedSpecUnavailable,
227        },
228        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
229            Agreement::Match
230        }
231        Some(_) => Agreement::Mismatch,
232    }
233}
234
235fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
236    if configured == spawned_from {
237        return true;
238    }
239    #[cfg(windows)]
240    {
241        // Windows spawn records the resolved verbatim path (\\?\...), while
242        // configuration keeps its original spelling. Resolve both with the OS
243        // instead of stripping prefixes or folding case, which could conflate
244        // distinct names in case-sensitive directories. Different resolved
245        // paths still differ even if their files have identical content.
246        match (
247            std::fs::canonicalize(configured),
248            std::fs::canonicalize(spawned_from),
249        ) {
250            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
251            _ => false,
252        }
253    }
254    #[cfg(not(windows))]
255    {
256        false
257    }
258}
259
260#[derive(Clone)]
261struct DaemonProvenanceFacts {
262    build: DaemonBuildProvenance,
263    pid: Option<u32>,
264    started_at_ms: Option<u64>,
265    start_clock: Option<crate::clock::StartClock>,
266    executable_path: Option<PathBuf>,
267    executable_identity: Option<SpawnedFileIdentity>,
268    process_start_time: Option<u64>,
269    probe: ExecutableIdentityProbe,
270}
271
272impl Default for DaemonProvenanceFacts {
273    fn default() -> Self {
274        Self {
275            build: DaemonBuildProvenance {
276                build_git_sha: None,
277                build_lock_digest: None,
278            },
279            pid: None,
280            started_at_ms: None,
281            start_clock: None,
282            executable_path: None,
283            executable_identity: None,
284            process_start_time: None,
285            probe: ExecutableIdentityProbe::default(),
286        }
287    }
288}
289
290#[derive(Debug, Clone)]
291struct SupervisorRescanContext {
292    supervisor: Supervisor,
293    config_path: PathBuf,
294    configured_port: Option<u16>,
295    storage_config: Option<crate::daemon_config::StorageConfig>,
296    admission_facts_carrier_module_id: Option<String>,
297    admission_facts_targets: Option<Vec<String>>,
298    scope_authority_owners: Vec<String>,
299    privileged_operator_prompt: PrivilegedOperatorPrompt,
300}
301
302/// Refusal labels passed to `observe_route_open_refusal` that mean the target
303/// module is not serving right now, and so open or extend an outage in the
304/// route outage tracker. Every one of them is only reachable after the target
305/// was found in the registry, which is what keeps an arbitrary client-chosen
306/// id from ever creating tracker state.
307///
308/// Deliberately absent: `not_registered` and `removed` (the id may be
309/// anything a client sent, and a removed module is gone on purpose),
310/// `protocol_none` (such a module never serves routes, so nothing is out),
311/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
312/// capability and admission-facts refusals (they refuse the caller, not a
313/// module outage), and `relay_reservation_failed` (its code ranges over
314/// capacity limits as well as a vanished connection). Capacity, breaker,
315/// relay-timeout and module-rejection refusals do not pass through that
316/// function at all; the breaker logs its own transitions.
317///
318/// The two not-serving refusals that bypass that function record themselves
319/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
320/// `required_capability_unprovided` is not tracked: the module itself is up,
321/// and the outage belongs to the missing provider.
322const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
323    "reloading",
324    "supervisor_not_live",
325    "registration_not_active",
326    "no_forwarding_connection",
327    "relay_send_failed",
328];
329
330/// Real channel-0 control handler for subc itself.
331#[derive(Clone)]
332pub struct ControlHandler {
333    registry: Arc<Registry>,
334    activity: Option<crate::activity::ActivityLedger>,
335    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
336    forwarding: Arc<ForwardingTable>,
337    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
338    supervisor: SupervisorHandle,
339    subc_capabilities: Arc<[String]>,
340    /// Daemon-wide route.bind relay budget. Used as the fallback when the
341    /// target module has no per-module override in
342    /// `route_bind_relay_timeouts`.
343    route_bind_relay_timeout: Duration,
344    /// Per-module route.bind relay budget overrides, keyed by module id. When
345    /// `handle_route_open` resolves the deadline for a target module, a
346    /// per-module entry wins over the daemon-wide value above.
347    route_bind_relay_timeouts: BTreeMap<String, Duration>,
348    /// Per-target-module bind-relay breaker state. Shared with the forwarding
349    /// table, which is where a new module connection resets it.
350    route_bind_breakers: RouteBindBreakers,
351    /// Live relay admissions keyed by target module. Shared through the
352    /// forwarding table so cloned or separately built handlers enforce one cap.
353    route_bind_concurrency: RouteBindConcurrency,
354    /// Start and end of each module's not-serving period as seen by
355    /// `route.open`, so an outage gets one line at each edge instead of only
356    /// the per-refusal INFO lines. Taken from the forwarding table, so every
357    /// handler built over one table shares it.
358    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
359    /// Consecutive relay timeouts that open a module's breaker.
360    route_bind_breaker_threshold: u32,
361    /// How long a breaker stays open before one probe is admitted.
362    route_bind_breaker_cooldown: Duration,
363    health_probe_timeout: Duration,
364    /// Central storage policy. When set, each registering module receives its
365    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
366    storage_config: Option<crate::daemon_config::StorageConfig>,
367    /// The machine id established at boot, served on every HELLO_ACK and on
368    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
369    /// changes the file, never this value. `None` serves no id.
370    machine_id: Option<crate::machine_id::MachineId>,
371    admission_facts_carrier_module_id: Option<String>,
372    admission_facts_targets: Option<Vec<String>>,
373    /// Scope records with their sync authorities and tombstones; see
374    /// `crate::scopes`. Shared by clones of this handler, so every connection
375    /// reads and writes one table.
376    scopes: Arc<RwLock<ScopeTable>>,
377    /// Wakes the expiry loop when an accepted scope change may have added a
378    /// deadline, so the loop can wait without ticking while no live scope has
379    /// one. A permit is stored when nothing waits, so no wake is lost.
380    scope_deadline_added: Arc<tokio::sync::Notify>,
381    /// The configured `scope_authority_owners`, kept so a rescan can report a
382    /// changed value as needing a daemon restart; rescan never applies it.
383    scope_authority_owners: Vec<String>,
384    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
385    privileged_operator_prompt: PrivilegedOperatorPrompt,
386    /// The local and peer address of each accepted TCP client connection, read
387    /// from the daemon's own socket, so a privileged request can name its caller.
388    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
389    #[cfg(test)]
390    caller_lookup: Option<TestCallerLookup>,
391    /// The launch nonce each module connection presented at HELLO, which is how
392    /// a `scope.sync` is matched to the owner's current launch.
393    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
394    rescan: Option<SupervisorRescanContext>,
395    connected_clients: ConnectedClients,
396    counters: DaemonCounters,
397    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
398    daemon_provenance: DaemonProvenanceFacts,
399    #[cfg(test)]
400    control_dispatch_delay: Option<Duration>,
401    #[cfg(test)]
402    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
403    /// Holds `route.open` after the route.bind is queued to the module and
404    /// before it is marked enqueued, so a test can make the module's reply (or
405    /// a client teardown) land exactly inside that window.
406    #[cfg(test)]
407    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
408    /// Holds a HELLO after its registry entry is reserved and before its
409    /// forwarding endpoint is installed, so a test can read the catalog and
410    /// open routes while the module is registered but not yet routable. A
411    /// blocking barrier because the HELLO handler is synchronous; the test
412    /// runs the HELLO on its own thread.
413    #[cfg(test)]
414    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
415}
416
417impl fmt::Debug for ControlHandler {
418    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
419        f.debug_struct("ControlHandler")
420            .field("registry", &self.registry)
421            .field("forwarding", &self.forwarding)
422            .field("process_liveness", &self.process_liveness.is_some())
423            .field("supervisor", &self.supervisor)
424            .field("subc_capabilities", &self.subc_capabilities)
425            .finish()
426    }
427}
428
429struct RouteOpenRequest {
430    target: RouteTarget,
431    identity: BindIdentity,
432    consumer_identity: Option<ConsumerIdentity>,
433    consumer_capabilities: Option<Vec<String>>,
434    role_versions: Option<BTreeMap<String, String>>,
435    admission_facts: Option<serde_json::Value>,
436    scope: Option<ScopeSelector>,
437}
438
439#[derive(Clone, Copy)]
440enum RouteOpenPhase {
441    Admission,
442    ProjectRoot,
443    EgressReservation,
444    BindSend,
445    BindAckWait,
446}
447
448/// Store elapsed time for each route-open phase, including attempts that end
449/// in an early refusal or backend error. Admission also includes rechecks and
450/// commit bookkeeping performed between the other timed phases.
451struct RouteOpenTimings {
452    module_id: String,
453    principal: String,
454    admission: Duration,
455    project_root: Duration,
456    egress_reservation: Duration,
457    bind_send: Duration,
458    bind_ack_wait: Duration,
459    phase: Option<RouteOpenPhase>,
460    phase_started_at: Instant,
461}
462
463impl RouteOpenTimings {
464    fn new() -> Self {
465        Self {
466            module_id: String::new(),
467            // Refusals before attestation must not claim a verified principal.
468            principal: "unresolved".to_string(),
469            admission: Duration::ZERO,
470            project_root: Duration::ZERO,
471            egress_reservation: Duration::ZERO,
472            bind_send: Duration::ZERO,
473            bind_ack_wait: Duration::ZERO,
474            phase: Some(RouteOpenPhase::Admission),
475            phase_started_at: Instant::now(),
476        }
477    }
478
479    fn finish(&mut self) {
480        let elapsed = self.phase_started_at.elapsed();
481        match self.phase.take() {
482            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
483            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
484            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
485            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
486            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
487            None => {}
488        }
489    }
490
491    fn enter(&mut self, phase: RouteOpenPhase) {
492        self.finish();
493        self.phase = Some(phase);
494        self.phase_started_at = Instant::now();
495    }
496}
497
498struct RouteBindReservationGuard {
499    forwarding: Arc<ForwardingTable>,
500    endpoint: ModuleEndpointId,
501    relay_corr: u64,
502    armed: bool,
503}
504
505struct ModuleControlRpcGuard {
506    forwarding: Arc<ForwardingTable>,
507    endpoint: ModuleEndpointId,
508    corr: u64,
509    armed: bool,
510}
511
512impl ModuleControlRpcGuard {
513    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
514        Self {
515            forwarding,
516            endpoint,
517            corr,
518            armed: true,
519        }
520    }
521
522    fn disarm(&mut self) {
523        self.armed = false;
524    }
525}
526
527impl Drop for ModuleControlRpcGuard {
528    fn drop(&mut self) {
529        if self.armed {
530            let _ = self
531                .forwarding
532                .cancel_module_control_rpc(self.endpoint, self.corr);
533        }
534    }
535}
536
537impl RouteBindReservationGuard {
538    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
539        Self {
540            forwarding,
541            endpoint,
542            relay_corr,
543            armed: true,
544        }
545    }
546
547    fn release_and_disarm(&mut self) {
548        if !self.armed {
549            return;
550        }
551        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
552            self.endpoint,
553            self.relay_corr,
554            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
555        ) {
556            send_goodbye_target_best_effort(
557                &self.forwarding.counters(),
558                &target,
559                "canceled route.bind",
560            );
561        }
562        self.armed = false;
563    }
564
565    fn disarm(&mut self) {
566        self.armed = false;
567    }
568}
569
570impl Drop for RouteBindReservationGuard {
571    fn drop(&mut self) {
572        self.release_and_disarm();
573    }
574}
575
576/// Per-target-module circuit breaker around the `route.bind` relay.
577///
578/// The connection reader is serial per connection, so a module whose `on_bind`
579/// sits on the ack blocks every LATER frame on the connections that call it,
580/// including calls to unrelated modules. This does not make any module's bind
581/// fast; it stops the daemon paying the full budget again and again for a
582/// condition it has already observed.
583///
584/// State is keyed by TARGET MODULE and shared by every connection: a wedged
585/// module wedges everyone, so what one connection learned should protect the
586/// rest.
587///
588/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
589/// relay to that module has actually timed out, and is removed again when a
590/// relay is accepted or the module reconnects, so it cannot grow with traffic
591/// or with modules that behave.
592///
593/// # Why a `std` mutex here is not the head-of-line defect again
594///
595/// Acquisition never awaits. The critical section is a hash lookup plus a few
596/// integer updates, with no I/O and no `.await` inside it, so a reader task
597/// cannot be descheduled behind it the way it can behind
598/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
599/// primitive, held for the same kind of work, as the refusal counter this very
600/// path already increments.
601///
602/// It is also NOT on the data-plane splice path: only `route.open` and module
603/// registration touch it, so bound-route frames gain no state check and no
604/// contention.
605#[derive(Debug, Clone, Default)]
606pub(crate) struct RouteBindBreakers {
607    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
608}
609
610#[derive(Debug, Clone, Default)]
611pub(crate) struct RouteBindConcurrency {
612    modules: Arc<Mutex<HashMap<String, usize>>>,
613}
614
615struct RouteBindConcurrencyGuard {
616    concurrency: RouteBindConcurrency,
617    module_id: String,
618}
619
620impl RouteBindConcurrency {
621    /// Admit without waiting. Waiting here would move the bind stall from the
622    /// module reply to a semaphore and restore reader head-of-line blocking.
623    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
624        let mut modules = self
625            .modules
626            .lock()
627            .expect("route.bind concurrency mutex poisoned");
628        let in_flight = modules.entry(module_id.to_string()).or_default();
629        if *in_flight >= limit {
630            return Err(*in_flight);
631        }
632        *in_flight += 1;
633        Ok(RouteBindConcurrencyGuard {
634            concurrency: self.clone(),
635            module_id: module_id.to_string(),
636        })
637    }
638}
639
640impl Drop for RouteBindConcurrencyGuard {
641    fn drop(&mut self) {
642        let mut modules = self
643            .concurrency
644            .modules
645            .lock()
646            .expect("route.bind concurrency mutex poisoned");
647        let remove = {
648            let in_flight = modules
649                .get_mut(&self.module_id)
650                .expect("admitted route.bind has a concurrency entry");
651            *in_flight -= 1;
652            *in_flight == 0
653        };
654        if remove {
655            modules.remove(&self.module_id);
656        }
657    }
658}
659
660#[derive(Debug, Default)]
661struct ModuleBreakerState {
662    /// Relay timeouts observed with no accepted relay in between.
663    consecutive_timeouts: u32,
664    /// `Some` while the breaker is open: the instant the cooldown expires and
665    /// the next arrival may probe. `None` means closed.
666    cooldown_until: Option<Instant>,
667    /// A half-open probe has been admitted and has not settled yet. This is
668    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
669    /// that read the cooldown, so concurrent opens arriving at the moment the
670    /// cooldown expires cannot all decide that they are the probe.
671    probe_in_flight: Option<Arc<()>>,
672}
673
674/// What the breaker decided for one `route.open`, before any relay work.
675enum RouteBindAdmission<'a> {
676    Admitted {
677        guard: RouteBindBreakerGuard<'a>,
678        /// This open is the single half-open probe, so the transition is worth
679        /// one log line.
680        probe: bool,
681    },
682    Refused {
683        consecutive_timeouts: u32,
684        /// What is left of the cooldown. Zero when the refusal is because the
685        /// one probe is already in flight rather than because the cooldown has
686        /// not elapsed.
687        retry_in: Duration,
688        probe_in_flight: bool,
689    },
690}
691
692/// An outstanding admission, which must be told how its relay settled.
693///
694/// `Drop` settles it as inconclusive, so an early return between admission and
695/// the relay -- or the whole handler being cancelled when the client
696/// disconnects -- releases a half-open probe slot instead of leaving the
697/// breaker wedged half-open with no further probes.
698struct RouteBindBreakerGuard<'a> {
699    breakers: RouteBindBreakers,
700    module_id: &'a str,
701    probe_token: Option<Arc<()>>,
702    settled: bool,
703}
704
705impl RouteBindBreakerGuard<'_> {
706    /// The module answered within the budget and took the bind. THE ONLY
707    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
708    /// breaker, which is a transition worth logging.
709    fn record_accepted(&mut self) -> bool {
710        self.settled = true;
711        self.breakers.record_accepted(self.module_id)
712    }
713
714    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
715    /// COUNTS TOWARD OPENING.
716    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
717        self.settled = true;
718        self.breakers.record_timeout(
719            self.module_id,
720            self.probe_token.as_ref(),
721            threshold,
722            cooldown,
723        )
724    }
725
726    /// Everything else: the module REJECTED the bind, its connection went away
727    /// mid-relay, or the waiter was cancelled.
728    ///
729    /// None of these is evidence that a module is slow, and each already has
730    /// its own refusal with its own code. A module that rejects a bind in
731    /// microseconds is healthy and must never be convicted for it; a module
732    /// that died has said nothing about the module that replaces it. So these
733    /// neither increment nor reset the count -- they only release a probe slot.
734    fn record_inconclusive(&mut self) {
735        self.settled = true;
736        self.breakers
737            .record_inconclusive(self.module_id, self.probe_token.as_ref());
738    }
739}
740
741impl Drop for RouteBindBreakerGuard<'_> {
742    fn drop(&mut self) {
743        if !self.settled {
744            self.breakers
745                .record_inconclusive(self.module_id, self.probe_token.as_ref());
746        }
747    }
748}
749
750/// The breaker moved to open, reported so the caller can log it outside the
751/// lock. Opening is rare and load-bearing; the refusals that follow are
752/// frequent and are counted rather than logged.
753struct BreakerOpened {
754    consecutive_timeouts: u32,
755    /// True when a failed probe re-opened an already-open breaker, which reads
756    /// very differently in a log from a first opening.
757    reopened_after_probe: bool,
758}
759
760impl RouteBindBreakers {
761    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
762        self.modules
763            .lock()
764            .expect("route.bind breaker mutex poisoned")
765    }
766
767    /// Decide whether this `route.open` may attempt its relay. Takes the map
768    /// lock and nothing else, and never awaits.
769    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
770        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
771            probe: probe_token.is_some(),
772            guard: RouteBindBreakerGuard {
773                breakers: self.clone(),
774                module_id,
775                probe_token,
776                settled: false,
777            },
778        };
779
780        let mut modules = self.lock();
781        let Some(state) = modules.get_mut(module_id) else {
782            return admitted(None);
783        };
784        let Some(cooldown_until) = state.cooldown_until else {
785            return admitted(None);
786        };
787        if state.probe_in_flight.is_some() {
788            return RouteBindAdmission::Refused {
789                consecutive_timeouts: state.consecutive_timeouts,
790                retry_in: Duration::ZERO,
791                probe_in_flight: true,
792            };
793        }
794        let now = Instant::now();
795        if now < cooldown_until {
796            return RouteBindAdmission::Refused {
797                consecutive_timeouts: state.consecutive_timeouts,
798                retry_in: cooldown_until - now,
799                probe_in_flight: false,
800            };
801        }
802        let token = Arc::new(());
803        state.probe_in_flight = Some(Arc::clone(&token));
804        admitted(Some(token))
805    }
806
807    fn record_accepted(&self, module_id: &str) -> bool {
808        self.lock()
809            .remove(module_id)
810            .is_some_and(|state| state.cooldown_until.is_some())
811    }
812
813    fn record_timeout(
814        &self,
815        module_id: &str,
816        probe_token: Option<&Arc<()>>,
817        threshold: u32,
818        cooldown: Duration,
819    ) -> Option<BreakerOpened> {
820        let mut modules = self.lock();
821        let state = modules.entry(module_id.to_string()).or_default();
822        let was_open = state.cooldown_until.is_some();
823        let was_probe = Self::owns_probe(state, probe_token);
824        if was_probe {
825            state.probe_in_flight = None;
826        }
827        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
828        if state.consecutive_timeouts < threshold {
829            return None;
830        }
831        state.cooldown_until = Some(Instant::now() + cooldown);
832        Some(BreakerOpened {
833            consecutive_timeouts: state.consecutive_timeouts,
834            reopened_after_probe: was_open && was_probe,
835        })
836    }
837
838    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
839        // A relay can finish after the breaker was reset or after it opened
840        // again and started a new probe. Only the guard whose token matches the
841        // active probe may release it, so a late relay never frees a newer probe.
842        state
843            .probe_in_flight
844            .as_ref()
845            .zip(token)
846            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
847    }
848
849    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
850        if let Some(state) = self.lock().get_mut(module_id) {
851            if Self::owns_probe(state, probe_token) {
852                state.probe_in_flight = None;
853            }
854        }
855    }
856
857    /// Discard what was learned about a module, because the process it was
858    /// learned about is gone. Returns the discarded count when it was non-zero.
859    ///
860    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
861    /// `module_id` is a configuration identity that outlives any particular
862    /// child; what the breaker observed was the process behind the module
863    /// connection of the moment. When a new connection registers under that id
864    /// the verdict's subject no longer exists, so the verdict is stale by
865    /// construction rather than merely likely to be wrong. Keeping it would
866    /// apply a dead process's record to a live one, which is the same defect
867    /// class this breaker exists to stop the daemon committing.
868    ///
869    /// A half-open probe in flight is discarded with the rest: it was a
870    /// question about the old process.
871    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
872        self.lock()
873            .remove(module_id)
874            .map(|state| state.consecutive_timeouts)
875            .filter(|discarded| *discarded > 0)
876    }
877
878    /// Open breakers, for the `server.describe` counters object. `None` when
879    /// none is open, so the key stays absent rather than present-and-empty.
880    ///
881    /// This is the operator's answer to "is this module refusing instantly or
882    /// is it fine?", which look identical from a client that retries and then
883    /// succeeds.
884    fn open_snapshot(&self) -> Option<serde_json::Value> {
885        let now = Instant::now();
886        let modules = self.lock();
887        let open = modules
888            .iter()
889            .filter_map(|(module_id, state)| {
890                let cooldown_until = state.cooldown_until?;
891                Some((
892                    module_id.clone(),
893                    serde_json::json!({
894                        "consecutive_timeouts": state.consecutive_timeouts,
895                        "cooldown_remaining_ms":
896                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
897                        "probe_in_flight": state.probe_in_flight.is_some(),
898                    }),
899                ))
900            })
901            .collect::<serde_json::Map<String, serde_json::Value>>();
902        (!open.is_empty()).then_some(serde_json::Value::Object(open))
903    }
904}
905
906impl ControlHandler {
907    pub fn new(registry: Arc<Registry>) -> Self {
908        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
909    }
910
911    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
912        let counters = forwarding.counters();
913        // Taken from the forwarding table rather than created here, so that the
914        // breaker a `route.open` consults is the same one a module's
915        // registration resets, however many handlers are built over one table.
916        let route_bind_breakers = forwarding.route_bind_breakers();
917        let route_bind_concurrency = forwarding.route_bind_concurrency();
918        let route_outages = forwarding.route_outages();
919        Self {
920            registry,
921            activity: None,
922            activity_streams: Arc::new(Mutex::new(HashMap::new())),
923            forwarding,
924            process_liveness: None,
925            supervisor: SupervisorHandle::new(),
926            subc_capabilities: Arc::from([
927                CAP_MANIFEST_REGISTRATION.to_string(),
928                CAP_CHANNEL_LIFECYCLE.to_string(),
929                CAP_PING_PONG.to_string(),
930                CAP_SESSION_ATTACH.to_string(),
931                CAP_ADMISSION_FACTS_RELAY.to_string(),
932                CAP_SCOPES_V1.to_string(),
933                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
934                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
935                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
936                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
937                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
938                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
939            ]),
940            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
941            route_bind_relay_timeouts: BTreeMap::new(),
942            route_bind_breakers,
943            route_bind_concurrency,
944            route_outages,
945            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
946            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
947            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
948            storage_config: None,
949            machine_id: None,
950            admission_facts_carrier_module_id: None,
951            admission_facts_targets: None,
952            scopes: Arc::new(RwLock::new(ScopeTable::new(
953                crate::daemon_config::default_scope_authority_owners(),
954            ))),
955            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
956            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
957            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
958            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
959            #[cfg(test)]
960            caller_lookup: None,
961            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
962            rescan: None,
963            connected_clients: ConnectedClients::new(),
964            counters,
965            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
966            daemon_provenance: DaemonProvenanceFacts::default(),
967            #[cfg(test)]
968            control_dispatch_delay: None,
969            #[cfg(test)]
970            provenance_probe_override: None,
971            #[cfg(test)]
972            route_bind_relay_sent_hold: None,
973            #[cfg(test)]
974            hello_registry_to_forwarding_hold: None,
975        }
976    }
977
978    /// Set the central storage policy: registering modules then receive their
979    /// resolved storage descriptor in HELLO_ACK.
980    pub fn with_storage_config(
981        mut self,
982        storage_config: Option<crate::daemon_config::StorageConfig>,
983    ) -> Self {
984        self.storage_config = storage_config;
985        self
986    }
987
988    /// Open the ledger on its own writer thread; an unavailable sink does not
989    /// prevent the daemon from serving routing or supervisor requests.
990    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
991        self.activity = Some(crate::activity::ActivityLedger::open(
992            run_dir.join("activity.jsonl"),
993            incarnation,
994        ));
995        self.supervisor.set_activity(self.activity.clone());
996        self
997    }
998
999    /// Set the machine id served to every registering module (HELLO_ACK) and on
1000    /// `server.describe`.
1001    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1002        self.machine_id = machine_id;
1003        self
1004    }
1005
1006    /// Configure the exact reserved module and target ids permitted to relay
1007    /// opaque admission facts. Config-file loading validates this authority;
1008    /// this builder keeps the same policy available to embedded test daemons.
1009    pub fn with_admission_facts_config(
1010        mut self,
1011        carrier_module_id: Option<String>,
1012        targets: Option<Vec<String>>,
1013    ) -> Self {
1014        self.admission_facts_carrier_module_id = carrier_module_id;
1015        self.admission_facts_targets = targets;
1016        self
1017    }
1018
1019    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1020    /// Replaces the scope table with an empty one under the new list, so call it
1021    /// while building the handler, before any module can sync.
1022    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1023        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1024        self.scope_authority_owners = owners;
1025        self
1026    }
1027
1028    /// Set the startup policy before installing the rescan context or serving requests.
1029    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1030        self.privileged_operator_prompt = mode;
1031        self
1032    }
1033
1034    /// Override the route.bind relay timeout. Used by tests that assert the
1035    /// timeout path so they don't block on the production-safe default.
1036    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1037        self.route_bind_relay_timeout = timeout;
1038        self
1039    }
1040
1041    /// Install per-module route.bind relay budget overrides. A module id
1042    /// listed here wins over the daemon-wide default set via
1043    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1044    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1045    /// the same `Duration` the bind path will use.
1046    pub fn with_route_bind_relay_timeouts(
1047        mut self,
1048        timeouts: impl IntoIterator<Item = (String, Duration)>,
1049    ) -> Self {
1050        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1051        self
1052    }
1053
1054    /// Resolve the route.bind relay budget for a specific target module id.
1055    /// Per-module overrides win; the daemon-wide value (set via
1056    /// `with_route_bind_relay_timeout` or the built-in default) is the
1057    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1058    /// the same resolution `handle_route_open` will use.
1059    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1060        self.route_bind_relay_timeouts
1061            .get(module_id)
1062            .copied()
1063            .unwrap_or(self.route_bind_relay_timeout)
1064    }
1065
1066    /// Override the per-module bind-relay breaker policy.
1067    ///
1068    /// Used by tests, which cannot spend three production budgets opening a
1069    /// breaker or twenty seconds waiting for its cooldown. The production
1070    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1071    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1072    /// reasoning for the numbers.
1073    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1074        self.route_bind_breaker_threshold = threshold.max(1);
1075        self.route_bind_breaker_cooldown = cooldown;
1076        self
1077    }
1078
1079    #[cfg(test)]
1080    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1081        self.health_probe_timeout = timeout;
1082        self
1083    }
1084
1085    #[cfg(test)]
1086    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1087        self.control_dispatch_delay = Some(delay);
1088        self
1089    }
1090
1091    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1092    /// announce it is parked, once to be released. Build the barrier for two
1093    /// parties, the test being the other one.
1094    #[cfg(test)]
1095    pub(crate) fn with_route_bind_relay_sent_hold(
1096        mut self,
1097        hold: Arc<tokio::sync::Barrier>,
1098    ) -> Self {
1099        self.route_bind_relay_sent_hold = Some(hold);
1100        self
1101    }
1102
1103    /// A HELLO waits on `hold` twice between reserving its registry entry and
1104    /// installing its forwarding endpoint: once to announce it is parked, once
1105    /// to be released. Build the barrier for two parties, the test being the
1106    /// other one, and run the HELLO on a thread of its own.
1107    #[cfg(test)]
1108    pub(crate) fn with_hello_registry_to_forwarding_hold(
1109        mut self,
1110        hold: Arc<std::sync::Barrier>,
1111    ) -> Self {
1112        self.hello_registry_to_forwarding_hold = Some(hold);
1113        self
1114    }
1115
1116    #[cfg(test)]
1117    fn hold_between_registry_and_forwarding(&self) {
1118        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1119            hold.wait();
1120            hold.wait();
1121        }
1122    }
1123
1124    pub fn with_process_liveness(
1125        mut self,
1126        process_liveness: Arc<dyn ModuleProcessLiveness>,
1127    ) -> Self {
1128        self.process_liveness = Some(process_liveness);
1129        self
1130    }
1131
1132    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1133        self.supervisor = supervisor;
1134        self.supervisor.set_activity(self.activity.clone());
1135        self
1136    }
1137
1138    pub fn with_daemon_provenance(
1139        mut self,
1140        pid: u32,
1141        started_at_ms: u64,
1142        executable_path: Option<PathBuf>,
1143        build_git_sha: Option<String>,
1144        build_lock_digest: Option<String>,
1145    ) -> Self {
1146        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1147        let process_start_time = process_start_time(pid);
1148        self.daemon_provenance = DaemonProvenanceFacts {
1149            build: DaemonBuildProvenance {
1150                build_git_sha,
1151                build_lock_digest,
1152            },
1153            pid: Some(pid),
1154            started_at_ms: Some(started_at_ms),
1155            start_clock: None,
1156            executable_path,
1157            executable_identity,
1158            process_start_time,
1159            probe: ExecutableIdentityProbe::default(),
1160        };
1161        self
1162    }
1163
1164    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1165        self.daemon_provenance.start_clock = Some(clock);
1166        self
1167    }
1168
1169    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1170    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1171        self.scopes
1172            .write()
1173            .unwrap_or_else(|poisoned| poisoned.into_inner())
1174            .set_wall_clock(Arc::new(clock));
1175        self
1176    }
1177
1178    #[cfg(test)]
1179    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1180        self.provenance_probe_override = Some(result);
1181        self
1182    }
1183
1184    /// Install the configured module set and its reserved capability bindings.
1185    /// Bindings are configuration-scoped and may point at a provider that has not
1186    /// been installed yet, so this does not require the bound module to exist.
1187    pub fn with_capability_config(
1188        self,
1189        modules: impl IntoIterator<Item = (String, bool)>,
1190        reserved_capabilities: BTreeMap<String, String>,
1191    ) -> Self {
1192        self.capability_evaluator
1193            .configure(modules, reserved_capabilities);
1194        self
1195    }
1196
1197    pub fn with_supervisor_rescan(
1198        mut self,
1199        supervisor: Supervisor,
1200        config_path: impl Into<PathBuf>,
1201        configured_port: Option<u16>,
1202    ) -> Self {
1203        self.rescan = Some(SupervisorRescanContext {
1204            supervisor,
1205            config_path: config_path.into(),
1206            configured_port,
1207            storage_config: self.storage_config.clone(),
1208            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1209            admission_facts_targets: self.admission_facts_targets.clone(),
1210            scope_authority_owners: self.scope_authority_owners.clone(),
1211            privileged_operator_prompt: self.privileged_operator_prompt,
1212        });
1213        self
1214    }
1215
1216    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1217        self.connected_clients = connected_clients;
1218        self
1219    }
1220
1221    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1222        Arc::clone(&self.forwarding)
1223    }
1224
1225    pub(crate) fn counters(&self) -> DaemonCounters {
1226        self.counters.clone()
1227    }
1228
1229    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1230    /// requirement event without depending on an operator polling a status command.
1231    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1232        tokio::spawn(async move {
1233            loop {
1234                self.capability_evaluator
1235                    .wait_for_change_or_deadline()
1236                    .await;
1237                self.refresh_capability_requirements();
1238            }
1239        });
1240    }
1241
1242    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1243    /// deadline passed during sleep is enforced on the first tick after wake.
1244    /// While no live scope carries a deadline the loop waits for an accepted
1245    /// scope change instead of ticking, so a daemon without deadlines never
1246    /// wakes for expiry.
1247    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1248        tokio::spawn(async move {
1249            loop {
1250                let has_deadlines = self
1251                    .scopes
1252                    .read()
1253                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1254                    .has_deadlines();
1255                if !has_deadlines {
1256                    self.scope_deadline_added.notified().await;
1257                    continue;
1258                }
1259                tokio::time::sleep(Duration::from_secs(1)).await;
1260                if let Err(error) = self.sweep_expired_scopes() {
1261                    warn!(%error, "scope expiry sweep failed");
1262                }
1263            }
1264        });
1265    }
1266
1267    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1268        let mut table = self
1269            .scopes
1270            .write()
1271            .unwrap_or_else(|poisoned| poisoned.into_inner());
1272        let swept = table.sweep_expired();
1273        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1274        drop(table);
1275        self.close_scope_drained_routes(drained);
1276        Ok(())
1277    }
1278
1279    fn publish_scope_changes(
1280        &self,
1281        changes: &[crate::scopes::ScopeTagChange],
1282        expired: &[crate::scopes::ScopeExpired],
1283    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1284        if changes.is_empty() {
1285            return Ok(Vec::new());
1286        }
1287        let drained = self
1288            .forwarding
1289            .publish_scope_changes(changes)
1290            .map_err(RouterError::Forwarding)?;
1291        let mut counts = HashMap::new();
1292        for route in &drained {
1293            *counts
1294                .entry((
1295                    route.scope.owner.as_str(),
1296                    route.scope.scope_ref.as_str(),
1297                    route.scope.tag.scope_epoch,
1298                ))
1299                .or_insert(0usize) += 1;
1300        }
1301        for scope in expired {
1302            let routes_closed = counts
1303                .get(&(
1304                    scope.owner.as_str(),
1305                    scope.scope_ref.as_str(),
1306                    scope.scope_epoch,
1307                ))
1308                .copied()
1309                .unwrap_or(0);
1310            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1311                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1312                routes_closed, "scope expired");
1313        }
1314        Ok(drained)
1315    }
1316
1317    fn runtime_capability_snapshot(
1318        &self,
1319    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1320        let runtime = self
1321            .supervisor
1322            .list()
1323            .into_iter()
1324            .map(|module| {
1325                let status = module.status().map_err(|err| {
1326                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1327                })?;
1328                Ok(RuntimeModule {
1329                    module_id: status.module_id,
1330                    state: status.state,
1331                    enabled: status.enabled,
1332                })
1333            })
1334            .collect::<Result<Vec<_>, RouterError>>()?;
1335        // Include every registration, published or not. The capability
1336        // evaluator counts a module as providing its capabilities as soon as it
1337        // has registered, even before `catalog.list` shows it, so a HELLO in
1338        // progress does not change other modules' required-capability
1339        // verdicts. See `Registry::list_registered_modules`.
1340        let registrations = self.registry.list_registered_modules().map_err(|err| {
1341            RouterError::backend(
1342                0,
1343                0,
1344                format!("failed to list capability registrations: {err}"),
1345            )
1346        })?;
1347        let registrations = registrations
1348            .into_iter()
1349            .map(|registration| RegisteredModule {
1350                module_id: registration.manifest.module_id,
1351                module_version: registration.manifest.module_version,
1352                capabilities: registration.manifest.capabilities,
1353            })
1354            .collect();
1355        Ok((runtime, registrations))
1356    }
1357
1358    /// The capability side effects of a module becoming the active registration
1359    /// for its id: cache its manifest (warning if its claims drifted), run the
1360    /// deny census when its declarations call for one, and recompute the
1361    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1362    /// candidate's does not, and the supervisor does it at promotion instead,
1363    /// through [`crate::supervise::SwapPromotionObserver`].
1364    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1365        let cached_registration = RegisteredModule {
1366            module_id: registration.manifest.module_id.clone(),
1367            module_version: registration.manifest.module_version.clone(),
1368            capabilities: registration.manifest.capabilities.clone(),
1369        };
1370        if self.capability_evaluator.record_hello(&cached_registration) {
1371            warn!(
1372                module_id = %cached_registration.module_id,
1373                "capability claims drifted from the cached manifest"
1374            );
1375        }
1376        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1377            self.enforce_capability_denies();
1378        }
1379        self.refresh_capability_requirements();
1380    }
1381
1382    /// Point the shared supervisor handle at this handler for swap promotions.
1383    /// Called wherever a handler is put behind the `Arc` the router serves, so
1384    /// it can be held weakly.
1385    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1386        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1387            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1388        self.supervisor.set_swap_promotion_observer(observer);
1389    }
1390
1391    pub fn refresh_capability_requirements(&self) {
1392        match self.runtime_capability_snapshot() {
1393            Ok((runtime, registrations)) => {
1394                log_requirement_events(
1395                    self.capability_evaluator
1396                        .evaluate_now(&runtime, &registrations),
1397                );
1398            }
1399            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1400        }
1401    }
1402
1403    /// Reconcile only live, attested route bindings after a capability deny edge
1404    /// or target claim was added. This is deliberately a control-plane census:
1405    /// the opaque forwarding hot path must not grow a per-frame capability check.
1406    fn enforce_capability_denies(&self) {
1407        let registrations = match self.registry.list_registered_modules() {
1408            Ok(snapshot) => snapshot,
1409            Err(err) => {
1410                warn!(error = %err, "failed to read registrations for capability deny census");
1411                return;
1412            }
1413        };
1414        let manifests = registrations
1415            .into_iter()
1416            .map(|registration| {
1417                (
1418                    registration.manifest.module_id.clone(),
1419                    registration.manifest,
1420                )
1421            })
1422            .collect::<BTreeMap<_, _>>();
1423        let census = match self.forwarding.route_census(None) {
1424            Ok(census) => census,
1425            Err(err) => {
1426                warn!(error = %err, "failed to read route census for capability deny enforcement");
1427                return;
1428            }
1429        };
1430
1431        for (target_module_id, routes) in census {
1432            let Some(target_manifest) = manifests.get(&target_module_id) else {
1433                continue;
1434            };
1435            let mut closed_routes = Vec::new();
1436            let mut module_goodbyes = Vec::new();
1437            for route in routes {
1438                let Principal::Reserved {
1439                    module_id: opening_module_id,
1440                } = &route.principal
1441                else {
1442                    continue;
1443                };
1444                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1445                    continue;
1446                };
1447                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1448                    continue;
1449                };
1450
1451                match self.forwarding.release_client_route(
1452                    route.goodbye_target.connection_id,
1453                    route.goodbye_target.channel,
1454                    route.goodbye_target.epoch,
1455                ) {
1456                    Ok(RouteRelease::Removed(module_goodbye)) => {
1457                        warn!(
1458                            opening_module_id,
1459                            target_module_id,
1460                            capability,
1461                            "force-closing route because an attested capability deny edge now matches"
1462                        );
1463                        closed_routes.push(route);
1464                        module_goodbyes.push(module_goodbye);
1465                    }
1466                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1467                    Err(err) => warn!(
1468                        opening_module_id,
1469                        target_module_id,
1470                        capability,
1471                        error = %err,
1472                        "failed to force-close capability-denied route"
1473                    ),
1474                }
1475            }
1476
1477            if closed_routes.is_empty() {
1478                continue;
1479            }
1480            send_route_control_pushes(
1481                &self.forwarding,
1482                closed_routes,
1483                ClientControlPush::RouteClosed {
1484                    module_id: target_module_id,
1485                    channels: Vec::new(),
1486                    reason: RouteCloseReason::CapabilityDenied,
1487                    drained: false,
1488                    abandoned: 0,
1489                    excluded_subscriptions: 0,
1490                    terminal: Some(false),
1491                },
1492            );
1493            self.emit_route_goodbyes(module_goodbyes);
1494        }
1495    }
1496
1497    /// Why a registered module is not accepting new route binds, or `None` when
1498    /// it is. This is the module's effective readiness: its declared readiness
1499    /// first, then every `need: required` capability it declares evaluating to
1500    /// `provided`. `route.open` and `catalog.list` both read it here so the
1501    /// catalog never reports a module routable that `route.open` would refuse.
1502    fn not_ready_reason(
1503        &self,
1504        registration: &crate::registry::ModuleRegistration,
1505    ) -> Option<NotReadyReason> {
1506        if !registration.ready {
1507            return Some(NotReadyReason {
1508                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1509                capability: None,
1510            });
1511        }
1512        self.first_unprovided_required_capability(registration)
1513            .map(|capability| NotReadyReason {
1514                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1515                capability: Some(capability),
1516            })
1517    }
1518
1519    /// The lexicographically first capability this registration declares
1520    /// `need: required` whose evaluator verdict is not `provided`.
1521    ///
1522    /// The verdicts are the capability evaluator's own; nothing here decides
1523    /// what "provided" means. The evaluator counts a capability provided as
1524    /// soon as a module claiming it has REGISTERED, not once that module is
1525    /// ready. That distinction is what keeps two modules that require each
1526    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1527    /// is ready", each would wait for the other to become ready first and
1528    /// neither ever would. Do not tighten it to readiness.
1529    ///
1530    /// A required capability with no verdict at all means this registration's
1531    /// HELLO or catalog.update landed after the last recompute; recompute once
1532    /// rather than let a missing verdict read as either answer. If it is still
1533    /// missing (the recompute itself failed) the capability counts as
1534    /// unprovided: the refusal is retryable, and routing a module whose
1535    /// required provider is unknown is the outcome this check exists to stop.
1536    fn first_unprovided_required_capability(
1537        &self,
1538        registration: &crate::registry::ModuleRegistration,
1539    ) -> Option<String> {
1540        let required = registration
1541            .manifest
1542            .capabilities
1543            .iter()
1544            .flat_map(|declarations| declarations.requires.iter())
1545            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1546            .map(|requirement| requirement.capability.as_str())
1547            .collect::<BTreeSet<_>>();
1548        if required.is_empty() {
1549            return None;
1550        }
1551        let module_id = registration.manifest.module_id.as_str();
1552        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1553        if required
1554            .iter()
1555            .any(|capability| verdict(capability).is_none())
1556        {
1557            self.refresh_capability_requirements();
1558        }
1559        required
1560            .into_iter()
1561            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1562            .map(str::to_string)
1563    }
1564
1565    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1566        self.capability_evaluator
1567            .statuses()
1568            .into_iter()
1569            .map(capability_requirement_status)
1570            .collect()
1571    }
1572
1573    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1574    /// registration-release watch. The signal is what the supervisor waits on
1575    /// before spawning a replacement, so it must only fire once forwarding
1576    /// teardown is also done (see [`Self::cleanup_connection`] /
1577    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1578    /// admitted but its forwarding endpoint could not be installed.
1579    fn deregister_connection(
1580        &self,
1581        connection_id: ConnectionId,
1582        reason: RegistrationEndReason,
1583    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1584        self.registry
1585            .deregister_connection_with_reason(connection_id, reason)
1586    }
1587
1588    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1589        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1590            return None;
1591        }
1592        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1593            parse_client_control_request(&frame.body)
1594        else {
1595            return None;
1596        };
1597        Some(target_module_id(&target).to_string())
1598    }
1599
1600    pub(crate) fn route_open_capacity_refusal(
1601        &self,
1602        ctx: &RouteCtx,
1603        frame: &Frame,
1604        target_module_id: &str,
1605        in_flight: usize,
1606        limit: usize,
1607    ) -> Result<Frame, RouterError> {
1608        self.route_open_admission_refusal_frame(
1609            ctx,
1610            frame,
1611            target_module_id,
1612            "open_admission_full",
1613            (in_flight, limit),
1614            format!(
1615                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1616            ),
1617        )
1618    }
1619
1620    fn route_open_target_capacity_refusal(
1621        &self,
1622        ctx: &RouteCtx,
1623        frame: &Frame,
1624        target_module_id: &str,
1625        in_flight: usize,
1626    ) -> Result<Frame, RouterError> {
1627        self.route_open_admission_refusal_frame(
1628            ctx,
1629            frame,
1630            target_module_id,
1631            "target_binds_full",
1632            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1633            format!(
1634                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1635            ),
1636        )
1637    }
1638
1639    /// Admission pressure clears as existing binds settle, so its refusal must
1640    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1641    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1642    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1643    /// cannot currently reach its target; `module_timeout` would falsely claim
1644    /// that a wait expired. A new, cleaner code would be terminal to deployed
1645    /// clients, so it requires a client-tolerance rollout before daemon emission.
1646    fn route_open_admission_refusal_frame(
1647        &self,
1648        ctx: &RouteCtx,
1649        frame: &Frame,
1650        target_module_id: &str,
1651        reason: &'static str,
1652        (in_flight, limit): (usize, usize),
1653        message: impl Into<String>,
1654    ) -> Result<Frame, RouterError> {
1655        let code = error_codes::TARGET_UNAVAILABLE;
1656        self.counters.increment_route_open_refused(code);
1657        info!(
1658            target: "control",
1659            code,
1660            reason,
1661            module_id = ?target_module_id,
1662            connection_id = ctx.connection_id.get(),
1663            in_flight,
1664            limit,
1665            "route.open refused"
1666        );
1667        control_error_frame(frame, code, message.into())
1668    }
1669
1670    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1671    ///
1672    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1673    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1674    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1675    #[cfg(test)]
1676    pub fn handle_control(
1677        &self,
1678        connection_id: ConnectionId,
1679        frame: Frame,
1680    ) -> Result<Vec<Frame>, RouterError> {
1681        match frame.header.ty {
1682            FrameType::Ping => Ok(vec![pong(&frame)?]),
1683            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1684            FrameType::Goodbye => self.handle_goodbye(connection_id),
1685            ty => Ok(vec![control_error_frame(
1686                &frame,
1687                "unsupported_control_frame",
1688                format!("unsupported channel-0 frame {ty:?}"),
1689            )?]),
1690        }
1691    }
1692
1693    pub async fn handle_control_frame(
1694        &self,
1695        ctx: &RouteCtx,
1696        frame: Frame,
1697    ) -> Result<Vec<Frame>, RouterError> {
1698        self.handle_control_frame_timed(ctx, frame, None).await
1699    }
1700
1701    pub(crate) fn record_caller_endpoints(
1702        &self,
1703        connection: ConnectionId,
1704        endpoints: Option<(SocketAddr, SocketAddr)>,
1705    ) {
1706        let mut callers = self
1707            .caller_endpoints
1708            .lock()
1709            .unwrap_or_else(|p| p.into_inner());
1710        callers.remove(&connection);
1711        if let Some(endpoints) = endpoints {
1712            callers.insert(connection, endpoints);
1713        }
1714    }
1715
1716    #[cfg(test)]
1717    pub(crate) fn with_caller_lookup_for_test(
1718        mut self,
1719        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1720    ) -> Self {
1721        self.caller_lookup = Some(Arc::new(lookup));
1722        self
1723    }
1724
1725    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1726        let endpoints = self
1727            .caller_endpoints
1728            .lock()
1729            .unwrap_or_else(|p| p.into_inner())
1730            .get(&connection)
1731            .copied();
1732        #[cfg(test)]
1733        let lookup = self.caller_lookup.clone();
1734        tokio::task::spawn_blocking(move || {
1735            #[cfg(test)]
1736            if let Some(lookup) = lookup {
1737                return lookup(endpoints);
1738            }
1739            endpoints
1740                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1741                .unwrap_or_default()
1742        })
1743        .await
1744        .unwrap_or_default()
1745    }
1746
1747    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1748        frame.header.channel == 0
1749            && frame.header.ty == FrameType::Request
1750            && self
1751                .forwarding
1752                .module_endpoint_for_connection(ctx.connection_id)
1753                .is_ok_and(|endpoint| endpoint.is_none())
1754            && parse_client_control_request(&frame.body)
1755                .ok()
1756                .is_some_and(|request| privileged_operation(&request).is_some())
1757    }
1758
1759    pub(crate) async fn handle_control_frame_timed(
1760        &self,
1761        ctx: &RouteCtx,
1762        frame: Frame,
1763        dispatch_started_at: Option<StdInstant>,
1764    ) -> Result<Vec<Frame>, RouterError> {
1765        match frame.header.ty {
1766            FrameType::Ping => Ok(vec![pong(&frame)?]),
1767            FrameType::Hello => {
1768                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1769            }
1770            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1771            FrameType::Cancel => {
1772                // A Cancel on channel 0 names a waiting operator.confirm or a
1773                // subscription. Successful cancellation has no ordinary reply.
1774                if self
1775                    .forwarding
1776                    .operator_confirms()
1777                    .cancel(ctx.connection_id, frame.header.corr)
1778                    || self
1779                        .supervisor
1780                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1781                    || self.cancel_activity_subscription(ctx, &frame)?
1782                {
1783                    Ok(Vec::new())
1784                } else {
1785                    Ok(vec![control_error_frame(
1786                        &frame,
1787                        "unknown_subscription",
1788                        "no subscription has this correlation id",
1789                    )?])
1790                }
1791            }
1792            FrameType::Request => {
1793                // This additive operation is not part of the existing exhaustive
1794                // module-control enum. Probe the op before decoding that enum.
1795                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1796                if op
1797                    .as_ref()
1798                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1799                {
1800                    return self.handle_activity_record(ctx, frame).await;
1801                }
1802                if op.as_ref().is_some_and(|probe| {
1803                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1804                }) {
1805                    return self.handle_activity_read(ctx, frame).await;
1806                }
1807                if op
1808                    .as_ref()
1809                    .is_some_and(|probe| probe.op == "operator.confirm")
1810                {
1811                    return self.handle_operator_confirm(ctx, frame);
1812                }
1813                if self
1814                    .forwarding
1815                    .module_endpoint_for_connection(ctx.connection_id)
1816                    .map_err(RouterError::Forwarding)?
1817                    .is_some()
1818                {
1819                    if !is_known_module_request_op(&frame.body) {
1820                        return Ok(vec![control_error_frame(
1821                            &frame,
1822                            "unsupported_control_frame",
1823                            "module-originated channel-0 REQUEST is not supported",
1824                        )?]);
1825                    }
1826                    let request = match parse_module_control_request_from_module(&frame.body) {
1827                        Ok(request) => request,
1828                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1829                            return Ok(vec![control_error_frame(
1830                                &frame,
1831                                "unsupported_control_frame",
1832                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1833                            )?])
1834                        }
1835                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1836                            return Ok(vec![control_error_frame(
1837                                &frame,
1838                                "invalid_control_body",
1839                                format!("malformed module control body: {err}"),
1840                            )?])
1841                        }
1842                    };
1843                    let op = module_control_request_op(&request);
1844                    let corr = frame.header.corr;
1845                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1846                    let result =
1847                        self.handle_module_control_request(ctx.connection_id, frame, request);
1848                    log_slow_control_dispatch(
1849                        dispatch_started_at,
1850                        op,
1851                        ctx.connection_id,
1852                        corr,
1853                        None,
1854                    );
1855                    return result;
1856                }
1857
1858                if is_known_module_request_op(&frame.body) {
1859                    return Ok(vec![control_error_frame(
1860                        &frame,
1861                        "not_registered",
1862                        "catalog.update requires an active module registration owned by this connection",
1863                    )?]);
1864                }
1865
1866                let request = match parse_client_control_request(&frame.body) {
1867                    Ok(request) => request,
1868                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1869                        return Ok(vec![control_error_frame(
1870                            &frame,
1871                            "unknown_control_op",
1872                            format!("unknown client control op: {err}"),
1873                        )?])
1874                    }
1875                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1876                        return Ok(vec![control_error_frame(
1877                            &frame,
1878                            "invalid_control_body",
1879                            format!("malformed client control body: {err}"),
1880                        )?])
1881                    }
1882                };
1883                let op = client_control_request_op(&request);
1884                let corr = frame.header.corr;
1885                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1886                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1887                #[cfg(test)]
1888                if let Some(delay) = self.control_dispatch_delay {
1889                    tokio::time::sleep(delay).await;
1890                }
1891                let result = self
1892                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1893                    .await;
1894                if let Some(timings) = route_open_timings.as_mut() {
1895                    timings.finish();
1896                }
1897                log_slow_control_dispatch(
1898                    dispatch_started_at,
1899                    op,
1900                    ctx.connection_id,
1901                    corr,
1902                    route_open_timings.as_ref(),
1903                );
1904                result
1905            }
1906            FrameType::Push => {
1907                let Some(endpoint) = self
1908                    .forwarding
1909                    .module_endpoint_for_connection(ctx.connection_id)
1910                    .map_err(RouterError::Forwarding)?
1911                else {
1912                    return Ok(vec![control_error_frame(
1913                        &frame,
1914                        "unsupported_control_frame",
1915                        "client-originated channel-0 PUSH is not supported",
1916                    )?]);
1917                };
1918                self.handle_status_update(endpoint, frame)
1919            }
1920            FrameType::Response | FrameType::Error
1921                if self
1922                    .forwarding
1923                    .module_endpoint_for_connection(ctx.connection_id)
1924                    .map_err(RouterError::Forwarding)?
1925                    .is_some() =>
1926            {
1927                self.handle_module_relay_response(ctx.connection_id, frame)
1928            }
1929            ty => Ok(vec![control_error_frame(
1930                &frame,
1931                "unsupported_control_frame",
1932                format!("unsupported channel-0 frame {ty:?}"),
1933            )?]),
1934        }
1935    }
1936
1937    pub fn cleanup_connection(
1938        &self,
1939        connection_id: ConnectionId,
1940    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1941        self.caller_endpoints
1942            .lock()
1943            .unwrap_or_else(|p| p.into_inner())
1944            .remove(&connection_id);
1945        self.forwarding
1946            .operator_confirms()
1947            .connection_closed(connection_id);
1948        self.cleanup_connection_with_end_reason(
1949            connection_id,
1950            RegistrationEndReason::ConnectionClosed,
1951        )
1952    }
1953
1954    fn cleanup_connection_with_end_reason(
1955        &self,
1956        connection_id: ConnectionId,
1957        requested_reason: RegistrationEndReason,
1958    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1959        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1960            self.registry
1961                .get_module_by_connection(connection_id)?
1962                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
1963                .and_then(|module| module.registration_end_reason().ok().flatten())
1964                .unwrap_or(requested_reason)
1965        } else {
1966            requested_reason
1967        };
1968        let crash_closed = self
1969            .registry
1970            .get_module_by_connection(connection_id)?
1971            .and_then(|registration| {
1972                self.forwarding
1973                    .module_endpoint_for_connection(connection_id)
1974                    .ok()
1975                    .flatten()
1976                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1977                    .map(|routes| (registration.manifest.module_id, routes))
1978            });
1979        let crash_closed = crash_closed.map(|(module_id, routes)| {
1980            let terminal = match self.supervisor.get(&module_id) {
1981                None => false,
1982                Some(module) => match module.will_recover_after_connection_loss() {
1983                    Ok(will_recover) => !will_recover,
1984                    Err(err) => {
1985                        warn!(
1986                            %module_id,
1987                            error = %err,
1988                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
1989                        );
1990                        false
1991                    }
1992                },
1993            };
1994            // The forwarding table gates all providers at the start of daemon
1995            // shutdown, before their connections are closed. An ordinary
1996            // module disconnect still reports crash if that gate is not set.
1997            let (reason, drained) = match self
1998                .forwarding
1999                .daemon_drain_result_for_connection(connection_id)
2000            {
2001                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2002                Ok(None) => (RouteCloseReason::Crash, false),
2003                Err(err) => {
2004                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2005                    (RouteCloseReason::Crash, false)
2006                }
2007            };
2008            (module_id, routes, reason, drained, terminal)
2009        });
2010        let registrations = self.deregister_connection(connection_id, end_reason);
2011        let cleanup = if crash_closed.is_some() {
2012            self.forwarding.cleanup_connection_counted(connection_id)
2013        } else {
2014            // A module connection's teardown needs the count of abandoned
2015            // route.bind relays for its route.closed notice below. Any other
2016            // connection, such as a client's, sends no such notice and needs
2017            // only its routes released, so it uses the route-only wrapper and
2018            // reports zero.
2019            self.forwarding
2020                .cleanup_connection(connection_id)
2021                .map(|released| crate::forwarding::ConnectionCleanup {
2022                    released,
2023                    abandoned_relays: 0,
2024                })
2025        };
2026        // The route.closed push waits for forwarding teardown because only
2027        // teardown knows how many pending route.bind relays it aborted. It still
2028        // goes out before the GOODBYEs for the released routes, and its targets
2029        // were captured above, before teardown removed those routes.
2030        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2031            let abandoned = cleanup
2032                .as_ref()
2033                .map_or(0, |cleanup| cleanup.abandoned_relays);
2034            send_route_control_pushes(
2035                &self.forwarding,
2036                routes,
2037                ClientControlPush::RouteClosed {
2038                    module_id,
2039                    channels: Vec::new(),
2040                    reason,
2041                    drained,
2042                    abandoned,
2043                    excluded_subscriptions: 0,
2044                    terminal: Some(terminal),
2045                },
2046            );
2047        }
2048        if let Ok(cleanup) = cleanup {
2049            self.emit_route_goodbyes(cleanup.released);
2050        }
2051        // Signal the registration-release watch only now that BOTH registry and
2052        // forwarding teardown are done, so a supervisor waiting to spawn a
2053        // replacement never observes release while old routes still exist.
2054        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2055            crate::supervise::notify_registration_release();
2056            self.capability_evaluator.wake_deadline_loop();
2057            self.refresh_capability_requirements();
2058        }
2059        self.supervisor.remove_spawn_subscribers(connection_id);
2060        self.activity_streams
2061            .lock()
2062            .unwrap_or_else(|p| p.into_inner())
2063            .retain(|(id, _), task| {
2064                if *id == connection_id {
2065                    task.abort();
2066                    false
2067                } else {
2068                    true
2069                }
2070            });
2071        // Sync authority dies with its connection, so the owner's next
2072        // connection can take it; the owner's scopes stay as they are.
2073        self.hello_launch_nonces
2074            .lock()
2075            .unwrap_or_else(|poisoned| poisoned.into_inner())
2076            .forget(connection_id);
2077        self.scopes
2078            .write()
2079            .unwrap_or_else(|poisoned| poisoned.into_inner())
2080            .release_connection(connection_id);
2081        registrations
2082    }
2083
2084    pub(crate) fn handle_route_goodbye(
2085        &self,
2086        connection_id: ConnectionId,
2087        route_channel: u16,
2088        route_epoch: u32,
2089    ) -> Result<bool, RouterError> {
2090        debug!(
2091            connection_id = connection_id.get(),
2092            route_channel, route_epoch, "handling route GOODBYE"
2093        );
2094        let RouteRelease::Removed(released_route) = self
2095            .forwarding
2096            .release_client_route(connection_id, route_channel, route_epoch)
2097            .map_err(RouterError::Forwarding)?
2098        else {
2099            return Ok(false);
2100        };
2101        self.emit_route_goodbyes(vec![released_route]);
2102        Ok(true)
2103    }
2104
2105    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2106        for released in released_routes {
2107            let frame = match Frame::build_with_version(
2108                released.negotiated_ver,
2109                FrameType::Goodbye,
2110                control_flags(),
2111                released.channel,
2112                released.epoch,
2113                0,
2114                Vec::new(),
2115            ) {
2116                Ok(frame) => frame,
2117                Err(err) => {
2118                    warn!(
2119                        route_channel = released.channel,
2120                        error = %err,
2121                        "failed to build route GOODBYE frame"
2122                    );
2123                    continue;
2124                }
2125            };
2126            if !released.close_on_delivery_failure() {
2127                crate::forwarding::send_module_route_goodbye(
2128                    &self.counters,
2129                    &released.sink,
2130                    frame,
2131                    released.module_id.as_deref(),
2132                    "client route released",
2133                );
2134                continue;
2135            }
2136            if let Err(err) = released.sink.try_send(frame) {
2137                warn!(
2138                    target_connection_id = released.connection_id.get(),
2139                    route_channel = released.channel,
2140                    error = %err,
2141                    "route GOODBYE was not delivered to client; closing target connection"
2142                );
2143                if self
2144                    .forwarding
2145                    .escalate_client_delivery_failure(
2146                        released.connection_id,
2147                        released.channel,
2148                        released.epoch,
2149                        CloseReason::new(
2150                            "route_goodbye_delivery_failed",
2151                            format!(
2152                                "failed to enqueue route GOODBYE for channel {}: {err}",
2153                                released.channel
2154                            ),
2155                        ),
2156                        crate::forwarding::UndeliveredFrame {
2157                            module_id: released.module_id.as_deref(),
2158                            sink: &released.sink,
2159                        },
2160                    )
2161                    .unwrap_or(false)
2162                {
2163                    self.counters.increment_goodbye_relay_client_failed();
2164                }
2165            }
2166        }
2167    }
2168
2169    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2170    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2171    /// own commit failed after the module had already accepted). Without this, a
2172    /// module that accepts late keeps a binding subc has torn down, so a later
2173    /// frame on that module channel could misdeliver if the channel is reused.
2174    ///
2175    /// Never closes the shared module connection on failure: a dropped notification
2176    /// only wastes a bounded amount of warm module-side state, which the module's
2177    /// own idle reaper reclaims. Only call this once the route.bind relay was
2178    /// actually enqueued to the module — if the relay send itself failed, the
2179    /// module never created a binding and there is nothing to tear down.
2180    fn send_abandoned_route_bind_goodbye(
2181        &self,
2182        module_sink: &crate::FrameSink,
2183        negotiated_ver: u8,
2184        module_channel: u16,
2185        module_epoch: u32,
2186    ) {
2187        let frame = match Frame::build_with_version(
2188            negotiated_ver,
2189            FrameType::Goodbye,
2190            control_flags(),
2191            module_channel,
2192            module_epoch,
2193            0,
2194            Vec::new(),
2195        ) {
2196            Ok(frame) => frame,
2197            Err(err) => {
2198                warn!(
2199                    route_channel = module_channel,
2200                    error = %err,
2201                    "failed to build GOODBYE for abandoned route.bind"
2202                );
2203                return;
2204            }
2205        };
2206        crate::forwarding::send_module_route_goodbye(
2207            &self.counters,
2208            module_sink,
2209            frame,
2210            None,
2211            "abandoned route.bind",
2212        );
2213    }
2214
2215    fn handle_hello(
2216        &self,
2217        connection_id: ConnectionId,
2218        sink: Option<crate::FrameSink>,
2219        frame: Frame,
2220    ) -> Result<Vec<Frame>, RouterError> {
2221        debug!(
2222            connection_id = connection_id.get(),
2223            corr = frame.header.corr,
2224            "handling HELLO"
2225        );
2226        // A module connection has one identity for its entire lifetime. A second
2227        // registration would leave the old registry owner behind while replacing
2228        // its forwarding endpoint and launch nonce.
2229        if self
2230            .registry
2231            .get_module_by_connection(connection_id)
2232            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2233            .is_some()
2234        {
2235            return Ok(vec![control_error_frame(
2236                &frame,
2237                "invalid_hello",
2238                "connection is already registered as a module",
2239            )?]);
2240        }
2241        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2242            Ok(value) => value,
2243            Err(err) => {
2244                return Ok(vec![control_error_frame(
2245                    &frame,
2246                    "invalid_hello",
2247                    format!("malformed HELLO body: {err}"),
2248                )?])
2249            }
2250        };
2251        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2252            return Ok(vec![control_error_frame(
2253                &frame,
2254                "invalid_capability_grammar",
2255                err.to_string(),
2256            )?]);
2257        }
2258        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2259            return Ok(vec![control_error_frame(
2260                &frame,
2261                "invalid_manifest",
2262                err.to_string(),
2263            )?]);
2264        }
2265        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2266            return Ok(vec![control_error_frame(
2267                &frame,
2268                "invalid_event_declaration",
2269                err.to_string(),
2270            )?]);
2271        }
2272        if let Some(provenance) = hello_value
2273            .get("manifest")
2274            .and_then(|manifest| manifest.get("provenance"))
2275        {
2276            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2277                return Ok(vec![control_error_frame(
2278                    &frame,
2279                    "invalid_manifest",
2280                    format!("malformed manifest provenance: {err}"),
2281                )?]);
2282            }
2283        }
2284        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2285            Ok(hello) => hello,
2286            Err(err) => {
2287                return Ok(vec![control_error_frame(
2288                    &frame,
2289                    "invalid_hello",
2290                    format!("malformed HELLO body: {err}"),
2291                )?])
2292            }
2293        };
2294
2295        if hello.protocol_ver != hello.manifest.protocol_ver {
2296            return Ok(vec![control_error_frame(
2297                &frame,
2298                "invalid_manifest",
2299                format!(
2300                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2301                    hello.protocol_ver, hello.manifest.protocol_ver
2302                ),
2303            )?]);
2304        }
2305
2306        if hello.manifest.module_id.trim().is_empty() {
2307            return Ok(vec![control_error_frame(
2308                &frame,
2309                "invalid_manifest",
2310                "manifest module_id must not be empty",
2311            )?]);
2312        }
2313
2314        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2315            Ok(negotiated_ver) => negotiated_ver,
2316            Err(message) => {
2317                return Ok(vec![control_error_frame(
2318                    &frame,
2319                    "version_unsupported",
2320                    message,
2321                )?])
2322            }
2323        };
2324
2325        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2326        // swap is open for this id, the only HELLO admitted as a second process
2327        // is the one carrying the candidate's launch nonce (the swap token), and
2328        // it registers into the candidate slot rather than being refused as a
2329        // duplicate. Run after the reserved gate, a reserved module's candidate
2330        // would be refused `reserved_module` for presenting a nonce that gate
2331        // does not know. See `SupervisorHandle::swap_hello_admission`.
2332        let swap_admission = self
2333            .supervisor
2334            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2335        if swap_admission == SwapHelloAdmission::Refused {
2336            warn!(
2337                module_id = %hello.manifest.module_id,
2338                connection_id = connection_id.get(),
2339                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2340            );
2341            return Ok(vec![control_error_frame(
2342                &frame,
2343                "swap_token_invalid",
2344                format!(
2345                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2346                    hello.manifest.module_id
2347                ),
2348            )?]);
2349        }
2350        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2351
2352        // Reserved-module identity gate: a module_id configured `reserved` may be
2353        // registered ONLY by the process subc spawned for it, proven by echoing the
2354        // one-time launch nonce subc injected. A non-reserved id has no recorded
2355        // nonce and always passes. This blocks a key-holder from impersonating a
2356        // security-boundary module (e.g. the credential vault) while the real one is
2357        // down/restarting and its registration slot is momentarily free. A swap
2358        // candidate has already proven the same thing with its own nonce above.
2359        if let Some(rejection) = (!swap_candidate)
2360            .then(|| {
2361                self.supervisor.reserved_hello_rejection(
2362                    &hello.manifest.module_id,
2363                    hello.launch_nonce.as_deref(),
2364                )
2365            })
2366            .flatten()
2367        {
2368            let message = match rejection {
2369                ReservedHelloRejection::Exact { module_id } => format!(
2370                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2371                ),
2372                ReservedHelloRejection::Prefix {
2373                    prefix,
2374                    owner_module_id,
2375                } => format!(
2376                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2377                    hello.manifest.module_id
2378                ),
2379            };
2380            return Ok(vec![control_error_frame(
2381                &frame,
2382                "reserved_module",
2383                message,
2384            )?]);
2385        }
2386
2387        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2388            &hello.manifest.module_id,
2389            hello.manifest.capabilities.as_ref(),
2390        );
2391        if let Some(refusal) = reserved_capability_refusals.first() {
2392            let capability = refusal.capability.clone();
2393            let bound_module = refusal.claimants[0].clone();
2394            log_duplicate_claim_events(reserved_capability_refusals);
2395            return Ok(vec![control_error_frame(
2396                &frame,
2397                "reserved_capability",
2398                format!(
2399                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2400                    capability, bound_module, hello.manifest.module_id
2401                ),
2402            )?]);
2403        }
2404
2405        // A connection that already opened client routes must not also register as
2406        // a module: cleanup would then release only one side and leak the other.
2407        if self
2408            .forwarding
2409            .connection_has_client_routes(connection_id)
2410            .map_err(RouterError::Forwarding)?
2411        {
2412            return Ok(vec![control_error_frame(
2413                &frame,
2414                "invalid_hello",
2415                "connection has open client routes and cannot also register as a module",
2416            )?]);
2417        }
2418
2419        // Kept for scope sync authority, which goes only to the connection that
2420        // presented the module's current launch nonce. Recorded before the
2421        // registration is attempted: a connection whose registration then fails
2422        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2423        self.hello_launch_nonces
2424            .lock()
2425            .unwrap_or_else(|poisoned| poisoned.into_inner())
2426            .record(connection_id, hello.launch_nonce.as_deref());
2427        let control_ops = effective_module_control_ops(hello.control_ops);
2428        // Built before anything is registered so an encoding failure leaves no
2429        // registry or forwarding state behind.
2430        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2431        if swap_candidate {
2432            return self.register_swap_candidate(
2433                connection_id,
2434                sink,
2435                &frame,
2436                hello.manifest,
2437                negotiated_ver,
2438                control_ops,
2439                hello_ack,
2440            );
2441        }
2442        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2443        // is refused, but `catalog.list` does not show the module and the
2444        // catalog generation does not move until it is published below, once
2445        // its forwarding endpoint exists. Listing it now would advertise a
2446        // module that `route.open` still refuses for having no forwarding
2447        // connection.
2448        let registration = match self.registry.register_unpublished_with_control_ops(
2449            hello.manifest,
2450            negotiated_ver,
2451            connection_id,
2452            control_ops,
2453        ) {
2454            Ok(registration) => registration,
2455            Err(RegistryError::DuplicateModuleId { module_id }) => {
2456                return Ok(vec![control_error_frame(
2457                    &frame,
2458                    "duplicate_module_id",
2459                    format!(
2460                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2461                    ),
2462                )?])
2463            }
2464            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2465                return Ok(vec![control_error_frame(
2466                    &frame,
2467                    "invalid_module_id",
2468                    err.to_string(),
2469                )?])
2470            }
2471            Err(err) => {
2472                return Ok(vec![control_error_frame(
2473                    &frame,
2474                    "registry_error",
2475                    err.to_string(),
2476                )?])
2477            }
2478        };
2479        #[cfg(test)]
2480        self.hold_between_registry_and_forwarding();
2481
2482        let reply = if let Some(sink) = sink {
2483            // The forwarding table's module store is also the daemon-to-module
2484            // control-RPC lane, so every HELLO gets a live endpoint even when the
2485            // manifest has no routable provider role. Non-routable modules still
2486            // cannot receive route.bind in production: `handle_route_open` checks
2487            // the registry manifest with `target_has_required_role` before the
2488            // only production call to `begin_route_bind_relay_for` below that
2489            // route.open path. The remaining direct relay callers are unit tests
2490            // and benchmark harnesses that construct forwarding state explicitly.
2491            //
2492            // The HELLO_ACK is queued by the forwarding table itself, before the
2493            // endpoint becomes visible, and is NOT returned as a reply. A module
2494            // reads HELLO_ACK first and exits on anything else; a reply is only
2495            // written after this handler returns, by which time a route.open on
2496            // another connection could already have queued a route.bind request
2497            // for this module ahead of it.
2498            let concurrency = manifest_concurrency(&registration.manifest);
2499            if let Err(err) = self.forwarding.register_module_connection_acked(
2500                connection_id,
2501                registration.manifest.module_id.clone(),
2502                negotiated_ver,
2503                concurrency,
2504                sink,
2505                hello_ack,
2506            ) {
2507                // Forwarding registration failed, so there is no forwarding
2508                // state to tear down. Remove the registry entry, which was never
2509                // published and so never listed or counted in the catalog
2510                // generation, and signal the release watch directly.
2511                if matches!(
2512                    self.deregister_connection(
2513                        connection_id,
2514                        RegistrationEndReason::RegistrationFailed,
2515                    ),
2516                    Ok(r) if !r.is_empty()
2517                ) {
2518                    crate::supervise::notify_registration_release();
2519                }
2520                return Ok(vec![control_error_frame(
2521                    &frame,
2522                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2523                        "invalid_hello"
2524                    } else {
2525                        forwarding_error_code(&err)
2526                    },
2527                    err.to_string(),
2528                )?]);
2529            }
2530            Vec::new()
2531        } else {
2532            // No sink means no forwarding endpoint, so nothing can be routed
2533            // ahead of the ack; it goes out as the reply.
2534            vec![hello_ack]
2535        };
2536
2537        // The module is routable now (or, without a sink, as routable as it
2538        // will ever be), so the catalog may list it. Publishing bumps the
2539        // catalog generation in the same registry critical section that makes
2540        // the module listed.
2541        if !self
2542            .registry
2543            .publish_registration(connection_id)
2544            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2545        {
2546            debug!(
2547                module_id = %registration.manifest.module_id,
2548                connection_id = connection_id.get(),
2549                "registration was removed before it could be published"
2550            );
2551        }
2552
2553        // Exposure over assumption: Concurrency's serde default is pinned to the
2554        // pre-field behavior (ModuleManaged), so a management surface that is
2555        // genuinely Serial and just never declared it inherits concurrent
2556        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2557        // turns "no module has been bitten yet" into the checkable claim "no
2558        // module is exposed" -- one read of the boot log instead of a fleet
2559        // audit. Detected from the raw HELLO bytes because the serde default
2560        // deliberately erases the absent/declared distinction from the type.
2561        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2562            info!(
2563                module_id = %registration.manifest.module_id,
2564                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2565            );
2566        }
2567
2568        self.apply_registration_capabilities(&registration);
2569
2570        self.supervisor
2571            .record_activity_registration(&registration.manifest.module_id);
2572
2573        info!(
2574            module_id = %registration.manifest.module_id,
2575            module_version = %registration.manifest.module_version,
2576            negotiated_ver,
2577            routable_provider = manifest_provides_routable_role(&registration.manifest),
2578            connection_id = connection_id.get(),
2579            "module registered"
2580        );
2581
2582        Ok(reply)
2583    }
2584
2585    /// Register a HELLO the swap gate admitted into the candidate slot of the
2586    /// registry and of forwarding, where it is reachable over its own
2587    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2588    /// nothing routes to it until the supervisor cuts over.
2589    ///
2590    /// Registry first (unpublished), then forwarding, then publication, the
2591    /// same order as an ordinary HELLO; a forwarding failure removes the
2592    /// registry entry again. The swap supervisor treats a published, ready
2593    /// candidate as registered and promotes it, and promotion needs the
2594    /// candidate's forwarding endpoint, so the candidate is published only
2595    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2596    /// recompute capability verdicts or run the capability deny census
2597    /// (`enforce_capability_denies`): both are about the module in the active
2598    /// slot, which `route.open` can target. A candidate sits in the candidate
2599    /// slot, so it neither provides capabilities to other modules nor can be
2600    /// routed to until promotion moves it into the active slot.
2601    #[allow(clippy::too_many_arguments)]
2602    fn register_swap_candidate(
2603        &self,
2604        connection_id: ConnectionId,
2605        sink: Option<crate::FrameSink>,
2606        frame: &Frame,
2607        manifest: ModuleManifest,
2608        negotiated_ver: u8,
2609        control_ops: Vec<String>,
2610        hello_ack: Frame,
2611    ) -> Result<Vec<Frame>, RouterError> {
2612        let module_id = manifest.module_id.clone();
2613        let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2614            manifest,
2615            negotiated_ver,
2616            connection_id,
2617            control_ops,
2618        ) {
2619            Ok(registration) => registration,
2620            Err(RegistryError::DuplicateModuleId { module_id }) => {
2621                return Ok(vec![control_error_frame(
2622                    frame,
2623                    "duplicate_module_id",
2624                    format!(
2625                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2626                    ),
2627                )?])
2628            }
2629            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2630                return Ok(vec![control_error_frame(
2631                    frame,
2632                    "invalid_module_id",
2633                    err.to_string(),
2634                )?])
2635            }
2636            Err(err) => {
2637                return Ok(vec![control_error_frame(
2638                    frame,
2639                    "registry_error",
2640                    err.to_string(),
2641                )?])
2642            }
2643        };
2644        #[cfg(test)]
2645        self.hold_between_registry_and_forwarding();
2646        let reply = if let Some(sink) = sink {
2647            // Same ordering as an ordinary HELLO: the forwarding table queues
2648            // the HELLO_ACK before the candidate endpoint is inserted, because
2649            // a module exits if its first frame after HELLO is anything else.
2650            let concurrency = manifest_concurrency(&registration.manifest);
2651            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2652                connection_id,
2653                module_id.clone(),
2654                negotiated_ver,
2655                concurrency,
2656                sink,
2657                hello_ack,
2658            ) {
2659                if matches!(
2660                    self.deregister_connection(
2661                        connection_id,
2662                        RegistrationEndReason::RegistrationFailed,
2663                    ),
2664                    Ok(r) if !r.is_empty()
2665                ) {
2666                    crate::supervise::notify_registration_release();
2667                }
2668                return Ok(vec![control_error_frame(
2669                    frame,
2670                    forwarding_error_code(&err),
2671                    err.to_string(),
2672                )?]);
2673            }
2674            Vec::new()
2675        } else {
2676            vec![hello_ack]
2677        };
2678        if !self
2679            .registry
2680            .publish_registration(connection_id)
2681            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2682        {
2683            debug!(
2684                module_id = %module_id,
2685                connection_id = connection_id.get(),
2686                "swap candidate registration was removed before it could be published"
2687            );
2688        }
2689        self.supervisor.mark_swap_candidate_admitted(&module_id);
2690        info!(
2691            module_id = %module_id,
2692            module_version = %registration.manifest.module_version,
2693            negotiated_ver,
2694            ready = registration.ready,
2695            connection_id = connection_id.get(),
2696            "swap candidate registered; not routable until cutover"
2697        );
2698        Ok(reply)
2699    }
2700
2701    fn build_hello_ack(
2702        &self,
2703        frame: &Frame,
2704        negotiated_ver: u8,
2705        module_id: &str,
2706    ) -> Result<Frame, RouterError> {
2707        let ack = ModuleHelloAckBody {
2708            negotiated_ver,
2709            subc_ops: module_subc_ops(),
2710            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2711            storage: self
2712                .storage_config
2713                .as_ref()
2714                .map(|cfg| cfg.descriptor_for(module_id)),
2715            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2716        };
2717        let body = serde_json::to_vec(&ack).map_err(|err| {
2718            RouterError::backend(
2719                0,
2720                frame.header.corr,
2721                format!("failed to encode HELLO_ACK: {err}"),
2722            )
2723        })?;
2724
2725        Frame::build_with_version(
2726            negotiated_ver,
2727            FrameType::HelloAck,
2728            control_flags(),
2729            0,
2730            0,
2731            frame.header.corr,
2732            body,
2733        )
2734        .map_err(RouterError::FrameBuild)
2735    }
2736
2737    async fn handle_client_control_request(
2738        &self,
2739        ctx: &RouteCtx,
2740        frame: Frame,
2741        request: ClientControlRequest,
2742        route_open_timings: &mut Option<RouteOpenTimings>,
2743    ) -> Result<Vec<Frame>, RouterError> {
2744        if let Some(operation) = privileged_operation(&request) {
2745            let caller = self.resolve_caller(ctx.connection_id).await;
2746            let op = client_control_request_op(&request);
2747            let text = prompt_text(operation, &caller);
2748            match self.privileged_operator_prompt {
2749                PrivilegedOperatorPrompt::Log => {
2750                    info!(target: "subc_daemon::privileged_prompt", operation = op,
2751                        target = ?operation, caller = %caller.description(), summary = %text,
2752                        "privileged request would prompt the operator");
2753                }
2754                PrivilegedOperatorPrompt::Enforce => {
2755                    let key =
2756                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2757                    let outcome = self
2758                        .forwarding
2759                        .operator_confirms()
2760                        .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2761                        .await;
2762                    if outcome.code != "confirmed" {
2763                        return Ok(vec![outcome.frame(&frame)]);
2764                    }
2765                }
2766            }
2767        }
2768        match request {
2769            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2770            ClientControlRequest::CatalogList { module_id } => {
2771                self.handle_catalog_list(frame, module_id)
2772            }
2773            ClientControlRequest::RouteOpen {
2774                target,
2775                identity,
2776                consumer_identity,
2777                consumer_capabilities,
2778                role_versions,
2779                admission_facts,
2780                scope,
2781            } => {
2782                self.handle_route_open(
2783                    ctx,
2784                    frame,
2785                    RouteOpenRequest {
2786                        target,
2787                        identity,
2788                        consumer_identity,
2789                        consumer_capabilities,
2790                        role_versions,
2791                        admission_facts,
2792                        scope,
2793                    },
2794                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2795                )
2796                .await
2797            }
2798            ClientControlRequest::RoutePoll {
2799                route_channel,
2800                route_epoch,
2801                kind,
2802            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2803            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2804            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2805                self.handle_supervisor_spawn_snapshot(frame)
2806            }
2807            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2808                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2809            }
2810            ClientControlRequest::SupervisorRestart {
2811                module_id,
2812                drain_timeout_ms,
2813            } => {
2814                self.log_supervisor_request_received(
2815                    ctx,
2816                    frame.header.corr,
2817                    ops::SUPERVISOR_RESTART,
2818                    Some(&module_id),
2819                    None,
2820                )?;
2821                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2822                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2823                    .await
2824            }
2825            ClientControlRequest::SupervisorSwap {
2826                module_id,
2827                ready_timeout_ms,
2828            } => {
2829                self.log_supervisor_request_received(
2830                    ctx,
2831                    frame.header.corr,
2832                    ops::SUPERVISOR_SWAP,
2833                    Some(&module_id),
2834                    None,
2835                )?;
2836                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2837                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2838                    .await
2839            }
2840            ClientControlRequest::SupervisorReload { module_id } => {
2841                self.log_supervisor_request_received(
2842                    ctx,
2843                    frame.header.corr,
2844                    ops::SUPERVISOR_RELOAD,
2845                    Some(&module_id),
2846                    None,
2847                )?;
2848                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2849                self.handle_supervisor_reload(frame, module_id, requester)
2850                    .await
2851            }
2852            ClientControlRequest::SupervisorRescan { preview } => {
2853                if !preview {
2854                    self.log_supervisor_request_received(
2855                        ctx,
2856                        frame.header.corr,
2857                        ops::SUPERVISOR_RESCAN,
2858                        None,
2859                        None,
2860                    )?;
2861                }
2862                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2863                self.handle_supervisor_rescan(frame, preview, requester)
2864                    .await
2865            }
2866            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2867                self.log_supervisor_request_received(
2868                    ctx,
2869                    frame.header.corr,
2870                    ops::SUPERVISOR_RELEASE_RESERVED,
2871                    Some(&module_id),
2872                    None,
2873                )?;
2874                self.handle_supervisor_release_reserved(frame, module_id)
2875                    .await
2876            }
2877            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2878                self.log_supervisor_request_received(
2879                    ctx,
2880                    frame.header.corr,
2881                    ops::SUPERVISOR_SET_ENABLED,
2882                    Some(&module_id),
2883                    Some(enabled),
2884                )?;
2885                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2886                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2887                    .await
2888            }
2889            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2890                self.handle_supervisor_health_probe(frame, module_id).await
2891            }
2892            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2893            ClientControlRequest::SupervisorRoutes { module_id } => {
2894                self.handle_supervisor_routes(frame, module_id)
2895            }
2896            ClientControlRequest::SupervisorProvenance { module_id } => {
2897                self.handle_supervisor_provenance(frame, module_id).await
2898            }
2899            ClientControlRequest::SupervisorStderrTail {
2900                module_id,
2901                max_lines,
2902                max_bytes,
2903            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2904            ClientControlRequest::SupervisorTerminals { module_id } => {
2905                self.handle_supervisor_terminals(frame, module_id).await
2906            }
2907        }
2908    }
2909
2910    fn log_supervisor_request_received(
2911        &self,
2912        ctx: &RouteCtx,
2913        corr: u64,
2914        op: &'static str,
2915        module_id: Option<&str>,
2916        enabled: Option<bool>,
2917    ) -> Result<(), RouterError> {
2918        let caller = self.supervisor_requester(ctx, corr)?;
2919        let caller = principal_label(&caller);
2920        // `Option` fields are recorded only when present, so a request with no
2921        // module id (rescan) or no enabled flag simply omits that field.
2922        info!(
2923            target: "control",
2924            op,
2925            module_id,
2926            enabled,
2927            connection_id = ctx.connection_id.get(),
2928            caller = %caller,
2929            "supervisor request received"
2930        );
2931        Ok(())
2932    }
2933
2934    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2935        Ok(self
2936            .registry
2937            .get_module_by_connection(ctx.connection_id)
2938            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2939            .map(|registration| Principal::Reserved {
2940                module_id: registration.manifest.module_id,
2941            })
2942            .unwrap_or(Principal::Direct))
2943    }
2944
2945    fn handle_module_control_request(
2946        &self,
2947        connection_id: ConnectionId,
2948        frame: Frame,
2949        request: ModuleControlRequestFromModule,
2950    ) -> Result<Vec<Frame>, RouterError> {
2951        match request {
2952            ModuleControlRequestFromModule::CatalogUpdate {
2953                provides,
2954                capabilities,
2955                ready,
2956            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2957            ModuleControlRequestFromModule::LiveRoots {} => {
2958                let registered = self
2959                    .registry
2960                    .get_module_by_connection(connection_id)
2961                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2962                let Some(registration) = registered else {
2963                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2964                };
2965                let response = self
2966                    .forwarding
2967                    .live_roots(&registration.manifest.module_id)
2968                    .map_err(RouterError::Forwarding)?;
2969                Ok(vec![control_response_body_frame(
2970                    &frame,
2971                    &response,
2972                    "ModuleControlResponseToModule::LiveRoots",
2973                )?])
2974            }
2975            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2976                self.handle_scope_sync(connection_id, frame, generation, scopes)
2977            }
2978            ModuleControlRequestFromModule::ScopeApply {
2979                generation,
2980                upsert,
2981                end,
2982            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2983            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2984                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2985            }
2986        }
2987    }
2988
2989    fn handle_operator_confirm(
2990        &self,
2991        ctx: &RouteCtx,
2992        frame: Frame,
2993    ) -> Result<Vec<Frame>, RouterError> {
2994        use crate::operator_confirm::{audit, Outcome};
2995        let registration = self
2996            .registry
2997            .get_module_by_connection(ctx.connection_id)
2998            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2999        let Some(registration) = registration else {
3000            let outcome = Outcome::refusal("not_registered");
3001            audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
3002            return Ok(vec![outcome.frame(&frame)]);
3003        };
3004        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3005            Ok(request) => request,
3006            Err(_) => {
3007                let outcome = Outcome::refusal("invalid_control_body");
3008                audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
3009                return Ok(vec![outcome.frame(&frame)]);
3010            }
3011        };
3012        let module_id = registration.manifest.module_id;
3013        // Read the launch nonce (under its own lock) before taking the forwarding
3014        // table's lock below: holding forwarding while waiting on another daemon
3015        // lock risks a lock-order deadlock with paths that take them the other way.
3016        let nonce = self
3017            .hello_launch_nonces
3018            .lock()
3019            .unwrap_or_else(|p| p.into_inner())
3020            .nonce(ctx.connection_id)
3021            .map(str::to_owned);
3022        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3023            self.supervisor
3024                .spawned_consumer_authorized(&module_id, nonce)
3025        });
3026        let confirms = self.forwarding.operator_confirms();
3027        self.forwarding
3028            .with_operator_route(
3029                ctx.connection_id,
3030                request.route_channel,
3031                request.route_epoch,
3032                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3033            )
3034            .map_err(RouterError::Forwarding)
3035    }
3036
3037    /// `scope.sync`: the owner is the module registered on this connection.
3038    /// A connection with no registration (every client connection, `direct`
3039    /// included) is refused `not_registered` before the table is consulted.
3040    fn handle_scope_sync(
3041        &self,
3042        connection_id: ConnectionId,
3043        frame: Frame,
3044        generation: u64,
3045        scopes: Vec<ScopeRecord>,
3046    ) -> Result<Vec<Frame>, RouterError> {
3047        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3048    }
3049
3050    fn handle_scope_change(
3051        &self,
3052        connection_id: ConnectionId,
3053        frame: Frame,
3054        generation: u64,
3055        scopes: Vec<ScopeRecord>,
3056        end: Option<Vec<ScopeEnd>>,
3057    ) -> Result<Vec<Frame>, RouterError> {
3058        let is_apply = end.is_some();
3059        let Some(registration) = self
3060            .registry
3061            .get_module_by_connection(connection_id)
3062            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3063        else {
3064            return Ok(vec![control_error_frame(
3065                &frame,
3066                "not_registered",
3067                "scope.sync and scope.apply require an active module registration owned by this connection",
3068            )?]);
3069        };
3070        let owner = registration.manifest.module_id;
3071        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3072        let is_current_launch = |connection: ConnectionId| {
3073            self.hello_launch_nonces
3074                .lock()
3075                .unwrap_or_else(|poisoned| poisoned.into_inner())
3076                .presented(connection, current_nonce.as_deref())
3077        };
3078        // Lock order is the scope table, then the forwarding table: the new
3079        // tags are published, and the routes the change closes are selected,
3080        // while the scope table is still write-locked, so no admission can read
3081        // a record whose tag is not yet published.
3082        let mut table = self
3083            .scopes
3084            .write()
3085            .unwrap_or_else(|poisoned| poisoned.into_inner());
3086        let outcome = match end {
3087            Some(end) => table.apply(
3088                &owner,
3089                connection_id,
3090                is_current_launch,
3091                generation,
3092                scopes,
3093                end,
3094            ),
3095            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3096        };
3097        let drained = match &outcome {
3098            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3099            Err(_) => Vec::new(),
3100        };
3101        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3102        drop(table);
3103        if has_deadlines {
3104            self.scope_deadline_added.notify_one();
3105        }
3106        match outcome {
3107            Ok(applied) => {
3108                let counts = ScopeOutcomeCounts::of(&applied.results);
3109                info!(
3110                    owner = %owner,
3111                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3112                    generation,
3113                    records = applied.results.len(),
3114                    created = counts.created,
3115                    replaced = counts.replaced,
3116                    updated = counts.updated,
3117                    unchanged = counts.unchanged,
3118                    refused = counts.refused,
3119                    ended = applied.ended.len(),
3120                    tag_changes = applied.tag_changes.len(),
3121                    routes_closed = drained.len(),
3122                    "scope change accepted"
3123                );
3124                // An accepted scope change can still refuse individual records, and the
3125                // owner is the only party that sees the reply. Name them here so
3126                // an operator can tell a refused session from a missing one
3127                // without the owner's logs. Capped so a call that refuses
3128                // thousands cannot flood the log; the count above is complete.
3129                for refused in applied
3130                    .results
3131                    .iter()
3132                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3133                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3134                {
3135                    warn!(
3136                        owner = %owner,
3137                        generation,
3138                        scope_ref = %refused.scope_ref,
3139                        scope_epoch = refused.scope_epoch,
3140                        code = refused.code.as_deref().unwrap_or(""),
3141                        "scope record refused"
3142                    );
3143                }
3144                self.close_scope_drained_routes(drained);
3145                let response = if is_apply {
3146                    ModuleControlResponseToModule::ScopeApply {
3147                        generation,
3148                        results: applied.results,
3149                        end_results: applied.end_results,
3150                        ended: applied.ended,
3151                    }
3152                } else {
3153                    ModuleControlResponseToModule::ScopeSync {
3154                        generation,
3155                        results: applied.results,
3156                        ended: applied.ended,
3157                    }
3158                };
3159                Ok(vec![control_response_body_frame(
3160                    &frame,
3161                    &response,
3162                    "scope change response",
3163                )?])
3164            }
3165            Err(refusal) => {
3166                info!(
3167                    owner = %owner,
3168                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3169                    generation,
3170                    code = refusal.code,
3171                    "scope change refused"
3172                );
3173                Ok(vec![control_error_frame(
3174                    &frame,
3175                    refusal.code,
3176                    refusal.message,
3177                )?])
3178            }
3179        }
3180    }
3181
3182    /// Tell both ends of each route a scope change closed. The module gets a
3183    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3184    /// the client's route handle. The client also gets `route.closed` with the
3185    /// scope reason, one push per module and reason, so it can tell a revoked
3186    /// route from an ordinary close and not reopen it.
3187    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3188        if drained.is_empty() {
3189            return;
3190        }
3191        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3192            BTreeMap::new();
3193        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3194        for route in drained {
3195            warn!(
3196                module_id = %route.module_id,
3197                reason = ?route.reason,
3198                client_connection_id = route.client.connection_id.get(),
3199                route_channel = route.client.channel,
3200                "closing route because its scope changed"
3201            );
3202            pushes
3203                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3204                .or_insert_with(|| (route.reason, Vec::new()))
3205                .1
3206                .push(EndpointRoute {
3207                    goodbye_target: route.client.clone(),
3208                    principal: Principal::Unverified,
3209                    bound_at: Instant::now(),
3210                    draining: false,
3211                    drain_reason: None,
3212                });
3213            goodbyes.push(route.module);
3214            goodbyes.push(route.client);
3215        }
3216        for ((module_id, _), (reason, routes)) in pushes {
3217            send_route_control_pushes(
3218                &self.forwarding,
3219                routes,
3220                ClientControlPush::RouteClosed {
3221                    module_id,
3222                    channels: Vec::new(),
3223                    reason,
3224                    drained: false,
3225                    abandoned: 0,
3226                    excluded_subscriptions: 0,
3227                    terminal: Some(false),
3228                },
3229            );
3230        }
3231        self.emit_route_goodbyes(goodbyes);
3232    }
3233
3234    /// `scope.describe`: any registered module may read any scope, because a
3235    /// provider must read the scope a route it serves is stamped with.
3236    fn handle_scope_describe(
3237        &self,
3238        connection_id: ConnectionId,
3239        frame: Frame,
3240        owner: Principal,
3241        scope_ref: String,
3242    ) -> Result<Vec<Frame>, RouterError> {
3243        let registered = self
3244            .registry
3245            .get_module_by_connection(connection_id)
3246            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3247        if registered.is_none() {
3248            return Ok(vec![control_error_frame(
3249                &frame,
3250                "not_registered",
3251                "scope.describe requires an active module registration owned by this connection",
3252            )?]);
3253        }
3254        let description = self
3255            .scopes
3256            .read()
3257            .unwrap_or_else(|poisoned| poisoned.into_inner())
3258            .describe(&owner, &scope_ref);
3259        let owner_configured = match &owner {
3260            // Ask whether the owner is configured (`is_configured`), not
3261            // whether it is on the roster (`get(..).is_some()`): a supervised
3262            // module's process can register and describe a scope before the
3263            // supervisor has put it on the roster.
3264            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3265            _ => false,
3266        };
3267        let response = ModuleControlResponseToModule::ScopeDescribe {
3268            status: description.status,
3269            scope_epoch: description.scope_epoch,
3270            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3271            owner_synced: description.owner_synced,
3272            owner_configured,
3273            scope: description.stamp,
3274        };
3275        Ok(vec![control_response_body_frame(
3276            &frame,
3277            &response,
3278            "ModuleControlResponseToModule::ScopeDescribe",
3279        )?])
3280    }
3281
3282    fn handle_catalog_update(
3283        &self,
3284        connection_id: ConnectionId,
3285        frame: Frame,
3286        provides: Vec<ProviderRole>,
3287        capabilities: Option<CapabilityDeclarations>,
3288        ready: Option<bool>,
3289    ) -> Result<Vec<Frame>, RouterError> {
3290        self.refresh_capability_requirements();
3291        let Some(registration) = self
3292            .registry
3293            .get_module_by_connection(connection_id)
3294            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3295        else {
3296            return Ok(vec![control_error_frame(
3297                &frame,
3298                "not_registered",
3299                "catalog.update requires an active module registration owned by this connection",
3300            )?]);
3301        };
3302
3303        if let Some(message) =
3304            catalog_update_frozen_field_message(&registration.manifest, &provides)
3305        {
3306            return Ok(vec![control_error_frame(
3307                &frame,
3308                "catalog_update_frozen_field",
3309                message,
3310            )?]);
3311        }
3312
3313        let mut candidate = registration.manifest.clone();
3314        candidate.provides = provides.clone();
3315        candidate.capabilities = capabilities
3316            .clone()
3317            .or_else(|| registration.manifest.capabilities.clone());
3318        if let Err(err) = candidate.validate_capability_grammar() {
3319            return Ok(vec![control_error_frame(
3320                &frame,
3321                "invalid_capability_grammar",
3322                err.to_string(),
3323            )?]);
3324        }
3325
3326        // Updates must honor the same reserved owner as initial registration;
3327        // otherwise an empty HELLO could acquire the claim after admission.
3328        let mut conflicts = self
3329            .capability_evaluator
3330            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3331        if let Some(conflict) = conflicts.first() {
3332            let message = format!(
3333                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3334                conflict.capability, conflict.claimants[0], candidate.module_id
3335            );
3336            for conflict in &mut conflicts {
3337                conflict.source = DuplicateClaimSource::CatalogUpdate;
3338            }
3339            log_duplicate_claim_events(conflicts);
3340            return Ok(vec![control_error_frame(
3341                &frame,
3342                "reserved_capability",
3343                message,
3344            )?]);
3345        }
3346
3347        let updated = self
3348            .registry
3349            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3350            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3351        if updated.is_none() {
3352            return Ok(vec![control_error_frame(
3353                &frame,
3354                "not_registered",
3355                "catalog.update requires an active module registration owned by this connection",
3356            )?]);
3357        }
3358        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3359            log_duplicate_claim_events(
3360                self.capability_evaluator
3361                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3362            );
3363        }
3364        if capability_census_trigger(
3365            registration.manifest.capabilities.as_ref(),
3366            updated
3367                .as_ref()
3368                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3369        ) {
3370            self.enforce_capability_denies();
3371        }
3372        self.refresh_capability_requirements();
3373
3374        let response = ModuleControlResponseToModule::CatalogUpdate {};
3375        control_response_body_frame(
3376            &frame,
3377            &response,
3378            "ModuleControlResponseToModule::CatalogUpdate",
3379        )
3380        .map(|frame| vec![frame])
3381    }
3382
3383    pub(crate) fn with_activity_ledger_config(
3384        mut self,
3385        run_dir: &Path,
3386        incarnation: String,
3387        config: crate::activity::ActivityConfig,
3388    ) -> Self {
3389        self.activity = Some(crate::activity::ActivityLedger::open_config(
3390            run_dir.join("activity.jsonl"),
3391            incarnation,
3392            config,
3393        ));
3394        self.supervisor.set_activity(self.activity.clone());
3395        self
3396    }
3397
3398    pub(crate) async fn configure_activity(
3399        &self,
3400        config: crate::activity::ActivityConfig,
3401    ) -> Result<(), &'static str> {
3402        match &self.activity {
3403            Some(ledger) => ledger.configure(config).await,
3404            None => Ok(()),
3405        }
3406    }
3407
3408    fn cancel_activity_subscription(
3409        &self,
3410        ctx: &RouteCtx,
3411        frame: &Frame,
3412    ) -> Result<bool, RouterError> {
3413        let task = self
3414            .activity_streams
3415            .lock()
3416            .unwrap_or_else(|p| p.into_inner())
3417            .remove(&(ctx.connection_id, frame.header.corr));
3418        let Some(task) = task else {
3419            return Ok(false);
3420        };
3421        task.abort();
3422        let end = Frame::build_with_version(
3423            response_version(frame),
3424            FrameType::StreamEnd,
3425            control_flags(),
3426            0,
3427            0,
3428            frame.header.corr,
3429            Vec::new(),
3430        )
3431        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3432        let sink = ctx.egress.clone();
3433        tokio::spawn(async move {
3434            let _ = sink.send(end).await;
3435        });
3436        Ok(true)
3437    }
3438
3439    async fn handle_activity_read(
3440        &self,
3441        ctx: &RouteCtx,
3442        frame: Frame,
3443    ) -> Result<Vec<Frame>, RouterError> {
3444        use subc_control::{
3445            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3446            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3447        };
3448        #[derive(Deserialize)]
3449        struct Identity {
3450            #[serde(default)]
3451            consumer_identity: Option<ConsumerIdentity>,
3452        }
3453        // A registered module connection already identifies its caller. Ignore
3454        // consumer_identity there so a module cannot borrow a mirror's identity.
3455        // Only separate consumer connections use the Direct/nonce trust boundary.
3456        let principal = match self
3457            .forwarding
3458            .module_id_for_connection(ctx.connection_id)
3459            .map_err(RouterError::Forwarding)?
3460        {
3461            Some(module_id) => Ok(Principal::Reserved { module_id }),
3462            None => {
3463                let identity: Identity = match serde_json::from_slice(&frame.body) {
3464                    Ok(identity) => identity,
3465                    Err(_) => {
3466                        return Ok(vec![control_error_frame(
3467                            &frame,
3468                            ACTIVITY_MALFORMED,
3469                            "malformed activity read",
3470                        )?])
3471                    }
3472                };
3473                self.route_open_principal(&frame, identity.consumer_identity)?
3474            }
3475        };
3476        let admitted = matches!(principal, Ok(Principal::Direct))
3477            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3478        if !admitted {
3479            return Ok(vec![control_error_frame(
3480                &frame,
3481                ACTIVITY_READ_FORBIDDEN,
3482                "activity reads require an operator or trusted mirror principal",
3483            )?]);
3484        }
3485        let Some(ledger) = &self.activity else {
3486            return Ok(vec![control_error_frame(
3487                &frame,
3488                ACTIVITY_UNAVAILABLE,
3489                "activity ledger unavailable",
3490            )?]);
3491        };
3492        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3493            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3494        {
3495            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3496                Ok(request) => ledger.query(request).await,
3497                Err(_) => Err(ACTIVITY_MALFORMED),
3498            };
3499            return match result {
3500                Ok(reply) => Ok(vec![control_response_body_frame(
3501                    &frame,
3502                    &reply,
3503                    ops::ACTIVITY_QUERY,
3504                )?]),
3505                Err(code) => Ok(vec![control_error_frame(
3506                    &frame,
3507                    code,
3508                    "activity query refused",
3509                )?]),
3510            };
3511        }
3512        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3513            Ok(request) => request,
3514            Err(_) => {
3515                return Ok(vec![control_error_frame(
3516                    &frame,
3517                    ACTIVITY_MALFORMED,
3518                    "malformed activity subscription",
3519                )?])
3520            }
3521        };
3522        let mut subscription = match ledger.subscribe(request.since).await {
3523            Ok(subscription) => subscription,
3524            Err(code) => {
3525                return Ok(vec![control_error_frame(
3526                    &frame,
3527                    code,
3528                    "activity subscription refused",
3529                )?])
3530            }
3531        };
3532        let key = (ctx.connection_id, frame.header.corr);
3533        let sink = ctx.egress.clone();
3534        let mut streams = self
3535            .activity_streams
3536            .lock()
3537            .unwrap_or_else(|p| p.into_inner());
3538        streams.retain(|_, task| !task.is_finished());
3539        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3540            return Ok(vec![control_error_frame(
3541                &frame,
3542                "duplicate_subscription",
3543                "activity subscription correlation id already in use",
3544            )?]);
3545        }
3546        let task = tokio::spawn(async move {
3547            loop {
3548                let entry = if let Some(entry) = subscription.replay.pop_front() {
3549                    Some(entry)
3550                } else {
3551                    subscription.receiver.recv().await
3552                };
3553                let Some(entry) = entry else {
3554                    break;
3555                };
3556                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3557                    return;
3558                };
3559                let Ok(next) = Frame::build_with_version(
3560                    response_version(&frame),
3561                    FrameType::StreamData,
3562                    control_flags(),
3563                    0,
3564                    0,
3565                    frame.header.corr,
3566                    body,
3567                ) else {
3568                    return;
3569                };
3570                if sink.send(next).await.is_err() {
3571                    return;
3572                }
3573            }
3574            // Dropping a full subscriber's sender drains its accepted entries
3575            // before the terminal error, preserving their order on the wire.
3576            if let Some(code) = subscription.terminal_code() {
3577                if let Ok(end) =
3578                    control_error_frame(&frame, code, "activity subscriber fell behind")
3579                {
3580                    let _ = sink.send(end).await;
3581                }
3582            }
3583        });
3584        streams.insert(key, task.abort_handle());
3585        Ok(Vec::new())
3586    }
3587
3588    async fn handle_activity_record(
3589        &self,
3590        ctx: &RouteCtx,
3591        frame: Frame,
3592    ) -> Result<Vec<Frame>, RouterError> {
3593        use subc_control::{
3594            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3595            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3596        };
3597        let result = async {
3598            let module = self
3599                .forwarding
3600                .module_id_for_connection(ctx.connection_id)
3601                .ok()
3602                .flatten()
3603                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3604            let request: ActivityRecordRequest =
3605                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3606            crate::activity::validate(&module, &request)?;
3607            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3608            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3609            let at_ms = std::time::SystemTime::now()
3610                .duration_since(std::time::UNIX_EPOCH)
3611                .unwrap_or_default()
3612                .as_millis() as u64;
3613            ledger.record(module, request, actor, at_ms).await
3614        }
3615        .await;
3616        match result {
3617            Ok(entry) => Ok(vec![control_response_body_frame(
3618                &frame,
3619                &ActivityRecordResponse { seq: entry.seq },
3620                "activity.record",
3621            )?]),
3622            Err(code) => Ok(vec![control_error_frame(
3623                &frame,
3624                code,
3625                "activity recording refused",
3626            )?]),
3627        }
3628    }
3629
3630    fn activity_actor(
3631        &self,
3632        connection: ConnectionId,
3633        module: &str,
3634        on: Option<&subc_control::ActivityOn>,
3635    ) -> Result<subc_control::ActivityActor, &'static str> {
3636        use crate::forwarding::{DataRoute, DataRouteState};
3637        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3638        let principal = |id: &str| Principal::Reserved {
3639            module_id: id.to_owned(),
3640        };
3641        let mut actor = ActivityActor {
3642            principal: principal(module),
3643            scope_owner: None,
3644            agent_id: None,
3645            run_id: None,
3646            flow_id: None,
3647            project_root: None,
3648        };
3649        let scope = match on {
3650            None => return Ok(actor),
3651            Some(ActivityOn::Scope(scope)) => {
3652                let selector = subc_protocol::scope::ScopeSelector::new(
3653                    principal(module),
3654                    scope.scope_ref.clone(),
3655                    Some(scope.epoch),
3656                );
3657                self.scopes
3658                    .read()
3659                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3660                    .attribute(&principal(module), module, &selector, true)
3661                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3662            }
3663            Some(ActivityOn::Route(route)) => {
3664                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3665                    .forwarding
3666                    .lookup_data_route(connection, route.channel, route.epoch)
3667                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3668                else {
3669                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3670                };
3671                if binding.module_id != module {
3672                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3673                }
3674                actor.principal = binding.principal.clone();
3675                actor.project_root = binding
3676                    .project_root
3677                    .as_ref()
3678                    .map(|root| root.as_path().to_path_buf());
3679                let Some(bound) = &binding.scope else {
3680                    return Ok(actor);
3681                };
3682                let owner = principal(&bound.owner);
3683                let selector = subc_protocol::scope::ScopeSelector::new(
3684                    owner.clone(),
3685                    bound.scope_ref.clone(),
3686                    Some(bound.tag.scope_epoch),
3687                );
3688                let admitted = self
3689                    .scopes
3690                    .read()
3691                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3692                    .attribute(&owner, module, &selector, true)
3693                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3694                if admitted.tag != bound.tag {
3695                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3696                }
3697                admitted
3698            }
3699        };
3700        actor.scope_owner = Some(scope.stamp.owner);
3701        actor.agent_id = scope.stamp.attributes.agent_id;
3702        actor.run_id = scope.stamp.attributes.run_id;
3703        actor.flow_id = scope.stamp.attributes.flow_id;
3704        Ok(actor)
3705    }
3706
3707    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3708        self.refresh_capability_requirements();
3709        // A bare connection count is ambiguous between many clients holding a
3710        // route each and one client accumulating hundreds, so publish the
3711        // concentration alongside it. Route state is best-effort here: a
3712        // diagnostic endpoint must still answer if the forwarding lock is
3713        // contended.
3714        let mut counters = self.counters.snapshot();
3715        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3716            obj.insert(
3717                "activity_unparseable_lines".into(),
3718                ledger.read_errors().into(),
3719            );
3720            obj.insert(
3721                "activity_lifecycle_append_failures".into(),
3722                ledger.lifecycle_failures().into(),
3723            );
3724            obj.insert(
3725                "activity_started_digest_skips".into(),
3726                ledger.lifecycle_digest_skips().into(),
3727            );
3728        }
3729        if let (Ok((connections_with_routes, max)), Some(obj)) = (
3730            self.forwarding.client_route_concentration(),
3731            counters.as_object_mut(),
3732        ) {
3733            obj.insert(
3734                "client_connections_with_routes".into(),
3735                connections_with_routes.into(),
3736            );
3737            obj.insert("max_routes_on_one_connection".into(), max.into());
3738        }
3739        // A module that is being fast-refused and a module that is fine look
3740        // identical from a client that retries and succeeds, so name the open
3741        // breakers here. This rides the existing free-form counters object
3742        // rather than a new wire field, so no sibling that deserializes
3743        // `ServerDescribe` has to be rebuilt to keep reading it.
3744        if let (Some(open_breakers), Some(obj)) = (
3745            self.route_bind_breakers.open_snapshot(),
3746            counters.as_object_mut(),
3747        ) {
3748            obj.insert("route_bind_breakers_open".into(), open_breakers);
3749        }
3750        let response = ClientControlResponse::ServerDescribe {
3751            protocol_ver: PROTOCOL_VERSION,
3752            subc_ops: subc_ops(),
3753            capabilities: self.subc_capabilities.as_ref().to_vec(),
3754            connected_clients: self.connected_clients.count(),
3755            counters: Some(counters),
3756            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3757            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3758            capability_requirements: self.capability_requirement_statuses(),
3759            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3760        };
3761        Ok(vec![control_response_body_frame(
3762            &frame,
3763            &response,
3764            "ClientControlResponse::ServerDescribe",
3765        )?])
3766    }
3767
3768    fn handle_catalog_list(
3769        &self,
3770        frame: Frame,
3771        module_id: Option<String>,
3772    ) -> Result<Vec<Frame>, RouterError> {
3773        let (generation, modules) = self.registry.list_modules().map_err(|err| {
3774            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3775        })?;
3776        let entries = modules
3777            .into_iter()
3778            .filter(|registration| {
3779                module_id
3780                    .as_deref()
3781                    .map(|wanted| registration.manifest.module_id == wanted)
3782                    .unwrap_or(true)
3783            })
3784            .map(|registration| {
3785                let not_ready = self.not_ready_reason(&registration);
3786                let roles = registration.manifest.provides;
3787                CatalogEntry::new(
3788                    registration.manifest.module_id,
3789                    roles,
3790                    registration.control_ops,
3791                )
3792                .with_ready(not_ready.is_none())
3793                .with_not_ready(not_ready)
3794                .with_module_version(Some(registration.manifest.module_version))
3795                .with_capabilities(registration.manifest.capabilities)
3796                .with_self_signals(registration.manifest.self_signals)
3797                .with_events(registration.manifest.events)
3798            })
3799            .collect();
3800        let response = ClientControlResponse::CatalogList {
3801            generation,
3802            modules: entries,
3803            subc_ops: subc_ops(),
3804        };
3805        Ok(vec![control_response_body_frame(
3806            &frame,
3807            &response,
3808            "ClientControlResponse::CatalogList",
3809        )?])
3810    }
3811
3812    fn route_open_principal(
3813        &self,
3814        frame: &Frame,
3815        consumer_identity: Option<ConsumerIdentity>,
3816    ) -> Result<Result<Principal, Frame>, RouterError> {
3817        let Some(consumer_identity) = consumer_identity else {
3818            return Ok(Ok(Principal::Direct));
3819        };
3820
3821        if self.supervisor.spawned_consumer_authorized(
3822            &consumer_identity.module_id,
3823            &consumer_identity.launch_nonce,
3824        ) {
3825            return Ok(Ok(Principal::Reserved {
3826                module_id: consumer_identity.module_id,
3827            }));
3828        }
3829
3830        Ok(Err(control_error_frame(
3831            frame,
3832            "bad_consumer_identity",
3833            format!(
3834                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3835                consumer_identity.module_id
3836            ),
3837        )?))
3838    }
3839
3840    /// Ordinary `route.open` refusals go through here; admission and breaker
3841    /// refusals log separately with their capacity or breaker state. The daemon can
3842    /// attest which code it sent: without the event, a client's "the daemon
3843    /// refused me" and the daemon's own view could only be reconciled by
3844    /// argument. Malformed input (`invalid_project_root`) does not come here;
3845    /// rejecting a request that was never a valid open is not a refusal of one.
3846    fn route_open_refusal_frame(
3847        &self,
3848        ctx: &RouteCtx,
3849        frame: &Frame,
3850        module_id: &str,
3851        reason: &'static str,
3852        code: &'static str,
3853        message: impl Into<String>,
3854    ) -> Result<Frame, RouterError> {
3855        self.observe_route_open_refusal(ctx, module_id, reason, code);
3856        control_error_frame(frame, code, message.into())
3857    }
3858
3859    /// Refuse a `route.open` because the target module's bind-relay breaker is
3860    /// open, without attempting the relay.
3861    ///
3862    /// The wire code is `module_timeout`, which is the truth (the module has
3863    /// not been answering binds) and which both SDKs already classify as
3864    /// retryable with capped backoff. Reusing it is what keeps this change out
3865    /// of both SDKs; the daemon-side distinction lives in the counter key
3866    /// instead.
3867    ///
3868    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
3869    /// While a breaker is open this fires on every open to that module, and the
3870    /// stall written up in `docs/designs/route-open-head-of-line.md` already
3871    /// produced 261 lines about a single module inside 3000 lines of daemon
3872    /// log. The rare transitions are logged at warn/info instead and the volume
3873    /// is carried by the counter, so the evidence survives without the flood.
3874    /// The debug line keeps a per-refusal record reachable for whoever turns
3875    /// the level up.
3876    fn route_open_breaker_refusal_frame(
3877        &self,
3878        ctx: &RouteCtx,
3879        frame: &Frame,
3880        module_id: &str,
3881        consecutive_timeouts: u32,
3882        retry_in: Duration,
3883        probe_in_flight: bool,
3884    ) -> Result<Frame, RouterError> {
3885        self.counters
3886            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3887        debug!(
3888            target: "control",
3889            code = "module_timeout",
3890            module_id = ?module_id,
3891            connection_id = ctx.connection_id.get(),
3892            consecutive_timeouts,
3893            retry_in_ms = retry_in.as_millis() as u64,
3894            probe_in_flight,
3895            "route.open refused by open bind-relay breaker"
3896        );
3897        // Say what a caller can act on. An open bind-relay breaker means the
3898        // module timed out accepting several new routes in a row. The module
3899        // is still running and its established routes keep working; only new
3900        // route.open requests are refused until the cooldown ends and one
3901        // test route (the probe) gets through. A message that only counts
3902        // failed relays reads as "the module is down" to a worker that sees it.
3903        let detail = if probe_in_flight {
3904            "one test route is already being tried; retry once it settles".to_string()
3905        } else {
3906            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3907        };
3908        control_error_frame(
3909            frame,
3910            "module_timeout",
3911            format!(
3912                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3913                 timed out in a row); {detail}; its established routes are unaffected"
3914            ),
3915        )
3916    }
3917
3918    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
3919    /// requester's bytes (an unknown target is whatever the client sent) and
3920    /// is Debug-formatted so control characters land in the log escaped
3921    /// rather than as terminal sequences for whoever tails it.
3922    ///
3923    /// `reason` names the check that refused, because one wire code has
3924    /// several senders: after a module registers, `target_unavailable` can
3925    /// come from a missing role, an inactive registration, a supervisor that
3926    /// has not marked the process live, a missing forwarding connection, or a
3927    /// failed relay, and a log that records only the code cannot say which of
3928    /// them fired. It is a static, daemon-chosen label per branch, so it is
3929    /// safe to print plainly and stays a closed set.
3930    fn observe_route_open_refusal(
3931        &self,
3932        ctx: &RouteCtx,
3933        module_id: &str,
3934        reason: &'static str,
3935        code: &'static str,
3936    ) {
3937        self.counters.increment_route_open_refused(code);
3938        info!(
3939            target: "control",
3940            code,
3941            reason,
3942            module_id = ?module_id,
3943            connection_id = ctx.connection_id.get(),
3944            "route.open refused"
3945        );
3946        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3947            self.route_outages.record_not_serving(module_id, reason);
3948        }
3949    }
3950
3951    /// Record an ACCEPTED route.open.
3952    ///
3953    /// Refusals have been logged and counted since the attestation work; accepts
3954    /// were invisible, so the daemon knew every principal it stamped and wrote
3955    /// none of them down. The party that attests the identity was the only party
3956    /// not recording it, which left a credential vault unable to name the sender
3957    /// of a call that reached it (claustrum #43) and left the launch-nonce
3958    /// concurrency question unanswerable from the outside.
3959    ///
3960    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
3961    /// `code`/`module_id`/`connection_id` returns both directions of the same
3962    /// decision rather than two shapes a reader has to join by hand.
3963    ///
3964    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
3965    /// and the difference carries information rather than being an
3966    /// inconsistency. This line is only reachable after a successful bind to a
3967    /// REGISTERED module, so the value has already passed HELLO validation
3968    /// including the path-hazard refusal and cannot contain control bytes. A
3969    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
3970    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
3971    ///
3972    /// Bare is also what every other daemon line already emits (`module
3973    /// registered`, `configured module supervised`). Shipping `?module_id` here
3974    /// made this instrument the only one in the file whose ids did not answer
3975    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
3976    /// line whose whole purpose is being grepped beside its sibling.
3977    ///
3978    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
3979    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
3980    /// forwards every field type through it and a bare `&str` and a `?`-escaped
3981    /// one are recorded identically. A test written against that harness passes
3982    /// either way -- I wrote one, measured it, and deleted it rather than ship a
3983    /// green assertion that cannot fail. The same limit applies to the escaping
3984    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
3985    /// it reads as a guard on the Debug escaping and cannot detect its removal.
3986    /// Fencing either needs the real formatter, not the capture layer.
3987    ///
3988    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
3989    /// unix-socket options and subc is loopback TCP, so there is no peer identity
3990    /// to record. The ephemeral port would decay within minutes and answer only a
3991    /// live question. The identity question is instead answered by counting
3992    /// distinct live connections presenting one module's `consumer_identity` --
3993    /// "is anyone else holding this secret" rather than "is this the right
3994    /// process".
3995    fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3996        self.route_outages.record_accepted(module_id);
3997        self.counters.increment_route_open_accepted(principal);
3998        info!(
3999            target: "control",
4000            principal,
4001            module_id,
4002            connection_id = ctx.connection_id.get(),
4003            "route.open accepted"
4004        );
4005    }
4006
4007    fn supervised_absent_route_open_refusal_frame(
4008        &self,
4009        ctx: &RouteCtx,
4010        frame: &Frame,
4011        module_id: &str,
4012        code: &'static str,
4013        status: &crate::supervise::ModuleStatus,
4014    ) -> Result<Frame, RouterError> {
4015        self.counters.increment_route_open_refused(code);
4016        info!(
4017            target: "control",
4018            code,
4019            reason = "supervised_not_registered",
4020            module_id = ?module_id,
4021            connection_id = ctx.connection_id.get(),
4022            state = %status.state,
4023            enabled = status.enabled,
4024            live = status.live,
4025            "route.open refused"
4026        );
4027        // A supervised module whose process has not registered is not
4028        // serving, whatever the reason; the supervisor knows this id, so it is
4029        // safe to track.
4030        self.route_outages
4031            .record_not_serving(module_id, "supervised_not_registered");
4032        let message = if let Some(pid) = status.waiting_for_previous_pid {
4033            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4034                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4035            })
4036        } else {
4037            format!(
4038                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4039                status.state, status.enabled, status.live
4040            )
4041        };
4042        control_error_frame(frame, code, message)
4043    }
4044
4045    async fn handle_route_open(
4046        &self,
4047        ctx: &RouteCtx,
4048        frame: Frame,
4049        request: RouteOpenRequest,
4050        timings: &mut RouteOpenTimings,
4051    ) -> Result<Vec<Frame>, RouterError> {
4052        let RouteOpenRequest {
4053            target,
4054            mut identity,
4055            consumer_identity,
4056            consumer_capabilities,
4057            role_versions,
4058            admission_facts,
4059            scope,
4060        } = request;
4061        let target_module_id = target_module_id(&target).to_string();
4062        timings.module_id.clone_from(&target_module_id);
4063        if self
4064            .registry
4065            .get_module_by_connection(ctx.connection_id)
4066            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4067            .is_some()
4068        {
4069            return Ok(vec![control_error_frame(
4070                &frame,
4071                "invalid_request",
4072                "module connections cannot open client routes",
4073            )?]);
4074        }
4075        debug!(
4076            connection_id = ctx.connection_id.get(),
4077            corr = frame.header.corr,
4078            module_id = %target_module_id,
4079            "handling route.open"
4080        );
4081
4082        // A malformed declaration is refused first, before anything about the
4083        // target is looked up: the same body would be refused against any
4084        // module, so the caller learns nothing by retrying or waiting. An empty
4085        // map declares nothing and travels as no field at all, so a provider
4086        // only ever sees a missing field or a non-empty one.
4087        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4088        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4089            self.observe_route_open_refusal(
4090                ctx,
4091                &target_module_id,
4092                "invalid_role_versions",
4093                error_codes::INVALID_REQUEST,
4094            );
4095            return Ok(vec![control_error_body_frame(
4096                &frame,
4097                ErrorBody {
4098                    code: error_codes::INVALID_REQUEST.to_string(),
4099                    message: error.to_string(),
4100                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4101                },
4102            )?]);
4103        }
4104
4105        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4106            if status.waiting_for_previous_pid.is_some() {
4107                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4108                    ctx,
4109                    &frame,
4110                    &target_module_id,
4111                    "module_warming",
4112                    &status,
4113                )?]);
4114            }
4115        }
4116
4117        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4118        // opposite. Below, a caller learns whether a module is unregistered,
4119        // supervised-but-down (with state/enabled/live), or registered without the
4120        // requested role. Elsewhere that is an enumeration leak: a probe learning
4121        // the shape of a fleet it cannot otherwise see.
4122        //
4123        // It is not one here, and the reason is the ACCESS MODEL rather than
4124        // anything about these errors. Reaching route.open requires the
4125        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4126        // connection file, so any caller who completes it already runs as this
4127        // user -- and can read subc.jsonc for the module list and `ck module
4128        // status` for live state. The reply discloses nothing the caller cannot
4129        // read more easily from disk, while the precision is load-bearing:
4130        // `unknown_module` is retryable and a missing role is not.
4131        //
4132        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4133        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4134        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4135        let Some(registration) = self
4136            .registry
4137            .get_module(&target_module_id)
4138            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4139        else {
4140            if let Some((status, warming)) =
4141                self.supervisor_status(&target_module_id, frame.header.corr)?
4142            {
4143                // BEFORE the two availability codes below, because for a module
4144                // that speaks no subc wire both of them are false comfort: they
4145                // say "not right now" and are retried, and this module will
4146                // never register no matter how long the caller waits. The
4147                // absence here is the declaration being honoured, not a module
4148                // that is late.
4149                if status.protocol == ModuleProtocol::None {
4150                    return Ok(vec![self.route_open_refusal_frame(
4151                        ctx,
4152                        &frame,
4153                        &target_module_id,
4154                        "protocol_none",
4155                        error_codes::MODULE_NO_PROTOCOL,
4156                        format!(
4157                            "module_id '{target_module_id}' is declared protocol: none; \
4158                             it speaks no subc wire and serves no routes"
4159                        ),
4160                    )?]);
4161                }
4162                let code = if warming {
4163                    "module_warming"
4164                } else {
4165                    "target_unavailable"
4166                };
4167                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4168                    ctx,
4169                    &frame,
4170                    &target_module_id,
4171                    code,
4172                    &status,
4173                )?]);
4174            }
4175            if let Some(removed_ago_ms) =
4176                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4177            {
4178                return Ok(vec![self.route_open_refusal_frame(
4179                    ctx,
4180                    &frame,
4181                    &target_module_id,
4182                    "removed",
4183                    error_codes::MODULE_REMOVED,
4184                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4185                )?]);
4186            }
4187            return Ok(vec![self.route_open_refusal_frame(
4188                ctx,
4189                &frame,
4190                &target_module_id,
4191                "not_registered",
4192                error_codes::UNKNOWN_MODULE,
4193                format!("module_id '{target_module_id}' is not registered"),
4194            )?]);
4195        };
4196
4197        // Best-effort only: registry readiness and forwarding reservation use
4198        // different locks, so a module can flip readiness between this read and
4199        // the relay. Modules must still tolerate an `on_bind` while not ready.
4200        if !registration.ready {
4201            self.counters
4202                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4203            info!(
4204                target: "control",
4205                code = error_codes::MODULE_WARMING,
4206                module_id = ?target_module_id,
4207                connection_id = ctx.connection_id.get(),
4208                reason = "declared_not_ready",
4209                "route.open refused"
4210            );
4211            // The module is registered but says it cannot take work, which is
4212            // an outage from the caller's side even though its process is up.
4213            self.route_outages
4214                .record_not_serving(&target_module_id, "declared_not_ready");
4215            return Ok(vec![control_error_body_frame(
4216                &frame,
4217                ErrorBody {
4218                    code: error_codes::MODULE_WARMING.to_string(),
4219                    message: format!(
4220                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4221                    ),
4222                    detail: Some(serde_json::json!({
4223                        "reason": "declared_not_ready"
4224                    })),
4225                },
4226            )?]);
4227        }
4228
4229        // Effective readiness, second half: a module that declares a capability
4230        // `need: required` is not routable while that capability has no
4231        // registered provider. It is enforced HERE, as a retryable routing
4232        // refusal, and deliberately not as spawn ordering or a boot block. The
4233        // module is still started and registered and can make its own calls;
4234        // spawn ordering is a promise that cannot be kept once a provider
4235        // crashes at runtime, and refusing to boot would stop the whole
4236        // machine, including the tools needed to fix its configuration.
4237        //
4238        // "Provided" is the evaluator's verdict, which counts a provider as
4239        // soon as it has REGISTERED, not once it is ready. Two modules that
4240        // require each other's capabilities are therefore both routable once
4241        // both register; counting readiness instead would deadlock them.
4242        //
4243        // Only new opens are refused. Routes already bound when a provider
4244        // goes away stay bound: nothing here tears them down, and the module
4245        // answers them as it can. Like the readiness read above this is
4246        // best-effort against a provider registering or leaving concurrently.
4247        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
4248            self.counters
4249                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4250            info!(
4251                target: "control",
4252                code = error_codes::MODULE_WARMING,
4253                module_id = ?target_module_id,
4254                connection_id = ctx.connection_id.get(),
4255                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4256                capability = %capability,
4257                "route.open refused"
4258            );
4259            return Ok(vec![control_error_body_frame(
4260                &frame,
4261                ErrorBody {
4262                    code: error_codes::MODULE_WARMING.to_string(),
4263                    message: format!(
4264                        "module_id '{target_module_id}' requires capability '{capability}', \
4265                         which no registered module provides; retry"
4266                    ),
4267                    detail: Some(serde_json::json!({
4268                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4269                        "capability": capability,
4270                    })),
4271                },
4272            )?]);
4273        }
4274
4275        if !target_has_required_role(&target, &registration.manifest.provides) {
4276            return Ok(vec![self.route_open_refusal_frame(
4277                ctx,
4278                &frame,
4279                &target_module_id,
4280                "role_not_provided",
4281                "target_unavailable",
4282                format!("module_id '{target_module_id}' does not provide the requested target"),
4283            )?]);
4284        }
4285
4286        if registration.state != ChannelState::Active {
4287            return Ok(vec![self.route_open_refusal_frame(
4288                ctx,
4289                &frame,
4290                &target_module_id,
4291                "registration_not_active",
4292                "target_unavailable",
4293                format!("module_id '{target_module_id}' is not active"),
4294            )?]);
4295        }
4296
4297        if self
4298            .forwarding
4299            .module_is_draining(&target_module_id)
4300            .map_err(RouterError::Forwarding)?
4301        {
4302            return Ok(vec![self.route_open_refusal_frame(
4303                ctx,
4304                &frame,
4305                &target_module_id,
4306                "reloading",
4307                "module_reloading",
4308                format!("module_id '{target_module_id}' is reloading"),
4309            )?]);
4310        }
4311
4312        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4313            process_liveness.process_live(&target_module_id) == Some(false)
4314        }) {
4315            // A module the supervisor is restarting or reloading can still hold
4316            // a registration: the old process before its connection closes, or
4317            // a new one that registered while the supervisor was draining. The
4318            // forwarding table does not see that as draining, but the consumer
4319            // should still be told to retry soon, exactly as for the drain
4320            // above, rather than that the target is unavailable.
4321            if process_liveness.process_replacing(&target_module_id) {
4322                return Ok(vec![self.route_open_refusal_frame(
4323                    ctx,
4324                    &frame,
4325                    &target_module_id,
4326                    "reloading",
4327                    "module_reloading",
4328                    format!("module_id '{target_module_id}' is reloading"),
4329                )?]);
4330            }
4331            return Ok(vec![self.route_open_refusal_frame(
4332                ctx,
4333                &frame,
4334                &target_module_id,
4335                "supervisor_not_live",
4336                "target_unavailable",
4337                format!("module_id '{target_module_id}' is not live"),
4338            )?]);
4339        }
4340
4341        if !self
4342            .forwarding
4343            .has_live_module_connection(&target_module_id)
4344            .map_err(RouterError::Forwarding)?
4345        {
4346            return Ok(vec![self.route_open_refusal_frame(
4347                ctx,
4348                &frame,
4349                &target_module_id,
4350                "no_forwarding_connection",
4351                "target_unavailable",
4352                format!("module_id '{target_module_id}' has no live forwarding connection"),
4353            )?]);
4354        }
4355
4356        if let Some(error) =
4357            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4358        {
4359            self.observe_route_open_refusal(
4360                ctx,
4361                &target_module_id,
4362                "op_not_allowed",
4363                "op_not_allowed",
4364            );
4365            return Ok(vec![error]);
4366        }
4367
4368        let principal = match self.route_open_principal(&frame, consumer_identity)? {
4369            Ok(principal) => principal,
4370            Err(error) => {
4371                self.observe_route_open_refusal(
4372                    ctx,
4373                    &target_module_id,
4374                    "bad_consumer_identity",
4375                    "bad_consumer_identity",
4376                );
4377                return Ok(vec![error]);
4378            }
4379        };
4380        timings.principal = principal_label(&principal);
4381
4382        // This is attested, control-plane policy for supervised module origins.
4383        // Keep it before route reservation and out of the opaque forwarding hot
4384        // path: data frames must never acquire a per-frame capability check.
4385        if let Principal::Reserved {
4386            module_id: opening_module_id,
4387        } = &principal
4388        {
4389            if let Some(opening_registration) = self
4390                .registry
4391                .get_module(opening_module_id)
4392                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4393            {
4394                if let Some(capability) =
4395                    denied_capability(&opening_registration.manifest, &registration.manifest)
4396                {
4397                    warn!(
4398                        opening_module_id,
4399                        target_module_id,
4400                        capability,
4401                        "refusing route.open because an attested capability deny edge matches"
4402                    );
4403                    return Ok(vec![self.route_open_refusal_frame(
4404                        ctx,
4405                        &frame,
4406                        &target_module_id,
4407                        "capability_deny_edge",
4408                        "capability_forbidden",
4409                        format!(
4410                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4411                        ),
4412                    )?]);
4413                }
4414            }
4415        }
4416
4417        if admission_facts.is_some() {
4418            let carrier_matches = matches!(
4419                &principal,
4420                Principal::Reserved { module_id }
4421                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4422            );
4423            if !carrier_matches {
4424                return Ok(vec![self.route_open_refusal_frame(
4425                    ctx,
4426                    &frame,
4427                    &target_module_id,
4428                    "admission_facts_carrier_not_permitted",
4429                    "admission_facts_not_permitted",
4430                    "admission facts may only be carried by the configured reserved module",
4431                )?]);
4432            }
4433
4434            let target_allowed = self
4435                .admission_facts_targets
4436                .as_ref()
4437                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4438            if !target_allowed {
4439                return Ok(vec![self.route_open_refusal_frame(
4440                    ctx,
4441                    &frame,
4442                    &target_module_id,
4443                    "admission_facts_target_not_listed",
4444                    "admission_facts_target_not_allowed",
4445                    format!(
4446                        "admission facts are not permitted for target module_id '{target_module_id}'"
4447                    ),
4448                )?]);
4449            }
4450
4451            // Keep the value opaque to subc. The downstream admission validator owns
4452            // schema and semantic checks; this daemon only enforces carrier authority
4453            // and the configured destination allowlist.
4454        }
4455
4456        // Scope admission, on the attested principal above and never on the
4457        // request body. The tag read here travels with the pending bind and is
4458        // compared with the published one at commit, so a sync between here
4459        // and the module's ack refuses the open instead of binding a stamp
4460        // that is no longer true.
4461        let (bound_scope, mut scope_stamp) = match &scope {
4462            None => (None, None),
4463            Some(selector) => {
4464                let owner_configured = match &selector.owner {
4465                    // A reserved owner counts as configured from before its
4466                    // process is spawned (see `SupervisorHandle::is_configured`).
4467                    // So an owner that has not synced its scopes yet is refused
4468                    // as retryable (`scope_not_synced`), not as one that will
4469                    // never sync.
4470                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4471                    _ => false,
4472                };
4473                let mut scopes = self
4474                    .scopes
4475                    .write()
4476                    .unwrap_or_else(|poisoned| poisoned.into_inner());
4477                let admitted = scopes
4478                    .admit(&principal, &target_module_id, selector, owner_configured)
4479                    .and_then(|admission| {
4480                        crate::scopes::check_target_flow_support(
4481                            &admission.stamp,
4482                            &target_module_id,
4483                            registration.manifest.capabilities.as_ref(),
4484                        )?;
4485                        crate::scopes::check_target_agent_run_support(
4486                            &admission.stamp,
4487                            &target_module_id,
4488                            registration.manifest.capabilities.as_ref(),
4489                        )?;
4490                        let bound = scopes.reserve_bind(
4491                            &admission,
4492                            &principal,
4493                            ctx.connection_id,
4494                            selector.hold_presence,
4495                        );
4496                        Ok((admission, bound))
4497                    });
4498                match admitted {
4499                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4500                    Err(refusal) => {
4501                        return Ok(vec![self.route_open_refusal_frame(
4502                            ctx,
4503                            &frame,
4504                            &target_module_id,
4505                            refusal.code,
4506                            refusal.code,
4507                            refusal.message,
4508                        )?]);
4509                    }
4510                }
4511            }
4512        };
4513
4514        // Bind admits a root that no longer exists on disk, because refusing here
4515        // closes the only exit from a paused run: cancel needs a bound route, and a
4516        // renamed or reclaimed directory makes that route unopenable forever. The
4517        // run itself is intact and still addressable by its recorded identity.
4518        //
4519        // This does NOT relax the rule the strict constructor protects. That rule is
4520        // that no root is ever aliased into NEW durable state -- a missing component
4521        // can reappear as a symlink elsewhere, which would move the identity and
4522        // split a session's history across two of them. The engine now refuses the
4523        // two operations that create such state (send and import) at admission,
4524        // which is a narrower way to hold the same invariant: reads and terminations
4525        // are admitted, writes are not. That refusal had to ship before this line
4526        // changed, or there is an interval where a send commits under a provisional
4527        // identity -- the exact failure the original policy existed to prevent.
4528        //
4529        // Resolution follows realpath rather than lexical cleanup: the longest
4530        // existing ancestor is canonicalized and the missing tail re-appended, so a
4531        // live root is unchanged and a vanished leaf keeps the identity it was
4532        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
4533        // same caller the moment the directory vanished, which strands the run more
4534        // quietly than refusing it.
4535        timings.enter(RouteOpenPhase::ProjectRoot);
4536        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4537            Ok(project_root) => project_root,
4538            Err(err) => {
4539                return Ok(vec![control_error_frame(
4540                    &frame,
4541                    "invalid_project_root",
4542                    err.to_string(),
4543                )?])
4544            }
4545        };
4546        timings.enter(RouteOpenPhase::Admission);
4547        identity.project_root = project_root.as_path().to_path_buf();
4548
4549        // Last gate before any relay work, and deliberately after the cheap
4550        // registry and availability checks above: those name a more precise
4551        // condition (unknown, removed, reloading) and a caller is better served
4552        // by the precise code than by this one.
4553        //
4554        // Everything below this point costs an egress permit, a reserved handle
4555        // pair and, if the module does not answer, the whole relay budget. The
4556        // reader no longer waits for that budget, so cap each target explicitly;
4557        // serial dispatch used to provide the accidental cap of one relay per
4558        // connection. Admission is a mutex-protected count and never waits.
4559        let _concurrency_guard = match self
4560            .route_bind_concurrency
4561            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4562        {
4563            Ok(guard) => guard,
4564            Err(in_flight) => {
4565                return Ok(vec![self.route_open_target_capacity_refusal(
4566                    ctx,
4567                    &frame,
4568                    &target_module_id,
4569                    in_flight,
4570                )?]);
4571            }
4572        };
4573
4574        // A module that has already burned the whole budget `threshold` times
4575        // in a row does not get to charge it again until a probe says it recovered.
4576        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4577            RouteBindAdmission::Admitted { guard, probe } => {
4578                if probe {
4579                    info!(
4580                        module_id = %target_module_id,
4581                        connection_id = ctx.connection_id.get(),
4582                        "route.bind breaker half-open: admitting one probe"
4583                    );
4584                }
4585                guard
4586            }
4587            RouteBindAdmission::Refused {
4588                consecutive_timeouts,
4589                retry_in,
4590                probe_in_flight,
4591            } => {
4592                return Ok(vec![self.route_open_breaker_refusal_frame(
4593                    ctx,
4594                    &frame,
4595                    &target_module_id,
4596                    consecutive_timeouts,
4597                    retry_in,
4598                    probe_in_flight,
4599                )?]);
4600            }
4601        };
4602
4603        // Resolve the per-module budget here so the wait matches the operator's
4604        // intent for this specific target. A per-module override in
4605        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
4606        // daemons) wins over the daemon-wide default.
4607        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4608        let relay_deadline = Instant::now() + route_bind_relay_timeout;
4609        timings.enter(RouteOpenPhase::EgressReservation);
4610        let pending = match self
4611            .forwarding
4612            .begin_route_bind_relay_for(
4613                ctx.connection_id,
4614                ctx.egress.clone(),
4615                response_version(&frame),
4616                frame.header.corr,
4617                &target_module_id,
4618                principal.clone(),
4619                bound_scope,
4620                Some(project_root),
4621                relay_deadline,
4622            )
4623            .await
4624        {
4625            Ok(pending) => pending,
4626            Err(err) => {
4627                return Ok(vec![self.route_open_refusal_frame(
4628                    ctx,
4629                    &frame,
4630                    &target_module_id,
4631                    "relay_reservation_failed",
4632                    forwarding_error_code(&err),
4633                    err.to_string(),
4634                )?])
4635            }
4636        };
4637        timings.enter(RouteOpenPhase::Admission);
4638        let crate::forwarding::PendingRouteBindRelay {
4639            endpoint,
4640            module_sink,
4641            negotiated_ver,
4642            client_channel,
4643            client_epoch,
4644            module_channel,
4645            module_epoch,
4646            corr: relay_corr,
4647            receiver,
4648        } = pending;
4649        let mut reservation =
4650            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4651
4652        // Reserving egress can wait while a module reconnects or a swap cuts
4653        // over. Check the relay's captured connection, not the earlier by-id
4654        // lookup: the original module's flow/run capabilities cannot authorize
4655        // its replacement. The captured sink stays bound to that connection.
4656        let relay_registration = self
4657            .registry
4658            .get_module_by_connection(endpoint.connection_id)
4659            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4660        let relay_capabilities = relay_registration
4661            .as_ref()
4662            .and_then(|registration| registration.manifest.capabilities.as_ref());
4663        if let Some(selector) = &scope {
4664            // Egress reservation may wait across a scope update. Recheck the
4665            // proof against the current record before sending the captured bind.
4666            if let Err(refusal) = self
4667                .scopes
4668                .read()
4669                .unwrap_or_else(|poisoned| poisoned.into_inner())
4670                .check_bind_proof(&principal, selector)
4671            {
4672                reservation.release_and_disarm();
4673                return Ok(vec![self.route_open_refusal_frame(
4674                    ctx,
4675                    &frame,
4676                    &target_module_id,
4677                    refusal.code,
4678                    refusal.code,
4679                    refusal.message,
4680                )?]);
4681            }
4682        }
4683        if let Some(stamp) = scope_stamp
4684            .as_ref()
4685            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4686        {
4687            if let Err(refusal) = crate::scopes::check_target_flow_support(
4688                stamp,
4689                &target_module_id,
4690                relay_capabilities,
4691            )
4692            .and_then(|()| {
4693                crate::scopes::check_target_agent_run_support(
4694                    stamp,
4695                    &target_module_id,
4696                    relay_capabilities,
4697                )
4698            }) {
4699                reservation.release_and_disarm();
4700                return Ok(vec![self.route_open_refusal_frame(
4701                    ctx,
4702                    &frame,
4703                    &target_module_id,
4704                    refusal.code,
4705                    refusal.code,
4706                    refusal.message,
4707                )?]);
4708            }
4709        }
4710
4711        // Older readers may reject unknown fields. Select both new fields from
4712        // the captured endpoint's declaration, including on unscoped binds.
4713        let call_mode = if relay_capabilities
4714            .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4715        {
4716            Some(derive_call_mode(scope_stamp.as_ref()))
4717        } else {
4718            if let Some(stamp) = scope_stamp.as_mut() {
4719                stamp.expires_at_ms = None;
4720            }
4721            None
4722        };
4723
4724        debug!(
4725            connection_id = ctx.connection_id.get(),
4726            client_channel,
4727            client_epoch,
4728            module_channel,
4729            module_epoch,
4730            "reserved route handle pair"
4731        );
4732        // Rendered BEFORE the move into the relay, because the accept arm below
4733        // is where it is logged and the principal is gone by then.
4734        let principal_label = principal_label(&principal);
4735        let relay = ModuleControlRequest::RouteBind {
4736            route_channel: module_channel,
4737            epoch: module_epoch,
4738            target,
4739            identity,
4740            principal: Some(principal),
4741            consumer_capabilities,
4742            role_versions,
4743            admission_facts,
4744            scope: scope_stamp,
4745            call_mode,
4746        };
4747        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4748            RouterError::backend(
4749                0,
4750                frame.header.corr,
4751                format!("failed to encode route.bind request: {err}"),
4752            )
4753        })?;
4754        let relay_frame = Frame::build_with_version(
4755            negotiated_ver,
4756            FrameType::Request,
4757            control_flags(),
4758            0,
4759            0,
4760            relay_corr,
4761            relay_body,
4762        )
4763        .map_err(RouterError::FrameBuild)?;
4764
4765        timings.enter(RouteOpenPhase::BindSend);
4766        let sent = module_sink.send(relay_frame).await;
4767        timings.enter(RouteOpenPhase::Admission);
4768        if let Err(err) = sent {
4769            reservation.release_and_disarm();
4770            return Ok(vec![self.route_open_refusal_frame(
4771                ctx,
4772                &frame,
4773                &target_module_id,
4774                "relay_send_failed",
4775                "target_unavailable",
4776                err.to_string(),
4777            )?]);
4778        }
4779
4780        #[cfg(test)]
4781        if let Some(hold) = &self.route_bind_relay_sent_hold {
4782            // First wait: tell the test the bind is queued. Second wait: stay
4783            // here until the test has driven whatever it wants to happen
4784            // between queueing the bind and marking it enqueued.
4785            hold.wait().await;
4786            hold.wait().await;
4787        }
4788
4789        // A missing relay is not by itself an abandoned bind: the module may
4790        // already have acked it and the route may be committed. Only a relay
4791        // dropped without any module reply needs this GOODBYE; one a reply
4792        // settled has already had its GOODBYE decided by the reply path.
4793        if self
4794            .forwarding
4795            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4796            .map_err(RouterError::Forwarding)?
4797            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4798        {
4799            self.send_abandoned_route_bind_goodbye(
4800                &module_sink,
4801                negotiated_ver,
4802                module_channel,
4803                module_epoch,
4804            );
4805        }
4806
4807        timings.enter(RouteOpenPhase::BindAckWait);
4808        let outcome = timeout_at(relay_deadline, receiver).await;
4809        timings.enter(RouteOpenPhase::Admission);
4810        match outcome {
4811            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4812                reservation.disarm();
4813                if breaker.record_accepted() {
4814                    info!(
4815                        module_id = %target_module_id,
4816                        "route.bind breaker closed: the probe was accepted"
4817                    );
4818                }
4819                self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4820                Ok(Vec::new())
4821            }
4822            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4823                reservation.release_and_disarm();
4824                // A module that says no in microseconds is healthy. Rejection
4825                // is a different condition with its own refusal and must not
4826                // move the breaker.
4827                breaker.record_inconclusive();
4828                // The daemon's own commit re-check refused the bind because the
4829                // scope ended or changed after admission. The module accepted;
4830                // counting it as a module rejection would blame the module.
4831                let scope_code = match body.code.as_str() {
4832                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4833                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4834                    _ => None,
4835                };
4836                if let Some(code) = scope_code {
4837                    self.observe_route_open_refusal(
4838                        ctx,
4839                        &target_module_id,
4840                        "scope_changed_before_commit",
4841                        code,
4842                    );
4843                    return Ok(vec![control_error_body_frame(&frame, body)?]);
4844                }
4845                self.counters
4846                    .increment_route_open_refused("module_rejected");
4847                info!(
4848                    target: "control",
4849                    code = "module_rejected",
4850                    module_code = ?body.code,
4851                    module_id = ?target_module_id,
4852                    connection_id = ctx.connection_id.get(),
4853                    "route.open refused"
4854                );
4855                Ok(vec![control_error_body_frame(&frame, body)?])
4856            }
4857            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4858                reservation.release_and_disarm();
4859                breaker.record_inconclusive();
4860                // Fires when the module's connection closes while a relayed
4861                // bind is pending -- typically a caller racing a module restart
4862                // whose bind was relayed BEFORE the drain mark went up. Logged
4863                // because the caller sees only its own error and the fleet has
4864                // already spent one diagnosis round unable to tell this arm
4865                // from a relay timeout without daemon-side evidence.
4866                tracing::warn!(
4867                    module_id = %target_module_id,
4868                    "route.bind relay abandoned: {message}"
4869                );
4870                Ok(vec![self.route_open_refusal_frame(
4871                    ctx,
4872                    &frame,
4873                    &target_module_id,
4874                    "relay_abandoned",
4875                    "target_unavailable",
4876                    message,
4877                )?])
4878            }
4879            Ok(Err(_)) => {
4880                reservation.release_and_disarm();
4881                breaker.record_inconclusive();
4882                Ok(vec![self.route_open_refusal_frame(
4883                    ctx,
4884                    &frame,
4885                    &target_module_id,
4886                    "relay_waiter_canceled",
4887                    "target_unavailable",
4888                    "route.bind relay waiter was canceled before the module responded",
4889                )?])
4890            }
4891            Err(_) => {
4892                reservation.release_and_disarm();
4893                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
4894                // answer at all is the one condition a fast refusal can
4895                // usefully stand in for; every other arm already answered.
4896                if let Some(opened) = breaker.record_timeout(
4897                    self.route_bind_breaker_threshold,
4898                    self.route_bind_breaker_cooldown,
4899                ) {
4900                    warn!(
4901                        module_id = %target_module_id,
4902                        consecutive_timeouts = opened.consecutive_timeouts,
4903                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4904                        reopened_after_probe = opened.reopened_after_probe,
4905                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4906                    );
4907                }
4908                // The generous budget just burned to no answer: the module is
4909                // registered and its connection is up, but its bind handler sat
4910                // on the ack for the full budget (warm-on-bind, cold configure,
4911                // or a wedged handler). Every earlier unavailability shape
4912                // fast-refuses BEFORE the relay, so this arm firing means the
4913                // slowness is module-side -- log it so the per-module timeline
4914                // is reconstructable without client audit rows.
4915                tracing::warn!(
4916                    module_id = %target_module_id,
4917                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4918                    "route.bind relay timed out: module did not ack within budget"
4919                );
4920                Ok(vec![self.route_open_refusal_frame(
4921                    ctx,
4922                    &frame,
4923                    &target_module_id,
4924                    "relay_timed_out",
4925                    "module_timeout",
4926                    format!(
4927                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
4928                        route_bind_relay_timeout
4929                    ),
4930                )?])
4931            }
4932        }
4933    }
4934
4935    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4936        let response = ClientControlResponse::SupervisorSpawnSnapshot {
4937            snapshot: self.supervisor.spawn_snapshot(),
4938        };
4939        Ok(vec![control_response_body_frame(
4940            &frame,
4941            &response,
4942            "ClientControlResponse::SupervisorSpawnSnapshot",
4943        )?])
4944    }
4945
4946    fn handle_supervisor_spawn_subscribe(
4947        &self,
4948        ctx: &RouteCtx,
4949        frame: Frame,
4950        since: Option<SpawnCursor>,
4951    ) -> Result<Vec<Frame>, RouterError> {
4952        match self.supervisor.subscribe_spawns(
4953            ctx.connection_id,
4954            frame.header.corr,
4955            response_version(&frame),
4956            since,
4957            ctx.egress.clone(),
4958        ) {
4959            Ok(()) => Ok(Vec::new()),
4960            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4961                Ok(vec![control_error_body_frame(
4962                    &frame,
4963                    ErrorBody {
4964                        code: "spawn_cursor_incarnation_mismatch".to_string(),
4965                        message: "spawn cursor belongs to a different daemon incarnation"
4966                            .to_string(),
4967                        detail: Some(serde_json::json!({
4968                            "current_daemon_incarnation": current
4969                        })),
4970                    },
4971                )?])
4972            }
4973            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4974                &frame,
4975                ErrorBody {
4976                    code: "spawn_cursor_too_old".to_string(),
4977                    message: "spawn cursor predates the retained event ring".to_string(),
4978                    detail: Some(serde_json::json!({
4979                        "oldest_retained_cursor": oldest
4980                    })),
4981                },
4982            )?]),
4983            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4984                0,
4985                frame.header.corr,
4986                format!("failed to open supervisor spawn subscription: {error}"),
4987            )),
4988        }
4989    }
4990
4991    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4992        let generation = self
4993            .registry
4994            .generation()
4995            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4996        let mut modules = Vec::new();
4997        for module in self.supervisor.list() {
4998            let status = module.status_for_control("list").map_err(|err| {
4999                RouterError::backend(
5000                    0,
5001                    frame.header.corr,
5002                    format!("failed to read supervisor status: {err}"),
5003                )
5004            })?;
5005            let (configured, _) = module.configuration().map_err(|err| {
5006                RouterError::backend(
5007                    0,
5008                    frame.header.corr,
5009                    format!("failed to read module configuration: {err}"),
5010                )
5011            })?;
5012            // Status and configuration snapshots release their locks before the image probe awaits.
5013            let image = module.running_image_agreement().await;
5014            // Read per request so the figure is current when the operator asks;
5015            // the daemon samples nothing in between.
5016            let resources = Some(module.child_resource_usage());
5017            let mut verdict =
5018                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5019            verdict.spec = Some(launch_spec_agreement(
5020                &configured,
5021                status.process_alive,
5022                status.launch_spec_digest.as_ref(),
5023            ));
5024            let pending_reload = Some(verdict);
5025            // Keep the retired policy field on the wire for one release so
5026            // existing status consumers still receive the platform policy.
5027            modules.push(
5028                SupervisorEntry::new(
5029                    status.module_id,
5030                    status.state.to_string(),
5031                    status.enabled,
5032                    status.live,
5033                    status.health.status,
5034                )
5035                .with_launch_nonce_env(Some(!cfg!(unix)))
5036                .with_launch_nonce_source(status.launch_nonce_source)
5037                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5038                .with_protocol(status.protocol)
5039                .with_pending_reload(pending_reload)
5040                .with_last_probe_ms(status.health.last_probe_ms)
5041                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5042                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5043                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5044                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5045                .with_restart_count(Some(status.restart_count))
5046                .with_max_restarts(Some(status.max_restarts))
5047                .with_lifetime_restarts(Some(status.lifetime_restarts))
5048                .with_spawn_generation(Some(status.spawn_generation))
5049                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5050                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5051                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5052                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5053                .with_resources(resources),
5054            );
5055        }
5056        let response = ClientControlResponse::SupervisorList {
5057            generation,
5058            modules,
5059        };
5060        Ok(vec![control_response_body_frame(
5061            &frame,
5062            &response,
5063            "ClientControlResponse::SupervisorList",
5064        )?])
5065    }
5066
5067    fn handle_supervisor_stderr_tail(
5068        &self,
5069        frame: Frame,
5070        module_id: String,
5071        max_lines: Option<u32>,
5072        max_bytes: Option<u32>,
5073    ) -> Result<Vec<Frame>, RouterError> {
5074        let Some(module) = self.supervisor.get(&module_id) else {
5075            return Ok(vec![control_error_frame(
5076                &frame,
5077                "unknown_module",
5078                format!("module_id '{module_id}' is not supervised"),
5079            )?]);
5080        };
5081
5082        let snapshot = module.stderr_tail(
5083            max_lines.map(|value| value as usize),
5084            max_bytes.map(|value| value as usize),
5085        );
5086
5087        let response = ClientControlResponse::SupervisorStderrTail {
5088            module_id,
5089            tail: StderrTail {
5090                capture: match snapshot.capture {
5091                    CaptureState::Captured => StderrCaptureState::Captured,
5092                    CaptureState::Incomplete { reason } => {
5093                        StderrCaptureState::Incomplete { reason }
5094                    }
5095                    CaptureState::NotCaptured { reason } => {
5096                        StderrCaptureState::NotCaptured { reason }
5097                    }
5098                },
5099                entries: snapshot
5100                    .entries
5101                    .into_iter()
5102                    .map(|entry| match entry {
5103                        TailEntry::Line {
5104                            text,
5105                            truncated,
5106                            at_ms,
5107                        } => StderrTailEntry::Line {
5108                            text,
5109                            truncated,
5110                            at_ms,
5111                        },
5112                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5113                    })
5114                    .collect(),
5115                dropped_lines: snapshot.dropped_lines,
5116            },
5117        };
5118        Ok(vec![control_response_body_frame(
5119            &frame,
5120            &response,
5121            "ClientControlResponse::SupervisorStderrTail",
5122        )?])
5123    }
5124
5125    async fn handle_supervisor_terminals(
5126        &self,
5127        frame: Frame,
5128        module_id: String,
5129    ) -> Result<Vec<Frame>, RouterError> {
5130        let Some(module) = self.supervisor.get(&module_id) else {
5131            return Ok(vec![control_error_frame(
5132                &frame,
5133                "unknown_module",
5134                format!("module_id '{module_id}' is not supervised"),
5135            )?]);
5136        };
5137
5138        // The journal read runs on a blocking thread: it can be megabytes of
5139        // file I/O and must not occupy a runtime worker.
5140        let terminals = module
5141            .read_durable_terminal_history()
5142            .await
5143            .map_err(|error| {
5144                RouterError::backend(
5145                    0,
5146                    frame.header.corr,
5147                    format!("failed to read terminal history: {error}"),
5148                )
5149            })?;
5150        let response = ClientControlResponse::SupervisorTerminals {
5151            module_id,
5152            terminals,
5153        };
5154        Ok(vec![control_response_body_frame(
5155            &frame,
5156            &response,
5157            "ClientControlResponse::SupervisorTerminals",
5158        )?])
5159    }
5160
5161    fn handle_supervisor_routes(
5162        &self,
5163        frame: Frame,
5164        module_id: Option<String>,
5165    ) -> Result<Vec<Frame>, RouterError> {
5166        let modules = self
5167            .forwarding
5168            .route_census(module_id.as_deref())
5169            .map_err(RouterError::Forwarding)?
5170            .into_iter()
5171            .map(|(module_id, routes)| SupervisorRouteModule {
5172                module_id,
5173                routes: routes
5174                    .into_iter()
5175                    .map(|route| SupervisorRoute {
5176                        consumer: match route.principal {
5177                            Principal::Reserved { module_id } => {
5178                                SupervisorRouteConsumer::Reserved { module_id }
5179                            }
5180                            Principal::Direct | Principal::Unverified => {
5181                                SupervisorRouteConsumer::Direct {
5182                                    connection_id: route.goodbye_target.connection_id.get(),
5183                                }
5184                            }
5185                        },
5186                        age_ms: Instant::now()
5187                            .saturating_duration_since(route.bound_at)
5188                            .as_millis()
5189                            .try_into()
5190                            .unwrap_or(u64::MAX),
5191                        draining: route.draining,
5192                        drain_reason: route.drain_reason,
5193                    })
5194                    .collect(),
5195            })
5196            .collect();
5197        let response = ClientControlResponse::SupervisorRoutes { modules };
5198        Ok(vec![control_response_body_frame(
5199            &frame,
5200            &response,
5201            "ClientControlResponse::SupervisorRoutes",
5202        )?])
5203    }
5204
5205    async fn handle_supervisor_provenance(
5206        &self,
5207        frame: Frame,
5208        module_id: Option<String>,
5209    ) -> Result<Vec<Frame>, RouterError> {
5210        let mut selected = if let Some(module_id) = module_id {
5211            let Some(module) = self.supervisor.get(&module_id) else {
5212                return Ok(vec![control_error_frame(
5213                    &frame,
5214                    "unknown_module",
5215                    format!("module_id '{module_id}' is not supervised"),
5216                )?]);
5217            };
5218            vec![module]
5219        } else {
5220            self.supervisor.list()
5221        };
5222
5223        let mut modules = Vec::with_capacity(selected.len());
5224        for module in selected.drain(..) {
5225            let (status, observed_image) = module
5226                .status_and_running_image_agreement()
5227                .await
5228                .map_err(|err| {
5229                    RouterError::backend(
5230                        0,
5231                        frame.header.corr,
5232                        format!("failed to read supervisor status: {err}"),
5233                    )
5234                })?;
5235            let module_declared = self
5236                .registry
5237                .get_module(&status.module_id)
5238                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5239                .and_then(|registration| registration.manifest.provenance)
5240                .map(|build| ModuleDeclaredProvenance::Reported { build })
5241                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5242            #[cfg(test)]
5243            let running_image = match &self.provenance_probe_override {
5244                Some(result) => result.clone(),
5245                None => observed_image,
5246            };
5247            #[cfg(not(test))]
5248            let running_image = observed_image;
5249            modules.push(SupervisorModuleProvenance {
5250                module_id: status.module_id,
5251                module_declared,
5252                daemon_observed: SupervisorObservedProcess {
5253                    launch_nonce_source: status.launch_nonce_source,
5254                    pid: status.pid,
5255                    spawned_at_ms: status.spawned_at_ms,
5256                    spawned_from: status.spawned_from,
5257                    running_image,
5258                },
5259            });
5260        }
5261        let daemon = SupervisorDaemonProvenance {
5262            daemon_build: self.daemon_provenance.build.clone(),
5263            daemon_observed: DaemonObservedProcess {
5264                pid: self.daemon_provenance.pid,
5265                started_at_ms: self
5266                    .daemon_provenance
5267                    .start_clock
5268                    .map(|clock| clock.started_at_ms())
5269                    .or(self.daemon_provenance.started_at_ms),
5270                running_image: self
5271                    .daemon_provenance
5272                    .probe
5273                    .observe(
5274                        self.daemon_provenance.pid,
5275                        self.daemon_provenance.executable_path.as_deref(),
5276                        self.daemon_provenance.executable_identity.clone(),
5277                        self.daemon_provenance.process_start_time,
5278                    )
5279                    .await,
5280            },
5281        };
5282        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5283        Ok(vec![control_response_body_frame(
5284            &frame,
5285            &response,
5286            "ClientControlResponse::SupervisorProvenance",
5287        )?])
5288    }
5289
5290    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5291        self.refresh_capability_requirements();
5292        let generation = self
5293            .registry
5294            .generation()
5295            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5296        let modules = self
5297            .supervisor
5298            .list()
5299            .into_iter()
5300            .map(|module| {
5301                let status = module.status_for_control("health").map_err(|err| {
5302                    RouterError::backend(
5303                        0,
5304                        frame.header.corr,
5305                        format!("failed to read supervisor health: {err}"),
5306                    )
5307                })?;
5308                let module_id = status.module_id;
5309                let capability_detail = self
5310                    .capability_evaluator
5311                    .required_problem_detail(&module_id);
5312                Ok(SupervisorHealthEntry {
5313                    module_id,
5314                    status: status.health.status,
5315                    detail: append_capability_problem_detail(
5316                        status.health.detail,
5317                        capability_detail,
5318                    ),
5319                    metrics: status.health.metrics,
5320                    consecutive_failures: status.health.consecutive_failures,
5321                    late_answer_count: status.health.late_answer_count,
5322                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5323                    last_action: status.health.last_action,
5324                    last_action_ms: status.health.last_action_ms,
5325                    last_probe_ms: status.health.last_probe_ms,
5326                })
5327            })
5328            .collect::<Result<Vec<_>, RouterError>>()?;
5329        let response = ClientControlResponse::SupervisorHealth {
5330            generation,
5331            modules,
5332        };
5333        Ok(vec![control_response_body_frame(
5334            &frame,
5335            &response,
5336            "ClientControlResponse::SupervisorHealth",
5337        )?])
5338    }
5339
5340    async fn handle_supervisor_restart(
5341        &self,
5342        frame: Frame,
5343        module_id: String,
5344        drain_timeout_ms: Option<u64>,
5345        requester: Principal,
5346    ) -> Result<Vec<Frame>, RouterError> {
5347        let operation_lock = self.supervisor.operation_lock();
5348        let _operation_guard = operation_lock.lock().await;
5349        let Some(module) = self.supervisor.get(&module_id) else {
5350            return Ok(vec![control_error_frame(
5351                &frame,
5352                "unknown_module",
5353                format!("module_id '{module_id}' is not supervised"),
5354            )?]);
5355        };
5356
5357        self.route_outages.mark_operator_action(&module_id);
5358        if let Err(err) = module
5359            .restart_requested(drain_timeout_ms, Some(requester))
5360            .await
5361        {
5362            self.route_outages
5363                .operator_action_ended_unrefused(&module_id);
5364            let (code, message) = match err {
5365                crate::supervise::SuperviseError::Disabled { .. } => {
5366                    ("module_disabled", err.to_string())
5367                }
5368                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5369                    ("swap_in_progress", err.to_string())
5370                }
5371                _ => (
5372                    "target_unavailable",
5373                    format!("failed to restart module_id '{module_id}': {err}"),
5374                ),
5375            };
5376            return Ok(vec![control_error_frame(&frame, code, message)?]);
5377        }
5378
5379        let response = ClientControlResponse::SupervisorAck {
5380            module_id,
5381            applied: true,
5382        };
5383        Ok(vec![control_response_body_frame(
5384            &frame,
5385            &response,
5386            "ClientControlResponse::SupervisorAck",
5387        )?])
5388    }
5389
5390    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
5391    /// when the old process has finished draining: a caller whose own lane
5392    /// rides the old process must get its reply before that drain waits on it.
5393    async fn handle_supervisor_swap(
5394        &self,
5395        frame: Frame,
5396        module_id: String,
5397        ready_timeout_ms: Option<u64>,
5398        requester: Principal,
5399    ) -> Result<Vec<Frame>, RouterError> {
5400        // The daemon-wide operation lock is held only to resolve the handle,
5401        // not across the swap. The swap can take its whole readiness budget,
5402        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
5403        // holding it here would park an operator's stop behind the swap it is
5404        // meant to abort. A rescan or stop that reaches the module during the
5405        // swap is served by the swap itself (see `supervise_swap`).
5406        let module = {
5407            let operation_lock = self.supervisor.operation_lock();
5408            let _operation_guard = operation_lock.lock().await;
5409            self.supervisor.get(&module_id)
5410        };
5411        let Some(module) = module else {
5412            return Ok(vec![control_error_frame(
5413                &frame,
5414                "unknown_module",
5415                format!("module_id '{module_id}' is not supervised"),
5416            )?]);
5417        };
5418
5419        self.route_outages.mark_operator_action(&module_id);
5420        if let Err(err) = module
5421            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5422            .await
5423        {
5424            self.route_outages
5425                .operator_action_ended_unrefused(&module_id);
5426            use crate::supervise::SuperviseError;
5427            let message = err.to_string();
5428            let error = match err {
5429                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5430                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5431                    code: "swap_refused".to_string(),
5432                    message,
5433                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5434                },
5435                SuperviseError::SwapFailed {
5436                    arm,
5437                    candidate_exit,
5438                    ..
5439                } => ErrorBody {
5440                    code: "swap_failed".to_string(),
5441                    message,
5442                    detail: Some(serde_json::json!({
5443                        "arm": arm.as_str(),
5444                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5445                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5446                    })),
5447                },
5448                _ => ErrorBody::new(
5449                    "target_unavailable",
5450                    format!("failed to swap module_id '{module_id}': {message}"),
5451                ),
5452            };
5453            return Ok(vec![control_error_body_frame(&frame, error)?]);
5454        }
5455        // A completed swap kept the incumbent serving until cutover, so it
5456        // usually opened no outage; a mark left behind would make the next,
5457        // unrelated outage read as requested.
5458        self.route_outages
5459            .operator_action_ended_unrefused(&module_id);
5460
5461        let response = ClientControlResponse::SupervisorAck {
5462            module_id,
5463            applied: true,
5464        };
5465        Ok(vec![control_response_body_frame(
5466            &frame,
5467            &response,
5468            "ClientControlResponse::SupervisorAck",
5469        )?])
5470    }
5471
5472    async fn handle_supervisor_reload(
5473        &self,
5474        frame: Frame,
5475        module_id: String,
5476        requester: Principal,
5477    ) -> Result<Vec<Frame>, RouterError> {
5478        let operation_lock = self.supervisor.operation_lock();
5479        let _operation_guard = operation_lock.lock().await;
5480        let Some(module) = self.supervisor.get(&module_id) else {
5481            return Ok(vec![control_error_frame(
5482                &frame,
5483                "unknown_module",
5484                format!("module_id '{module_id}' is not supervised"),
5485            )?]);
5486        };
5487
5488        self.route_outages.mark_operator_action(&module_id);
5489        if let Err(err) = module.reload_requested(Some(requester)).await {
5490            self.route_outages
5491                .operator_action_ended_unrefused(&module_id);
5492            let (code, message) = match err {
5493                crate::supervise::SuperviseError::Disabled { .. } => {
5494                    ("module_disabled", err.to_string())
5495                }
5496                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5497                    ("swap_in_progress", err.to_string())
5498                }
5499                _ => (
5500                    "reload_failed",
5501                    format!("failed to reload module_id '{module_id}': {err}"),
5502                ),
5503            };
5504            return Ok(vec![control_error_frame(&frame, code, message)?]);
5505        }
5506
5507        let response = ClientControlResponse::SupervisorAck {
5508            module_id,
5509            applied: true,
5510        };
5511        Ok(vec![control_response_body_frame(
5512            &frame,
5513            &response,
5514            "ClientControlResponse::SupervisorAck",
5515        )?])
5516    }
5517
5518    async fn handle_supervisor_rescan(
5519        &self,
5520        frame: Frame,
5521        preview: bool,
5522        requester: Principal,
5523    ) -> Result<Vec<Frame>, RouterError> {
5524        let Some(context) = self.rescan.clone() else {
5525            return Ok(vec![control_error_frame(
5526                &frame,
5527                "rescan_unavailable",
5528                "the daemon was not started with a reloadable config path".to_string(),
5529            )?]);
5530        };
5531
5532        let operation_lock = self.supervisor.operation_lock();
5533        let _operation_guard = operation_lock.lock().await;
5534        let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5535            Ok(config) => config,
5536            Err(err) => {
5537                return Ok(vec![control_error_frame(
5538                    &frame,
5539                    "invalid_daemon_config",
5540                    format!("supervisor rescan rejected daemon config: {err}"),
5541                )?])
5542            }
5543        };
5544        // `load` reports a missing file as Ok(None), which is correct at boot
5545        // (no config, nothing to supervise) and catastrophic here: rescan treats
5546        // "not in the config" as "remove it", so an absent file would read as an
5547        // empty module list and retire the entire running fleet. An editor
5548        // writing via write-new-then-rename, or a half-finished edit, is enough
5549        // to open that window. Refuse instead: a config that cannot be read
5550        // carries no instruction to remove anything.
5551        let Some((config, activity_config)) = loaded else {
5552            return Ok(vec![control_error_frame(
5553                &frame,
5554                "invalid_daemon_config",
5555                format!(
5556                    "daemon config not found at {}; refusing to rescan (an absent config would \
5557                     retire every supervised module)",
5558                    context.config_path.display()
5559                ),
5560            )?]);
5561        };
5562        let (
5563            configured_port,
5564            storage_config,
5565            admission_facts_carrier_module_id,
5566            admission_facts_targets,
5567            scope_authority_owners,
5568            privileged_operator_prompt,
5569            modules,
5570            reserved_capabilities,
5571        ) = (
5572            config.port,
5573            config.storage,
5574            config.admission_facts_carrier_module_id,
5575            config.admission_facts_targets,
5576            config.scope_authority_owners,
5577            config.privileged_operator_prompt,
5578            config.modules,
5579            config.reserved_capabilities,
5580        );
5581
5582        // Collect the sections rescan cannot apply, so the REPLY carries them.
5583        //
5584        // The warning below has always been correct and has always gone only to
5585        // the journal -- addressed to whoever reads logs, while the person who
5586        // just edited the config is looking at the CLI. Naming each section
5587        // individually rather than setting a flag: "something outside modules
5588        // changed" sends the operator back to diffing their own file, which is
5589        // the work this is meant to save.
5590        let mut restart_required = Vec::new();
5591        for section in RestartRequiredSection::ALL {
5592            let changed = match section {
5593                RestartRequiredSection::Port => configured_port != context.configured_port,
5594                RestartRequiredSection::Storage => storage_config != context.storage_config,
5595                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5596                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5597                }
5598                RestartRequiredSection::AdmissionFactsTargets => {
5599                    admission_facts_targets != context.admission_facts_targets
5600                }
5601                RestartRequiredSection::ScopeAuthorityOwners => {
5602                    scope_authority_owners != context.scope_authority_owners
5603                }
5604                RestartRequiredSection::PrivilegedOperatorPrompt => {
5605                    privileged_operator_prompt != context.privileged_operator_prompt
5606                }
5607            };
5608            if changed {
5609                restart_required.push(section.label().to_string());
5610            }
5611        }
5612        if !restart_required.is_empty() {
5613            warn!(
5614                config_path = %context.config_path.display(),
5615                sections = %restart_required.join(", "),
5616                "daemon config changed outside the modules section; restart the daemon to apply those changes"
5617            );
5618        }
5619
5620        for configured in &modules {
5621            if let Err(err) = validate_spec(&configured.module_spec()) {
5622                return Ok(vec![control_error_frame(
5623                    &frame,
5624                    "invalid_daemon_config",
5625                    format!("supervisor rescan rejected daemon config: {err}"),
5626                )?]);
5627            }
5628        }
5629
5630        let configured_capabilities = modules
5631            .iter()
5632            .map(|module| (module.module_id.clone(), module.enabled))
5633            .collect::<Vec<_>>();
5634        let preview_capability_warnings = if preview {
5635            let (_, registrations) = self.runtime_capability_snapshot()?;
5636            let current_modules = self
5637                .supervisor
5638                .list()
5639                .into_iter()
5640                .map(|module| module.module_id().to_string())
5641                .collect::<BTreeSet<_>>();
5642            let resulting_modules = configured_capabilities.clone();
5643            let removed = current_modules
5644                .into_iter()
5645                .filter(|module_id| {
5646                    !resulting_modules
5647                        .iter()
5648                        .any(|(configured_id, _)| configured_id == module_id)
5649                })
5650                .collect::<Vec<_>>();
5651            self.capability_evaluator.preview_removal_warnings(
5652                resulting_modules,
5653                &removed,
5654                &registrations,
5655            )
5656        } else {
5657            Vec::new()
5658        };
5659        let result = match self
5660            .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5661            .await
5662        {
5663            Ok(result) => result,
5664            Err(message) => {
5665                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5666            }
5667        };
5668        if !preview {
5669            if let Some(ledger) = &self.activity {
5670                ledger.module_rescanned(&result.added, &result.removed, requester);
5671            }
5672            if let Err(code) = self.configure_activity(activity_config).await {
5673                warn!(
5674                    code,
5675                    "activity retention could not be applied during rescan"
5676                );
5677            }
5678            self.capability_evaluator
5679                .configure(configured_capabilities, reserved_capabilities);
5680            self.capability_evaluator.wake_deadline_loop();
5681            self.refresh_capability_requirements();
5682        }
5683        let mut result = result;
5684        result.restart_required = restart_required;
5685        result.capability_warnings = preview_capability_warnings;
5686        let response = ClientControlResponse::SupervisorRescan { result };
5687        Ok(vec![control_response_body_frame(
5688            &frame,
5689            &response,
5690            "ClientControlResponse::SupervisorRescan",
5691        )?])
5692    }
5693
5694    async fn handle_supervisor_release_reserved(
5695        &self,
5696        frame: Frame,
5697        module_id: String,
5698    ) -> Result<Vec<Frame>, RouterError> {
5699        let Some(context) = self.rescan.clone() else {
5700            return Ok(vec![control_error_frame(
5701                &frame,
5702                "release_unavailable",
5703                "reserved-id release requires a daemon started with a reloadable config path",
5704            )?]);
5705        };
5706        let operation_lock = self.supervisor.operation_lock();
5707        let _operation_guard = operation_lock.lock().await;
5708        let loaded = match crate::daemon_config::load(&context.config_path) {
5709            Ok(Some(config)) => config,
5710            Ok(None) => {
5711                return Ok(vec![control_error_frame(
5712                    &frame,
5713                    "invalid_daemon_config",
5714                    format!(
5715                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5716                        context.config_path.display()
5717                    ),
5718                )?])
5719            }
5720            Err(err) => {
5721                return Ok(vec![control_error_frame(
5722                    &frame,
5723                    "invalid_daemon_config",
5724                    format!("unable to verify reserved-id release against daemon config: {err}"),
5725                )?])
5726            }
5727        };
5728        if loaded
5729            .modules
5730            .iter()
5731            .any(|configured| configured.module_id == module_id)
5732        {
5733            return Ok(vec![control_error_frame(
5734                &frame,
5735                "reserved_module_configured",
5736                format!(
5737                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5738                ),
5739            )?]);
5740        }
5741        if !self.supervisor.release_retained_reserved_gate(&module_id) {
5742            return Ok(vec![control_error_frame(
5743                &frame,
5744                "reserved_gate_not_retained",
5745                format!(
5746                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5747                ),
5748            )?]);
5749        }
5750
5751        let response = ClientControlResponse::SupervisorAck {
5752            module_id,
5753            applied: true,
5754        };
5755        Ok(vec![control_response_body_frame(
5756            &frame,
5757            &response,
5758            "ClientControlResponse::SupervisorAck",
5759        )?])
5760    }
5761
5762    /// Reconcile the running module set against the configured one.
5763    ///
5764    /// With `preview` set, the diff is computed and returned WITHOUT applying any
5765    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
5766    /// deliberately shares this function with the executing path rather than
5767    /// computing the same diff somewhere else -- two implementations of one
5768    /// decision agree until they do not, and the whole value of a preview is that
5769    /// it describes the operation that will actually run.
5770    async fn reconcile_supervised_modules(
5771        &self,
5772        supervisor: &Supervisor,
5773        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5774        preview: bool,
5775        requester: Principal,
5776    ) -> Result<SupervisorRescanResult, String> {
5777        let mut current = BTreeMap::new();
5778        for module in self.supervisor.list() {
5779            let (spec, health) = module.configuration().map_err(|err| {
5780                format!(
5781                    "failed to read configuration for module_id '{}': {err}",
5782                    module.module_id()
5783                )
5784            })?;
5785            let enabled = module
5786                .status()
5787                .map_err(|err| {
5788                    format!(
5789                        "failed to read status for module_id '{}': {err}",
5790                        module.module_id()
5791                    )
5792                })?
5793                .enabled;
5794            current.insert(
5795                module.module_id().to_string(),
5796                (module, spec, health, enabled),
5797            );
5798        }
5799        let configured = configured_modules
5800            .into_iter()
5801            .map(|module| (module.module_id.clone(), module))
5802            .collect::<BTreeMap<_, _>>();
5803
5804        let added = configured
5805            .keys()
5806            .filter(|module_id| !current.contains_key(*module_id))
5807            .cloned()
5808            .collect::<Vec<_>>();
5809        let removed = current
5810            .keys()
5811            .filter(|module_id| !configured.contains_key(*module_id))
5812            .cloned()
5813            .collect::<Vec<_>>();
5814        let mut changed_pending_reload = Vec::new();
5815        let mut configuration_changes = BTreeSet::new();
5816        let mut enabled_changes = BTreeSet::new();
5817        let mut unchanged = 0_u32;
5818
5819        for (module_id, configured_module) in &configured {
5820            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5821            else {
5822                continue;
5823            };
5824            // Compare the whole launch spec so a future launch field cannot
5825            // accidentally become a live-only policy change. Health is stored
5826            // separately and applies live without replacing the process.
5827            let launch_changed = *current_spec != configured_module.module_spec();
5828            let configuration_changed =
5829                launch_changed || *current_health != configured_module.health;
5830            let enabled_changed = *current_enabled != configured_module.enabled;
5831            if configuration_changed {
5832                configuration_changes.insert(module_id.clone());
5833            }
5834            if launch_changed {
5835                changed_pending_reload.push(module_id.clone());
5836            }
5837            if enabled_changed {
5838                enabled_changes.insert(module_id.clone());
5839            }
5840            if !configuration_changed && !enabled_changed {
5841                unchanged = unchanged.saturating_add(1);
5842            }
5843        }
5844
5845        // Everything above this point is pure computation over two snapshots.
5846        // Everything below MUTATES. The preview returns here so the boundary is a
5847        // single early return rather than a condition repeated at each mutation
5848        // site, where one missed guard would apply part of a change the caller was
5849        // told would not happen.
5850        if preview {
5851            return Ok(SupervisorRescanResult {
5852                added,
5853                removed,
5854                changed_pending_reload,
5855                enabled_changes: enabled_changes.iter().cloned().collect(),
5856                unchanged,
5857                preview: true,
5858                // Filled by the caller on both paths, so the preview reports
5859                // restart-required sections identically to an executed rescan --
5860                // the preview is where an operator is most likely to be looking.
5861                restart_required: Vec::new(),
5862                capability_warnings: Vec::new(),
5863            });
5864        }
5865
5866        for module_id in &removed {
5867            let module = &current
5868                .get(module_id)
5869                .expect("removed module came from current supervisor state")
5870                .0;
5871            module
5872                .retire_requested(Some(requester.clone()))
5873                .await
5874                .map_err(|err| {
5875                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
5876                })?;
5877            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
5878            //
5879            // `handle_route_open` resolves an absent module in three steps:
5880            // registry, then supervisor status, then tombstone. Retiring first
5881            // opens a window where ALL THREE ARE ABSENT -- the registry entry
5882            // went with the teardown above, the supervisor entry went with
5883            // `retire`, and the tombstone does not exist yet -- so a route.open
5884            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
5885            // it") for a module that was deliberately removed and whose caller
5886            // should get `module_removed` (TERMINAL, carrying a removal age).
5887            //
5888            // Writing the tombstone first closes it: during the window the
5889            // supervisor entry still answers, so the caller gets
5890            // `target_unavailable` -- retryable, and TRUE, because the module
5891            // is mid-teardown. After both statements it is `module_removed`.
5892            // No instant remains where a removed module reads as one that
5893            // never existed.
5894            //
5895            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
5896            // the absence of a test beside a fix invites deletion: these are
5897            // two sync statements with no await between them, so reaching the
5898            // window needs a second worker thread to land exactly between them
5899            // and there is no hook to force it. MEASURED: the 25 daemon_config
5900            // tests pass identically with the old order and the new one, so
5901            // the existing suite cannot see this and a green run is not
5902            // evidence either way. What the suite does hold is the
5903            // post-condition -- a removed module answers `module_removed` --
5904            // which this preserves.
5905            //
5906            // Found by an Athena panel reading the shipped tree against a
5907            // design note (2026-09-19), as the one concrete instance of that
5908            // note's class that survived contact with source. Direction is
5909            // benign: retryable where terminal was intended, never the reverse.
5910            self.supervisor.record_rescan_removal(module_id);
5911            self.supervisor.retire(module_id);
5912            self.route_outages.forget(module_id);
5913        }
5914
5915        for module_id in configured.keys() {
5916            let Some((module, _, _, _)) = current.get(module_id) else {
5917                continue;
5918            };
5919            let configured_module = configured
5920                .get(module_id)
5921                .expect("configured module id came from configured map");
5922            if configuration_changes.contains(module_id) {
5923                module
5924                    .update_configuration(
5925                        configured_module.module_spec(),
5926                        configured_module.health.clone(),
5927                        configured_module.drain_timeout_ms,
5928                    )
5929                    .await
5930                    .map_err(|err| {
5931                        format!(
5932                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
5933                        )
5934                    })?;
5935            }
5936            if enabled_changes.contains(module_id) {
5937                // A rescan that starts or stops a module applies an operator's
5938                // edit to the config, so the resulting outage was asked for.
5939                self.route_outages.mark_operator_action(module_id);
5940                module
5941                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
5942                    .await
5943                    .map_err(|err| {
5944                        self.route_outages.operator_action_ended_unrefused(module_id);
5945                        format!(
5946                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5947                            configured_module.enabled
5948                        )
5949                    })?;
5950            }
5951        }
5952
5953        for module_id in &added {
5954            let configured_module = configured
5955                .get(module_id)
5956                .expect("added module id came from configured map");
5957            supervisor
5958                .supervise_configured_requested(
5959                    configured_module.module_spec(),
5960                    configured_module.enabled,
5961                    configured_module.health.clone(),
5962                    configured_module.drain_timeout_ms,
5963                    configured_module.restart,
5964                    Some(requester.clone()),
5965                )
5966                .map_err(|err| {
5967                    format!("failed to add module_id '{module_id}' during rescan: {err}")
5968                })?;
5969        }
5970
5971        Ok(SupervisorRescanResult {
5972            added,
5973            removed,
5974            changed_pending_reload,
5975            enabled_changes: enabled_changes.iter().cloned().collect(),
5976            unchanged,
5977            preview: false,
5978            // Filled by the caller, which is the only layer that can see the
5979            // previous config to diff against.
5980            restart_required: Vec::new(),
5981            capability_warnings: Vec::new(),
5982        })
5983    }
5984
5985    async fn handle_supervisor_set_enabled(
5986        &self,
5987        frame: Frame,
5988        module_id: String,
5989        enabled: bool,
5990        requester: Principal,
5991    ) -> Result<Vec<Frame>, RouterError> {
5992        let operation_lock = self.supervisor.operation_lock();
5993        let _operation_guard = operation_lock.lock().await;
5994        let Some(module) = self.supervisor.get(&module_id) else {
5995            return Ok(vec![control_error_frame(
5996                &frame,
5997                "unknown_module",
5998                format!("module_id '{module_id}' is not supervised"),
5999            )?]);
6000        };
6001
6002        // Enabling counts as well as disabling: a module an operator starts
6003        // is refused until it registers, and that wait was asked for.
6004        self.route_outages.mark_operator_action(&module_id);
6005        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6006            Ok(applied) => applied,
6007            Err(err) => {
6008                self.route_outages
6009                    .operator_action_ended_unrefused(&module_id);
6010                return Ok(vec![control_error_frame(
6011                    &frame,
6012                    "target_unavailable",
6013                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6014                )?]);
6015            }
6016        };
6017        if !applied {
6018            // Already in the requested state: nothing was made unavailable,
6019            // so the mark must not outlive this request.
6020            self.route_outages
6021                .operator_action_ended_unrefused(&module_id);
6022        }
6023
6024        self.capability_evaluator.wake_deadline_loop();
6025        self.refresh_capability_requirements();
6026        let response = ClientControlResponse::SupervisorAck { module_id, applied };
6027        Ok(vec![control_response_body_frame(
6028            &frame,
6029            &response,
6030            "ClientControlResponse::SupervisorAck",
6031        )?])
6032    }
6033
6034    async fn handle_supervisor_health_probe(
6035        &self,
6036        frame: Frame,
6037        module_id: String,
6038    ) -> Result<Vec<Frame>, RouterError> {
6039        self.refresh_capability_requirements();
6040        let Some(registration) = self
6041            .registry
6042            .get_module(&module_id)
6043            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6044        else {
6045            return Ok(vec![control_error_frame(
6046                &frame,
6047                "unknown_module",
6048                format!("module_id '{module_id}' is not registered"),
6049            )?]);
6050        };
6051
6052        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
6053        // named for it. Making `module_registration_grants_op` return false
6054        // unconditionally reddens five tests, and every one is named for something
6055        // else -- capability relay, probe/bind demultiplexing, supervision-only
6056        // probing. They exercise a successful advertisement check on the way to their
6057        // own subject.
6058        //
6059        // Real protection, fragile in a specific way: narrowing any of those tests to
6060        // focus on its stated subject would silently remove coverage nobody knows
6061        // they are carrying. Recorded here rather than as a sixth test, because the
6062        // useful fact is WHICH tests hold the guard up -- a new test would add
6063        // coverage without telling the next person what the existing ones quietly do.
6064        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6065        {
6066            return Ok(vec![control_error_frame(
6067                &frame,
6068                "health_not_advertised",
6069                format!("module_id '{module_id}' did not advertise health.check"),
6070            )?]);
6071        }
6072
6073        let deadline = Instant::now() + self.health_probe_timeout;
6074        let pending = match self.forwarding.begin_module_control_rpc_for(
6075            &module_id,
6076            MODULE_CONTROL_OP_HEALTH_CHECK,
6077            deadline,
6078        ) {
6079            Ok(pending) => pending,
6080            Err(err) => {
6081                return Ok(vec![control_error_frame(
6082                    &frame,
6083                    forwarding_error_code(&err),
6084                    err.to_string(),
6085                )?])
6086            }
6087        };
6088
6089        let PendingModuleControlRpc {
6090            endpoint,
6091            module_sink,
6092            negotiated_ver,
6093            corr: probe_corr,
6094            receiver,
6095        } = pending;
6096        let mut guard =
6097            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6098        let probe_body =
6099            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6100                RouterError::backend(
6101                    0,
6102                    frame.header.corr,
6103                    format!("failed to encode health.check request: {err}"),
6104                )
6105            })?;
6106        let probe_frame = Frame::build_with_version(
6107            negotiated_ver,
6108            FrameType::Request,
6109            control_flags(),
6110            0,
6111            0,
6112            probe_corr,
6113            probe_body,
6114        )
6115        .map_err(RouterError::FrameBuild)?;
6116
6117        // Queue admission and the reply share one deadline, so a module that
6118        // stops reading cannot park the requesting connection's reader forever.
6119        match timeout_at(deadline, module_sink.send(probe_frame)).await {
6120            Ok(Ok(())) => {}
6121            Ok(Err(err)) => {
6122                return Ok(vec![control_error_frame(
6123                    &frame,
6124                    "target_unavailable",
6125                    err.to_string(),
6126                )?]);
6127            }
6128            Err(_) => {
6129                return Ok(vec![control_error_frame(
6130                    &frame,
6131                    "module_timeout",
6132                    format!(
6133                        "module_id '{module_id}' did not answer health.check within {:?}",
6134                        self.health_probe_timeout
6135                    ),
6136                )?]);
6137            }
6138        }
6139
6140        match timeout_at(deadline, receiver).await {
6141            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6142                guard.disarm();
6143                let Some(report) = response.health_report() else {
6144                    return Ok(vec![control_error_frame(
6145                        &frame,
6146                        "invalid_control_body",
6147                        "health.check RPC returned a non-health response",
6148                    )?]);
6149                };
6150                // Metrics go out whole here. The supervisor's cached snapshot
6151                // caps this blob (see truncate_health_metrics), and this path
6152                // exists precisely to answer without that cap -- so applying it
6153                // here would leave no way to see what the cached view drops.
6154                let HealthReport {
6155                    status,
6156                    detail,
6157                    metrics,
6158                } = report;
6159                let capability_detail = self
6160                    .capability_evaluator
6161                    .required_problem_detail(&module_id);
6162                let response = ClientControlResponse::SupervisorHealthProbe {
6163                    module_id,
6164                    status,
6165                    detail: append_capability_problem_detail(detail, capability_detail),
6166                    metrics,
6167                };
6168                Ok(vec![control_response_body_frame(
6169                    &frame,
6170                    &response,
6171                    "ClientControlResponse::SupervisorHealthProbe",
6172                )?])
6173            }
6174            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6175                guard.disarm();
6176                Ok(vec![control_error_body_frame(&frame, body)?])
6177            }
6178            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6179                guard.disarm();
6180                Ok(vec![control_error_frame(
6181                    &frame,
6182                    "target_unavailable",
6183                    message,
6184                )?])
6185            }
6186            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6187                guard.disarm();
6188                Ok(vec![control_error_frame(
6189                    &frame,
6190                    "invalid_control_body",
6191                    message,
6192                )?])
6193            }
6194            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6195                guard.disarm();
6196                Ok(vec![control_error_frame(
6197                    &frame,
6198                    "invalid_control_body",
6199                    format!("expected module-control op '{expected}', got '{actual}'"),
6200                )?])
6201            }
6202            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6203                guard.disarm();
6204                Ok(vec![control_error_frame(
6205                    &frame,
6206                    "module_timeout",
6207                    format!(
6208                        "module_id '{module_id}' answered health.check after {:?}",
6209                        self.health_probe_timeout
6210                    ),
6211                )?])
6212            }
6213            Ok(Err(_)) => Ok(vec![control_error_frame(
6214                &frame,
6215                "target_unavailable",
6216                "health.check waiter was canceled before the module responded",
6217            )?]),
6218            Err(_) => Ok(vec![control_error_frame(
6219                &frame,
6220                "module_timeout",
6221                format!(
6222                    "module_id '{module_id}' did not answer health.check within {:?}",
6223                    self.health_probe_timeout
6224                ),
6225            )?]),
6226        }
6227    }
6228
6229    fn supervisor_status(
6230        &self,
6231        module_id: &str,
6232        corr: u64,
6233    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6234        self.supervisor
6235            .get(module_id)
6236            .map(|module| {
6237                let warming = module.is_warming_for_control("status").map_err(|err| {
6238                    RouterError::backend(
6239                        0,
6240                        corr,
6241                        format!(
6242                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
6243                        ),
6244                    )
6245                })?;
6246                module.status_for_control("status").map_err(|err| {
6247                    RouterError::backend(
6248                        0,
6249                        corr,
6250                        format!(
6251                            "failed to read supervisor status for module_id '{module_id}': {err}"
6252                        ),
6253                    )
6254                }).map(|status| (status, warming))
6255            })
6256            .transpose()
6257    }
6258
6259    fn guard_module_control_op(
6260        &self,
6261        frame: &Frame,
6262        module_id: &str,
6263        op: &str,
6264    ) -> Result<Option<Frame>, RouterError> {
6265        if self.module_grants_op(module_id, op, frame.header.corr)? {
6266            return Ok(None);
6267        }
6268
6269        Ok(Some(control_error_frame(
6270            frame,
6271            "op_not_allowed",
6272            format!("module_id '{module_id}' did not grant control op '{op}'"),
6273        )?))
6274    }
6275
6276    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6277        let Some(registration) = self
6278            .registry
6279            .get_module(module_id)
6280            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6281        else {
6282            return Ok(false);
6283        };
6284        Ok(module_registration_grants_op(&registration.control_ops, op))
6285    }
6286
6287    fn handle_status_update(
6288        &self,
6289        endpoint: ModuleEndpointId,
6290        frame: Frame,
6291    ) -> Result<Vec<Frame>, RouterError> {
6292        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6293            Ok(update) => update,
6294            Err(err) => {
6295                // Forward-compat: a newer module may push a channel-0 op this subc
6296                // version doesn't know. The control contract says unknown push ops
6297                // are IGNORED, never answered with an error. Only a malformed body
6298                // for an op we DO know is a real error worth surfacing.
6299                if is_known_module_push_op(&frame.body) {
6300                    return Ok(vec![control_error_frame(
6301                        &frame,
6302                        "invalid_control_body",
6303                        format!("malformed module control push body: {err}"),
6304                    )?]);
6305                }
6306                return Ok(Vec::new());
6307            }
6308        };
6309
6310        match update {
6311            ModuleControlPush::RouteStatus {
6312                route_channel,
6313                route_epoch,
6314                status,
6315            } => {
6316                self.forwarding
6317                    .cache_status(endpoint, route_channel, route_epoch, status)
6318                    .map_err(RouterError::Forwarding)?;
6319            }
6320        }
6321        Ok(Vec::new())
6322    }
6323
6324    fn handle_route_poll(
6325        &self,
6326        ctx: &RouteCtx,
6327        frame: Frame,
6328        route_channel: u16,
6329        route_epoch: u32,
6330        kind: PollKind,
6331    ) -> Result<Vec<Frame>, RouterError> {
6332        let snapshot = self
6333            .forwarding
6334            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6335            .map_err(RouterError::Forwarding)?;
6336        let response = match (kind, snapshot) {
6337            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6338                ClientControlResponse::RoutePoll {
6339                    route_channel,
6340                    route_epoch,
6341                    status,
6342                    live: None,
6343                }
6344            }
6345            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6346                route_channel,
6347                route_epoch,
6348                status: None,
6349                live: None,
6350            },
6351            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6352                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
6353                // is what makes reporting `true` correct rather than a
6354                // confident guess. `process_live` returns None only when the
6355                // module id has no supervisor snapshot at all -- an
6356                // externally-started module the daemon did not spawn -- and
6357                // for those the supervisor has no opinion to offer, ever. It
6358                // is never None for a supervised module in an unknown state:
6359                // a supervised module always has a snapshot, and the answer
6360                // comes from `state == Running && process_alive`.
6361                //
6362                // The route is Bound, so the module completed a HELLO on a
6363                // live connection; "the process this route points at is
6364                // running" is therefore attested by the binding rather than
6365                // assumed. Reporting `false` for an unsupervised module would
6366                // be the actual lie -- it would tell a client its healthy
6367                // route is dead because the daemon does not manage the
6368                // process.
6369                //
6370                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
6371                // module whose liveness is genuinely unknown, e.g. a snapshot
6372                // that has not been populated yet -- THIS DEFAULT BECOMES
6373                // WRONG and must split: unsupervised stays true, unknown
6374                // becomes null so the client can tell the two apart. The
6375                // response field is already `Option<bool>`, so the wire can
6376                // carry that distinction today.
6377                let live = self
6378                    .process_liveness
6379                    .as_ref()
6380                    .and_then(|source| source.process_live(&module_id))
6381                    .unwrap_or(true);
6382                ClientControlResponse::RoutePoll {
6383                    route_channel,
6384                    route_epoch,
6385                    status: None,
6386                    live: Some(live),
6387                }
6388            }
6389            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6390                route_channel,
6391                route_epoch,
6392                status: None,
6393                live: Some(false),
6394            },
6395        };
6396
6397        Ok(vec![control_response_body_frame(
6398            &frame,
6399            &response,
6400            "ClientControlResponse::RoutePoll",
6401        )?])
6402    }
6403
6404    pub(crate) fn observe_module_control_completion(
6405        &self,
6406        completion: ModuleControlRpcCompletion,
6407    ) -> bool {
6408        match completion {
6409            ModuleControlRpcCompletion::Unknown => false,
6410            ModuleControlRpcCompletion::Settled => true,
6411            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6412                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6413                info!(
6414                    module_id = %module_id,
6415                    latency_ms,
6416                    "late health.check answer proves the module is alive"
6417                );
6418                match self
6419                    .supervisor
6420                    .record_late_health_answer(&module_id, latency_ms)
6421                {
6422                    Ok(true) => {}
6423                    Ok(false) => debug!(
6424                        module_id = %module_id,
6425                        latency_ms,
6426                        "late health.check answer has no active supervisor snapshot"
6427                    ),
6428                    Err(err) => warn!(
6429                        module_id = %module_id,
6430                        latency_ms,
6431                        error = %err,
6432                        "failed to record late health.check answer"
6433                    ),
6434                }
6435                true
6436            }
6437        }
6438    }
6439
6440    /// Decide whether a failure while settling a relayed `route.bind` belongs to
6441    /// the module connection whose frame is being handled, or to the client that
6442    /// relay was opened for.
6443    ///
6444    /// This runs on the MODULE connection's frame handler, where returning `Err`
6445    /// ends that connection -- and a module connection carries every client's
6446    /// routes to that module, so ending it costs the whole fleet its tools.
6447    /// `ConnectionClosing` carries the id of the connection that is closing, and
6448    /// when that id is a CLIENT's, the condition is entirely about that one
6449    /// client's route.open. A client-scoped condition has no authority over a
6450    /// shared module connection, so it is logged and the single relay is dropped:
6451    /// the client is going away, and `complete_pending_relay` already removed the
6452    /// relay before failing, so there is nothing left to settle. Anything that
6453    /// relay still reserved is released by that client's own connection teardown,
6454    /// which is already under way -- that is what "closing" means.
6455    ///
6456    /// Every other failure is a statement about THIS connection and stays fatal:
6457    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
6458    /// id in `ConnectionClosing` all mean this connection cannot keep serving
6459    /// frames correctly.
6460    fn refuse_to_end_module_connection_for_a_client(
6461        &self,
6462        module_connection_id: ConnectionId,
6463        corr: u64,
6464        err: ForwardingError,
6465    ) -> Result<(), RouterError> {
6466        if let ForwardingError::ConnectionClosing { connection_id } = err {
6467            if connection_id != module_connection_id {
6468                warn!(
6469                    module_connection_id = module_connection_id.get(),
6470                    client_connection_id = connection_id.get(),
6471                    corr,
6472                    "dropping a route.bind response for a closing client; the module connection keeps serving"
6473                );
6474                return Ok(());
6475            }
6476        }
6477        Err(RouterError::Forwarding(err))
6478    }
6479
6480    fn handle_module_relay_response(
6481        &self,
6482        connection_id: ConnectionId,
6483        frame: Frame,
6484    ) -> Result<Vec<Frame>, RouterError> {
6485        let mut secondary_error = None;
6486        let outcome = match frame.header.ty {
6487            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6488                Ok(probe) if probe.op == "route.bind" => {
6489                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6490                        Ok(ModuleControlResponse::RouteBindAck {}) => {
6491                            RouteBindRelayOutcome::Accepted
6492                        }
6493                        Ok(other) => {
6494                            let message =
6495                                format!("route.bind response carried unexpected body: {other:?}");
6496                            secondary_error = Some(control_error_frame(
6497                                &frame,
6498                                "invalid_control_body",
6499                                message.clone(),
6500                            )?);
6501                            RouteBindRelayOutcome::ModuleGone(message)
6502                        }
6503                        Err(err) => {
6504                            let message = format!("malformed route.bind response body: {err}");
6505                            secondary_error = Some(control_error_frame(
6506                                &frame,
6507                                "invalid_control_body",
6508                                message.clone(),
6509                            )?);
6510                            RouteBindRelayOutcome::ModuleGone(message)
6511                        }
6512                    }
6513                }
6514                Ok(probe) => {
6515                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6516                    {
6517                        Ok(response) => ModuleControlRpcOutcome::Response(response),
6518                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6519                            "malformed {} response body: {err}",
6520                            probe.op
6521                        )),
6522                    };
6523                    let completion = self
6524                        .forwarding
6525                        .complete_module_control_rpc(
6526                            connection_id,
6527                            frame.header.corr,
6528                            Some(&probe.op),
6529                            outcome,
6530                        )
6531                        .map_err(RouterError::Forwarding)?;
6532                    if !self.observe_module_control_completion(completion) {
6533                        debug!(
6534                            connection_id = connection_id.get(),
6535                            corr = frame.header.corr,
6536                            op = %probe.op,
6537                            "dropping late or unknown module-control RPC response"
6538                        );
6539                    }
6540                    return Ok(Vec::new());
6541                }
6542                Err(err) => {
6543                    if let Some(expected_op) = self
6544                        .forwarding
6545                        .pending_module_control_op(connection_id, frame.header.corr)
6546                        .map_err(RouterError::Forwarding)?
6547                    {
6548                        let completion = self
6549                            .forwarding
6550                            .complete_module_control_rpc(
6551                                connection_id,
6552                                frame.header.corr,
6553                                None,
6554                                ModuleControlRpcOutcome::MalformedResponse(format!(
6555                                    "malformed {expected_op} response body: {err}"
6556                                )),
6557                            )
6558                            .map_err(RouterError::Forwarding)?;
6559                        if !self.observe_module_control_completion(completion) {
6560                            debug!(
6561                                connection_id = connection_id.get(),
6562                                corr = frame.header.corr,
6563                                "dropping late malformed module-control RPC response"
6564                            );
6565                        }
6566                        return Ok(Vec::new());
6567                    }
6568                    let message = format!("malformed route.bind response body: {err}");
6569                    secondary_error = Some(control_error_frame(
6570                        &frame,
6571                        "invalid_control_body",
6572                        message.clone(),
6573                    )?);
6574                    RouteBindRelayOutcome::ModuleGone(message)
6575                }
6576            },
6577            FrameType::Error => {
6578                if self
6579                    .forwarding
6580                    .pending_module_control_op(connection_id, frame.header.corr)
6581                    .map_err(RouterError::Forwarding)?
6582                    .is_some()
6583                {
6584                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6585                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6586                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6587                            "malformed module-control ERROR body: {err}"
6588                        )),
6589                    };
6590                    let completion = self
6591                        .forwarding
6592                        .complete_module_control_rpc(
6593                            connection_id,
6594                            frame.header.corr,
6595                            None,
6596                            outcome,
6597                        )
6598                        .map_err(RouterError::Forwarding)?;
6599                    if !self.observe_module_control_completion(completion) {
6600                        debug!(
6601                            connection_id = connection_id.get(),
6602                            corr = frame.header.corr,
6603                            "dropping late or unknown module-control RPC error"
6604                        );
6605                    }
6606                    return Ok(Vec::new());
6607                }
6608                match serde_json::from_slice::<ErrorBody>(&frame.body) {
6609                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
6610                    Err(err) => {
6611                        let message = format!("malformed route.bind ERROR body: {err}");
6612                        secondary_error = Some(control_error_frame(
6613                            &frame,
6614                            "invalid_control_body",
6615                            message.clone(),
6616                        )?);
6617                        RouteBindRelayOutcome::ModuleGone(message)
6618                    }
6619                }
6620            }
6621            ty => {
6622                return Ok(vec![control_error_frame(
6623                    &frame,
6624                    "unsupported_control_frame",
6625                    format!("unsupported module channel-0 frame {ty:?}"),
6626                )?])
6627            }
6628        };
6629
6630        let settled =
6631            self.forwarding
6632                .complete_pending_relay(connection_id, frame.header.corr, outcome);
6633        let completion = match settled {
6634            Ok(completion) => completion,
6635            Err(err) => {
6636                self.refuse_to_end_module_connection_for_a_client(
6637                    connection_id,
6638                    frame.header.corr,
6639                    err,
6640                )?;
6641                return Ok(secondary_error.into_iter().collect());
6642            }
6643        };
6644        if let Some(target) = completion.abandoned.as_ref() {
6645            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6646        }
6647        if !completion.settled {
6648            debug!(
6649                connection_id = connection_id.get(),
6650                corr = frame.header.corr,
6651                frame_type = ?frame.header.ty,
6652                "dropping late or unknown route.bind relay response"
6653            );
6654        }
6655        Ok(secondary_error.into_iter().collect())
6656    }
6657
6658    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6659        debug!(connection_id = connection_id.get(), "handling GOODBYE");
6660        // GOODBYE ends the connection's logical session even when its socket
6661        // stays open. Use disconnect teardown so verdicts, client notices and
6662        // scope authority are released at the same lifecycle boundary.
6663        self.cleanup_connection_with_end_reason(
6664            connection_id,
6665            RegistrationEndReason::ExplicitGoodbye,
6666        )
6667        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6668        Ok(Vec::new())
6669    }
6670}
6671
6672impl Default for ControlHandler {
6673    fn default() -> Self {
6674        Self::new(Arc::new(Registry::default()))
6675    }
6676}
6677
6678impl crate::supervise::SwapPromotionObserver for ControlHandler {
6679    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6680        self.apply_registration_capabilities(registration);
6681    }
6682}
6683
6684fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6685    CapabilityRequirementStatus {
6686        consumer: status.consumer,
6687        capability: status.capability,
6688        need: match status.need {
6689            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6690            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6691        },
6692        verdict: status.verdict.as_str().to_string(),
6693        episode_seq: status.episode_seq,
6694        config_satisfiable: status.config_satisfiable,
6695        runtime_available: status.runtime_available,
6696        detail: status.detail,
6697    }
6698}
6699
6700fn append_capability_problem_detail(
6701    detail: Option<String>,
6702    capability_detail: Option<String>,
6703) -> Option<String> {
6704    match (detail, capability_detail) {
6705        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6706        (Some(detail), None) => Some(detail),
6707        (None, Some(capability_detail)) => Some(capability_detail),
6708        (None, None) => None,
6709    }
6710}
6711
6712fn subc_ops() -> Vec<String> {
6713    SUBC_CONTROL_OPS
6714        .iter()
6715        .map(|op| (*op).to_string())
6716        .collect()
6717}
6718
6719fn module_subc_ops() -> Vec<String> {
6720    SUBC_CONTROL_OPS
6721        .iter()
6722        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6723        .map(|op| (*op).to_string())
6724        .collect()
6725}
6726
6727#[cfg(test)]
6728fn module_baseline_control_ops() -> Vec<String> {
6729    MODULE_BASELINE_CONTROL_OPS
6730        .iter()
6731        .map(|op| (*op).to_string())
6732        .collect()
6733}
6734
6735fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6736    let mut seen = HashSet::new();
6737    let mut effective = Vec::new();
6738    for op in MODULE_BASELINE_CONTROL_OPS {
6739        if seen.insert((*op).to_string()) {
6740            effective.push((*op).to_string());
6741        }
6742    }
6743    for op in declared.unwrap_or_default() {
6744        if seen.insert(op.clone()) {
6745            effective.push(op);
6746        }
6747    }
6748    effective
6749}
6750
6751fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6752    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6753}
6754
6755fn target_module_id(target: &RouteTarget) -> &str {
6756    match target {
6757        RouteTarget::ToolProvider { module_id }
6758        | RouteTarget::ManagementSurface { module_id }
6759        | RouteTarget::InternalService { module_id, .. } => module_id,
6760    }
6761}
6762
6763fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6764    roles.iter().any(|role| match (target, role) {
6765        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6766        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6767        (
6768            RouteTarget::InternalService { service_id, .. },
6769            ProviderRole::InternalService {
6770                service_id: provided,
6771                ..
6772            },
6773        ) => service_id == provided,
6774        _ => false,
6775    })
6776}
6777
6778fn is_routable_role(role: &ProviderRole) -> bool {
6779    matches!(
6780        role,
6781        ProviderRole::ToolProvider { .. }
6782            | ProviderRole::ManagementSurface { .. }
6783            | ProviderRole::InternalService { .. }
6784    )
6785}
6786
6787#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6788enum ControlRequestBodyError {
6789    UnknownOp,
6790    InvalidBody,
6791}
6792
6793#[derive(Debug, Deserialize)]
6794struct ControlOpProbe {
6795    op: String,
6796}
6797
6798/// Channel-0 push ops this subc version understands. A push whose `op` is not in
6799/// this set is treated as a forward-compat unknown and ignored rather than errored.
6800const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6801
6802fn is_known_module_push_op(body: &[u8]) -> bool {
6803    serde_json::from_slice::<ControlOpProbe>(body)
6804        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6805        .unwrap_or(false)
6806}
6807
6808fn is_known_module_request_op(body: &[u8]) -> bool {
6809    serde_json::from_slice::<ControlOpProbe>(body)
6810        .map(|probe| is_module_to_subc_op(&probe.op))
6811        .unwrap_or(false)
6812}
6813
6814fn is_module_to_subc_op(op: &str) -> bool {
6815    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6816}
6817
6818fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6819    debug!(
6820        op = %op,
6821        connection_id = connection_id.get(),
6822        corr,
6823        "control dispatch"
6824    );
6825}
6826
6827fn log_slow_control_dispatch(
6828    dispatch_started_at: Option<StdInstant>,
6829    op: &'static str,
6830    connection_id: ConnectionId,
6831    corr: u64,
6832    route_open_timings: Option<&RouteOpenTimings>,
6833) {
6834    let Some(dispatch_started_at) = dispatch_started_at else {
6835        return;
6836    };
6837    // Use the same monotonic clock as the phase timers and relay deadline.
6838    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
6839    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6840        if let Some(timings) = route_open_timings {
6841            warn!(
6842                op = %op,
6843                connection_id = connection_id.get(),
6844                corr,
6845                elapsed_ms = elapsed.as_millis() as u64,
6846                module_id = ?timings.module_id,
6847                principal = %timings.principal,
6848                admission_ms = timings.admission.as_millis() as u64,
6849                project_root_ms = timings.project_root.as_millis() as u64,
6850                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
6851                bind_send_ms = timings.bind_send.as_millis() as u64,
6852                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
6853                "slow control dispatch"
6854            );
6855            return;
6856        }
6857        warn!(
6858            op = %op,
6859            connection_id = connection_id.get(),
6860            corr,
6861            elapsed_ms = elapsed.as_millis() as u64,
6862            "slow control dispatch"
6863        );
6864    }
6865}
6866
6867/// A test stand-in for the OS caller lookup, given the recorded endpoints.
6868#[cfg(test)]
6869type TestCallerLookup =
6870    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
6871
6872/// Every new client operation needs an explicit privilege decision here.
6873fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
6874    match request {
6875        ClientControlRequest::SupervisorRestart { module_id, .. } => {
6876            Some(PrivilegedOperation::Restart { module_id })
6877        }
6878        ClientControlRequest::SupervisorSwap { module_id, .. } => {
6879            Some(PrivilegedOperation::Swap { module_id })
6880        }
6881        ClientControlRequest::SupervisorReload { module_id } => {
6882            Some(PrivilegedOperation::Reload { module_id })
6883        }
6884        ClientControlRequest::SupervisorRescan { preview } => {
6885            if *preview {
6886                None
6887            } else {
6888                Some(PrivilegedOperation::Rescan)
6889            }
6890        }
6891        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
6892            Some(PrivilegedOperation::SetEnabled {
6893                module_id,
6894                enabled: *enabled,
6895            })
6896        }
6897        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
6898            Some(PrivilegedOperation::ReleaseReserved { module_id })
6899        }
6900        ClientControlRequest::ServerDescribe {}
6901        | ClientControlRequest::CatalogList { .. }
6902        | ClientControlRequest::RouteOpen { .. }
6903        | ClientControlRequest::RoutePoll { .. }
6904        | ClientControlRequest::SupervisorList {}
6905        | ClientControlRequest::SupervisorHealth {}
6906        | ClientControlRequest::SupervisorHealthProbe { .. }
6907        | ClientControlRequest::SupervisorRoutes { .. }
6908        | ClientControlRequest::SupervisorProvenance { .. }
6909        | ClientControlRequest::SupervisorStderrTail { .. }
6910        | ClientControlRequest::SupervisorTerminals { .. }
6911        | ClientControlRequest::SupervisorSpawnSnapshot {}
6912        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
6913    }
6914}
6915
6916fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6917    match request {
6918        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6919        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6920        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6921        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6922        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6923        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6924        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6925        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6926        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6927        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6928        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6929        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6930        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6931        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6932        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6933        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6934        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6935        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6936        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6937    }
6938}
6939
6940fn principal_label(principal: &Principal) -> String {
6941    match principal {
6942        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6943        Principal::Direct => "direct".to_string(),
6944        other => format!("{other:?}"),
6945    }
6946}
6947
6948fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6949    match request {
6950        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6951        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6952        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6953        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6954        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6955    }
6956}
6957
6958fn parse_client_control_request(
6959    body: &[u8],
6960) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6961    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6962        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6963            Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6964                ControlRequestBodyError::InvalidBody
6965            }
6966            Ok(_) => ControlRequestBodyError::UnknownOp,
6967            Err(_) => ControlRequestBodyError::InvalidBody,
6968        };
6969        (err, classification)
6970    })
6971}
6972
6973fn parse_module_control_request_from_module(
6974    body: &[u8],
6975) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6976    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6977        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6978            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6979            Ok(_) => ControlRequestBodyError::UnknownOp,
6980            Err(_) => ControlRequestBodyError::InvalidBody,
6981        };
6982        (err, classification)
6983    })
6984}
6985
6986#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6987enum ProviderRoleKind {
6988    ToolProvider,
6989    PipelineStage,
6990    ManagementSurface,
6991    InternalService,
6992}
6993
6994fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6995    match role {
6996        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6997        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6998        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6999        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7000    }
7001}
7002
7003fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7004    roles.iter().map(provider_role_kind).collect()
7005}
7006
7007/// Most refused scope records named individually in the log per sync; the
7008/// `refused` count on the accepted line is always complete.
7009const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7010
7011/// Per-outcome counts of one accepted `scope.sync`, for its log line.
7012#[derive(Debug, Default, PartialEq, Eq)]
7013struct ScopeOutcomeCounts {
7014    created: usize,
7015    replaced: usize,
7016    updated: usize,
7017    unchanged: usize,
7018    refused: usize,
7019}
7020
7021impl ScopeOutcomeCounts {
7022    fn of(results: &[ScopeRecordResult]) -> Self {
7023        let mut counts = Self::default();
7024        for result in results {
7025            let slot = match result.outcome {
7026                ScopeRecordOutcome::Created => &mut counts.created,
7027                ScopeRecordOutcome::Replaced => &mut counts.replaced,
7028                ScopeRecordOutcome::Updated => &mut counts.updated,
7029                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7030                ScopeRecordOutcome::Refused => &mut counts.refused,
7031            };
7032            *slot += 1;
7033        }
7034        counts
7035    }
7036}
7037
7038#[cfg(test)]
7039mod scope_outcome_count_tests {
7040    use super::*;
7041
7042    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7043        ScopeRecordResult {
7044            scope_ref: "r".to_string(),
7045            scope_epoch: 1,
7046            outcome,
7047            code: None,
7048            message: None,
7049            version: None,
7050            parent_state: None,
7051        }
7052    }
7053
7054    /// Each outcome lands in its own count, so a refused record can never be
7055    /// hidden inside the total the log already printed.
7056    #[test]
7057    fn every_outcome_is_counted_in_its_own_field() {
7058        let results = [
7059            result(ScopeRecordOutcome::Created),
7060            result(ScopeRecordOutcome::Created),
7061            result(ScopeRecordOutcome::Replaced),
7062            result(ScopeRecordOutcome::Updated),
7063            result(ScopeRecordOutcome::Unchanged),
7064            result(ScopeRecordOutcome::Refused),
7065            result(ScopeRecordOutcome::Refused),
7066            result(ScopeRecordOutcome::Refused),
7067        ];
7068        assert_eq!(
7069            ScopeOutcomeCounts::of(&results),
7070            ScopeOutcomeCounts {
7071                created: 2,
7072                replaced: 1,
7073                updated: 1,
7074                unchanged: 1,
7075                refused: 3,
7076            }
7077        );
7078    }
7079}
7080
7081/// Return whether a catalog change can create a newly violating live route.
7082/// Removing an attested claim is intentionally excluded: it makes fewer routes
7083/// forbidden and therefore must leave the existing route census untouched.
7084fn capability_census_trigger(
7085    old: Option<&CapabilityDeclarations>,
7086    new: Option<&CapabilityDeclarations>,
7087) -> bool {
7088    let old_provides = old
7089        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7090        .unwrap_or_default();
7091    let old_denies = old
7092        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7093        .unwrap_or_default();
7094    let new = new.cloned().unwrap_or(CapabilityDeclarations {
7095        provides: Vec::new(),
7096        requires: Vec::new(),
7097        must_never_reach: Vec::new(),
7098    });
7099
7100    new.provides
7101        .iter()
7102        .any(|capability| !old_provides.contains(capability))
7103        || new
7104            .must_never_reach
7105            .iter()
7106            .any(|capability| !old_denies.contains(capability))
7107}
7108
7109/// Find the first capability an attested opener denies that an attested target
7110/// claims. Both manifests are live registry records, never cached or client data.
7111fn denied_capability<'a>(
7112    opening_manifest: &'a ModuleManifest,
7113    target_manifest: &ModuleManifest,
7114) -> Option<&'a str> {
7115    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7116    let target_capabilities = target_manifest.capabilities.as_ref()?;
7117    opening_capabilities
7118        .must_never_reach
7119        .iter()
7120        .find(|denied| {
7121            target_capabilities
7122                .provides
7123                .iter()
7124                .any(|provided| provided == *denied)
7125        })
7126        .map(String::as_str)
7127}
7128
7129fn catalog_update_frozen_field_message(
7130    registered: &ModuleManifest,
7131    provides: &[ProviderRole],
7132) -> Option<String> {
7133    let old_has_provides = !registered.provides.is_empty();
7134    let new_has_provides = !provides.is_empty();
7135    if old_has_provides != new_has_provides {
7136        return Some(format!(
7137            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7138            registered.module_id
7139        ));
7140    }
7141
7142    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
7143        return Some(format!(
7144            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7145            registered.module_id
7146        ));
7147    }
7148
7149    let registered_concurrency = manifest_concurrency(registered);
7150    let mut candidate = registered.clone();
7151    candidate.provides = provides.to_vec();
7152    let candidate_concurrency = manifest_concurrency(&candidate);
7153    if candidate_concurrency != registered_concurrency {
7154        return Some(format!(
7155            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7156            registered.module_id, registered_concurrency, candidate_concurrency
7157        ));
7158    }
7159
7160    // control_ops live beside the manifest in the HELLO body, not inside
7161    // ModuleManifest, so a provides-only catalog.update cannot change them.
7162    None
7163}
7164
7165fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7166    manifest.provides.iter().any(is_routable_role)
7167}
7168
7169/// Returns the routable-provider concurrency subc should enforce for this manifest.
7170///
7171/// ToolProvider and ManagementSurface store their delivery concurrency directly.
7172/// InternalService has no role-specific concurrency field, so it retains the
7173/// existing ModuleManaged default for backward compatibility.
7174fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7175    manifest
7176        .provides
7177        .iter()
7178        .find_map(|provider| match provider {
7179            ProviderRole::ToolProvider { concurrency, .. }
7180            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7181            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7182        })
7183        .unwrap_or(Concurrency::ModuleManaged)
7184}
7185
7186/// True when the manifest carries a ManagementSurface role whose concurrency
7187/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
7188/// bytes because the typed manifest deliberately erases that distinction: the
7189/// default exists for wire compatibility, and this probe exists so the default
7190/// stays observable. Any parse irregularity returns false -- the caller only
7191/// logs, and a malformed body already failed registration upstream.
7192fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7193    let has_management_surface = manifest
7194        .provides
7195        .iter()
7196        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7197    if !has_management_surface {
7198        return false;
7199    }
7200    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7201        return false;
7202    };
7203    let Some(provides) = raw
7204        .get("manifest")
7205        .and_then(|manifest| manifest.get("provides"))
7206        .and_then(serde_json::Value::as_array)
7207    else {
7208        return false;
7209    };
7210    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
7211    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
7212    // against the management_surface_manifest_without_concurrency golden, not
7213    // recalled (the externally-tagged guess was this function's first bug).
7214    provides.iter().any(|role| {
7215        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7216            && role.get("concurrency").is_none()
7217    })
7218}
7219
7220fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7221    if peer_version != PROTOCOL_VERSION {
7222        return Err(format!(
7223            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7224        ));
7225    }
7226    Ok(PROTOCOL_VERSION)
7227}
7228
7229fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7230    Frame::build_with_version(
7231        response_version(frame),
7232        FrameType::Pong,
7233        frame.header.flags,
7234        0,
7235        0,
7236        frame.header.corr,
7237        Vec::new(),
7238    )
7239    .map_err(RouterError::FrameBuild)
7240}
7241
7242pub(crate) fn control_error_frame(
7243    frame: &Frame,
7244    code: &'static str,
7245    message: impl Into<String>,
7246) -> Result<Frame, RouterError> {
7247    control_error_body_frame(
7248        frame,
7249        ErrorBody {
7250            code: code.to_string(),
7251            message: message.into(),
7252            detail: None,
7253        },
7254    )
7255}
7256
7257fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7258    let body = serde_json::to_vec(&error).map_err(|err| {
7259        RouterError::backend(
7260            0,
7261            frame.header.corr,
7262            format!("failed to encode control ERROR: {err}"),
7263        )
7264    })?;
7265
7266    Frame::build_with_version(
7267        response_version(frame),
7268        FrameType::Error,
7269        control_flags(),
7270        0,
7271        0,
7272        frame.header.corr,
7273        body,
7274    )
7275    .map_err(RouterError::FrameBuild)
7276}
7277
7278fn control_response_body_frame<T: Serialize>(
7279    frame: &Frame,
7280    reply: &T,
7281    label: &'static str,
7282) -> Result<Frame, RouterError> {
7283    let body = serde_json::to_vec(reply).map_err(|err| {
7284        RouterError::backend(
7285            0,
7286            frame.header.corr,
7287            format!("failed to encode {label}: {err}"),
7288        )
7289    })?;
7290
7291    Frame::build_with_version(
7292        response_version(frame),
7293        FrameType::Response,
7294        control_flags(),
7295        0,
7296        0,
7297        frame.header.corr,
7298        body,
7299    )
7300    .map_err(RouterError::FrameBuild)
7301}
7302
7303/// Map a forwarding failure to the wire code a client sees.
7304///
7305/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
7306/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
7307/// chosen here decides whether a caller retries or gives up.
7308///
7309/// That makes attribution the load-bearing property, not merely having a code. A
7310/// permanent fault published as a retryable one produces a fleet-wide retry storm
7311/// against something that can never recover; a transient fault published as
7312/// permanent gives up on work that would have succeeded. Both look correct in a
7313/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
7314/// the mapping per variant rather than merely asserting that some code exists.
7315///
7316/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
7317/// two codes on the same side of the boundary passes it. Measured rather than
7318/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
7319/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
7320/// a test named for something else that happens to assert the string.
7321///
7322/// That accidental coverage is deliberately left alone rather than promoted to a
7323/// named test, because it guards a property this function does not promise.
7324/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
7325/// specific code within a class, so identity is free to change and only the
7326/// partition is a contract. Splitting it out would assert a guarantee nothing
7327/// depends on — and a suite that promises more than the code does is the harder
7328/// thing to correct later, because the next reader cannot tell which assertions
7329/// are load-bearing.
7330///
7331/// Pin identity here the moment a consumer branches on a specific code.
7332fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7333    match err {
7334        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7335        ForwardingError::NoModuleConnection => "target_unavailable",
7336        ForwardingError::ModuleReloading { .. } => "module_reloading",
7337        ForwardingError::ClientRouteChannelExhausted { .. }
7338        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7339        ForwardingError::StaleModuleEndpoint
7340        | ForwardingError::UnknownReservation { .. }
7341        | ForwardingError::ConnectionClosing { .. }
7342        | ForwardingError::ClientEgressClosed { .. }
7343        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7344        // Only a swap candidate's registration can produce this, and it means
7345        // exactly what a second active HELLO for a live id means.
7346        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7347        ForwardingError::RelayCorrelationExhausted
7348        | ForwardingError::RouteOpenBuild(_)
7349        | ForwardingError::Poisoned => "forwarding_error",
7350    }
7351}
7352
7353fn response_version(frame: &Frame) -> u8 {
7354    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7355        frame.header.ver
7356    } else {
7357        PROTOCOL_VERSION
7358    }
7359}
7360
7361fn control_flags() -> Flags {
7362    Flags::new(false, Priority::Passive, false)
7363}
7364
7365/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
7366/// channel. The target is the module (a client never saw the route), so this
7367/// takes the module path: delivered late rather than dropped when the module's
7368/// queue is momentarily full, and never closing its connection.
7369fn send_goodbye_target_best_effort(
7370    counters: &DaemonCounters,
7371    target: &GoodbyeTarget,
7372    context: &'static str,
7373) {
7374    let Ok(frame) = Frame::build_with_version(
7375        target.negotiated_ver,
7376        FrameType::Goodbye,
7377        control_flags(),
7378        target.channel,
7379        target.epoch,
7380        0,
7381        Vec::new(),
7382    ) else {
7383        return;
7384    };
7385    crate::forwarding::send_module_route_goodbye(
7386        counters,
7387        &target.sink,
7388        frame,
7389        target.module_id.as_deref(),
7390        context,
7391    );
7392}
7393
7394pub(crate) fn send_route_control_pushes(
7395    forwarding: &ForwardingTable,
7396    routes: Vec<EndpointRoute>,
7397    push: ClientControlPush,
7398) {
7399    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7400    for route in routes {
7401        let target = route.goodbye_target;
7402        if let Some((existing, channels)) = targets
7403            .iter_mut()
7404            .find(|(existing, _)| existing.connection_id == target.connection_id)
7405        {
7406            debug_assert_eq!(
7407                existing.negotiated_ver, target.negotiated_ver,
7408                "one connection cannot negotiate multiple frame versions"
7409            );
7410            if !channels.contains(&target.channel) {
7411                channels.push(target.channel);
7412            }
7413            continue;
7414        }
7415        let channel = target.channel;
7416        targets.push((target, vec![channel]));
7417    }
7418    for (target, mut channels) in targets {
7419        channels.sort_unstable();
7420        let mut push = push.clone();
7421        match &mut push {
7422            ClientControlPush::RouteClosing {
7423                channels: covered, ..
7424            }
7425            | ClientControlPush::RouteClosed {
7426                channels: covered, ..
7427            } => *covered = channels,
7428        }
7429        let body = match serde_json::to_vec(&push) {
7430            Ok(body) => body,
7431            Err(err) => {
7432                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7433                continue;
7434            }
7435        };
7436        let frame = match Frame::build_with_version(
7437            target.negotiated_ver,
7438            FrameType::Push,
7439            control_flags(),
7440            0,
7441            0,
7442            0,
7443            body.clone(),
7444        ) {
7445            Ok(frame) => frame,
7446            Err(err) => {
7447                warn!(
7448                    route_channel = target.channel,
7449                    error = %err,
7450                    "failed to build route lifecycle control PUSH frame"
7451                );
7452                continue;
7453            }
7454        };
7455        if let Err(err) = target.sink.try_send(frame) {
7456            if target.close_on_delivery_failure() {
7457                warn!(
7458                    target_connection_id = target.connection_id.get(),
7459                    route_channel = target.channel,
7460                    error = %err,
7461                    "route lifecycle control PUSH was not delivered to client; closing target connection"
7462                );
7463                let _ = forwarding.escalate_client_delivery_failure(
7464                    target.connection_id,
7465                    target.channel,
7466                    target.epoch,
7467                    CloseReason::new(
7468                        "route_lifecycle_push_delivery_failed",
7469                        format!(
7470                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7471                            target.channel
7472                        ),
7473                    ),
7474                    crate::forwarding::UndeliveredFrame {
7475                        module_id: target.module_id.as_deref(),
7476                        sink: &target.sink,
7477                    },
7478                );
7479            }
7480        }
7481    }
7482}
7483
7484#[cfg(test)]
7485mod tests {
7486    mod activity_read {
7487        include!("activity_read_tests.rs");
7488    }
7489
7490    mod lifecycle_activity {
7491        include!("control_lifecycle_tests.rs");
7492    }
7493    #[cfg(unix)]
7494    #[tokio::test]
7495    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7496        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7497        let path = dir.join("subc.jsonc");
7498        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7499            "program":"/bin/sleep","args":["60"],"protocol":"none",
7500            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7501        }}}).to_string()).unwrap();
7502        let mut configured = crate::daemon_config::load(&path)
7503            .unwrap()
7504            .unwrap()
7505            .modules
7506            .pop()
7507            .unwrap();
7508        let registry = std::sync::Arc::new(crate::Registry::default());
7509        let handle = crate::SupervisorHandle::new();
7510        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7511            .with_handle(handle.clone());
7512        let module = supervisor
7513            .supervise_configured_with_health(
7514                configured.module_spec(),
7515                true,
7516                configured.health.clone(),
7517                None,
7518                configured.restart,
7519            )
7520            .unwrap();
7521        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7522        let before = module.status().unwrap().pid;
7523        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7524        configured.health.cadence = std::time::Duration::from_secs(3600);
7525        let health_only = handler
7526            .reconcile_supervised_modules(
7527                &supervisor,
7528                vec![configured.clone()],
7529                false,
7530                Principal::Direct,
7531            )
7532            .await
7533            .unwrap();
7534        assert!(
7535            health_only.changed_pending_reload.is_empty(),
7536            "health policy is already applied live"
7537        );
7538        assert_eq!(module.status().unwrap().pid, before);
7539        assert_eq!(
7540            module.configuration().unwrap().1.http,
7541            configured.health.http
7542        );
7543        configured.args = vec!["61".into()];
7544        let launch = handler
7545            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7546            .await
7547            .unwrap();
7548        assert_eq!(launch.changed_pending_reload, ["stock"]);
7549        assert_eq!(
7550            module.status().unwrap().pid,
7551            before,
7552            "a launch edit is stored until reload"
7553        );
7554        module.drain().await.unwrap();
7555    }
7556    use cortexkit_test_support::ScratchDir;
7557    use std::{
7558        collections::BTreeMap,
7559        fmt,
7560        path::PathBuf,
7561        sync::{Arc, Mutex},
7562        time::Duration,
7563    };
7564
7565    use serde_json::{json, Value};
7566    use subc_protocol::{
7567        manifest::{
7568            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7569            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7570        },
7571        session::HealthStatus,
7572        FrameType,
7573    };
7574
7575    use super::*;
7576    use crate::{
7577        forwarding::{DataRoute, DataRouteState},
7578        registry::ChannelState,
7579        router::FrameSink,
7580        stderr_tail::DEFAULT_MAX_LINE_BYTES,
7581        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7582        RouteCtx, Router,
7583    };
7584    use tokio::{
7585        sync::mpsc,
7586        time::{sleep, Instant},
7587    };
7588    use tracing::{
7589        field::{Field, Visit},
7590        Event, Subscriber,
7591    };
7592    use tracing_subscriber::{layer::Context, prelude::*, Layer};
7593
7594    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
7595    ///
7596    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
7597    /// is only populated for `tests/*.rs` integration test binaries -- this file
7598    /// compiles as part of the library target, which gets neither. This test's
7599    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
7600    /// and the sibling binary lives two directories up at
7601    /// `<target-dir>/<profile>/fake-aft-stub`.
7602    ///
7603    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
7604    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
7605    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
7606    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
7607    /// `NotFound`, which reads as a broken test rather than an unbuilt
7608    /// dependency -- so state the cause and the remedy instead. Deliberately a
7609    /// panic and not a silent skip: a test that quietly passes when it could not
7610    /// run is worse than one that fails, because it reports health it never
7611    /// verified.
7612    fn fake_aft_stub_path() -> PathBuf {
7613        let mut path = std::env::current_exe().expect("current_exe available in tests");
7614        path.pop(); // .../deps/
7615        path.pop(); // .../<profile>/
7616        path.push(if cfg!(windows) {
7617            "fake-aft-stub.exe"
7618        } else {
7619            "fake-aft-stub"
7620        });
7621        assert!(
7622            path.exists(),
7623            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7624             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7625            path.display()
7626        );
7627        path
7628    }
7629
7630    /// Whether clients retry `code` in place: the predicate itself, never a copy
7631    /// of its set. A copied list breaks silently when a code is added to or
7632    /// removed from the real one, and a stale copy here would let exactly the
7633    /// failure this test exists to catch pass.
7634    fn client_retries(code: &str) -> bool {
7635        subc_protocol::error_codes::is_retryable_route_open(code)
7636    }
7637
7638    /// A code is not a label — clients branch on it, so publishing the wrong KIND
7639    /// of failure is worse than publishing none. A permanent fault dressed as
7640    /// retryable makes every client in the fleet retry forever against something
7641    /// that cannot recover; a transient fault dressed as permanent abandons work
7642    /// that would have succeeded.
7643    ///
7644    /// Asserting "a code exists" cannot catch either, because the string is free
7645    /// to say anything. This enumerates every variant and pins which side of the
7646    /// retry boundary it lands on, so a new variant must be classified here
7647    /// deliberately rather than inheriting whichever arm it was appended to.
7648    #[test]
7649    fn retryability_of_forwarding_codes_matches_the_failure() {
7650        // Transient by nature: the target is booting, reloading, or its endpoint
7651        // was swapped mid-flight. Retrying is how these resolve.
7652        let transient = [
7653            ForwardingError::NoModuleConnection,
7654            ForwardingError::ModuleReloading {
7655                module_id: "m".into(),
7656            },
7657            ForwardingError::StaleModuleEndpoint,
7658            ForwardingError::UnknownReservation {
7659                client_channel: 1,
7660                module_channel: 1,
7661            },
7662            ForwardingError::ConnectionClosing {
7663                connection_id: ConnectionId::new(1),
7664            },
7665            ForwardingError::ClientEgressClosed {
7666                connection_id: ConnectionId::new(1),
7667            },
7668            ForwardingError::ModuleEgressUnavailable {
7669                connection_id: ConnectionId::new(1),
7670            },
7671        ];
7672        for err in transient {
7673            let code = forwarding_error_code(&err);
7674            assert!(
7675                client_retries(code),
7676                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7677            );
7678        }
7679
7680        // Not fixed by retrying. Channel and correlation exhaustion need the
7681        // caller to close routes, and a poisoned lock is a daemon that cannot
7682        // recover at all — the worst thing to advertise as retryable, since every
7683        // client would storm a daemon that will never answer.
7684        let permanent = [
7685            ForwardingError::ConnectionRoleConflict {
7686                connection_id: ConnectionId::new(1),
7687            },
7688            ForwardingError::ClientRouteChannelExhausted {
7689                connection_id: ConnectionId::new(1),
7690            },
7691            ForwardingError::ModuleRouteChannelExhausted {
7692                endpoint: ModuleEndpointId {
7693                    connection_id: ConnectionId::new(1),
7694                    generation: 1,
7695                },
7696            },
7697            ForwardingError::RelayCorrelationExhausted,
7698            ForwardingError::RouteOpenBuild("x".into()),
7699            ForwardingError::Poisoned,
7700        ];
7701        for err in permanent {
7702            let code = forwarding_error_code(&err);
7703            assert!(
7704                !client_retries(code),
7705                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7706            );
7707        }
7708    }
7709
7710    /// The principal is the daemon's answer to "who is calling", and modules
7711    /// branch on it: aft gates bash on it, cerebellum gates browser control,
7712    /// plexus gates connector invocation. So a stamp is an authorization input in
7713    /// another process, not a label — and both possible answers SUCCEED, which is
7714    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
7715    /// first-party capability to something that never proved it; a supervised one
7716    /// stamped `Direct` silently strips a module of capability it is entitled to.
7717    ///
7718    /// Neither shows up in a test that only checks the bind succeeded. Before this
7719    /// test the only coverage was accidental —
7720    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
7721    /// stamped principal on its way past, so narrowing that wire-shape test to its
7722    /// stated subject would have deleted the last assertion on this value. It
7723    /// still asserts the stamp, which is now redundancy rather than the only
7724    /// guard: both fail under the same mutation, and this one names the reason.
7725    /// SCOPE: this handler's supervisor has spawned nothing, so
7726    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
7727    /// is unreachable here. Both assertions below are refusals, and a mutant that
7728    /// refuses everything would satisfy them.
7729    ///
7730    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
7731    /// spawns a supervised consumer, reads its live nonce, and asserts the module
7732    /// observed `principal.kind == "reserved"` carrying that module_id — verified
7733    /// at source rather than assumed, since a citation is a claim about another
7734    /// file and ages like one. Recorded because a harness that structurally
7735    /// cannot reach an arm reports "none" for that arm identically to one that
7736    /// covers it and found nothing.
7737    #[tokio::test]
7738    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7739        let handler = ControlHandler::default();
7740        let frame =
7741            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7742
7743        // Absent consumer_identity is the ordinary case: a human at a terminal, or
7744        // any process holding the connection file. Nothing was proved, so nothing
7745        // may be granted beyond the unattested floor.
7746        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7747        assert_eq!(
7748            stamped,
7749            Principal::Direct,
7750            "a caller that proved nothing must not be stamped as a supervised module"
7751        );
7752
7753        // A claimed module_id with a nonce no supervised child was given is a
7754        // forgery attempt, not a weaker caller: it must be REFUSED rather than
7755        // quietly demoted to Direct, or an impersonation attempt looks identical
7756        // to an ordinary unattested connection.
7757        let forged = handler
7758            .route_open_principal(
7759                &frame,
7760                Some(ConsumerIdentity {
7761                    module_id: "aft".to_string(),
7762                    launch_nonce: "not-a-real-nonce".to_string(),
7763                }),
7764            )
7765            .unwrap();
7766        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7767        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7768    }
7769
7770    /// The test above hands `route_open_principal` an identity it built itself,
7771    /// which proves the stamping rule and nothing about where the identity comes
7772    /// from. The real producer is a wire body, and the two are joined by a serde
7773    /// field name that nothing else asserts.
7774    ///
7775    /// That join fails quietly in one specific way: an unrecognised key is simply
7776    /// absent after parsing, so a renamed or misspelled `consumer_identity`
7777    /// yields `None` and every supervised module silently drops to `Direct`.
7778    /// Capability-wise that is the safe direction, but it surfaces far from its
7779    /// cause — as a module mysteriously refused bash — and it would pass every
7780    /// test that builds its own input.
7781    ///
7782    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
7783    /// would break every client the moment the daemon gains a field, trading a
7784    /// quiet demotion for a hard refusal on additive change. Asserting the join
7785    /// instead means a rename breaks a test here rather than the fleet.
7786    #[test]
7787    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7788        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7789        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7790        let ClientControlRequest::RouteOpen {
7791            consumer_identity, ..
7792        } = parsed
7793        else {
7794            panic!("route.open body must parse as RouteOpen");
7795        };
7796        assert_eq!(
7797            consumer_identity,
7798            Some(ConsumerIdentity {
7799                module_id: "aft".to_string(),
7800                launch_nonce: "n".to_string(),
7801            }),
7802            "the wire field name must reach the value route_open_principal reads"
7803        );
7804    }
7805
7806    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7807        ModuleManifest::builder(module_id, "0.1.0")
7808            .protocol_ver(protocol_ver)
7809            .provides(vec![ProviderRole::ToolProvider {
7810                tools: vec![Tool {
7811                    name: "read".to_string(),
7812                    description: None,
7813                    execution_mode: ExecutionMode::Pure,
7814                    schema: json!({"type": "object"}),
7815                }],
7816                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7817                concurrency: Concurrency::ModuleManaged,
7818                emits_push: true,
7819                sub_supervises: true,
7820            }])
7821            .build()
7822    }
7823
7824    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7825        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7826    }
7827
7828    fn hello_frame_with_control_ops(
7829        module_id: &str,
7830        protocol_ver: u8,
7831        corr: u64,
7832        control_ops: Option<Vec<String>>,
7833    ) -> Frame {
7834        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7835    }
7836
7837    fn hello_frame_with_nonce(
7838        module_id: &str,
7839        protocol_ver: u8,
7840        corr: u64,
7841        launch_nonce: Option<&str>,
7842    ) -> Frame {
7843        hello_frame_full(
7844            module_id,
7845            protocol_ver,
7846            corr,
7847            None,
7848            launch_nonce.map(ToOwned::to_owned),
7849        )
7850    }
7851
7852    fn hello_frame_full(
7853        module_id: &str,
7854        protocol_ver: u8,
7855        corr: u64,
7856        control_ops: Option<Vec<String>>,
7857        launch_nonce: Option<String>,
7858    ) -> Frame {
7859        let body = serde_json::to_vec(&ModuleHelloBody {
7860            manifest: manifest(module_id, protocol_ver),
7861            protocol_ver,
7862            control_ops,
7863            launch_nonce,
7864        })
7865        .unwrap();
7866        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7867    }
7868
7869    fn non_routable_hello_frame_with_control_ops(
7870        module_id: &str,
7871        corr: u64,
7872        control_ops: Option<Vec<String>>,
7873    ) -> Frame {
7874        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7875        manifest.provides.clear();
7876        let body = serde_json::to_vec(&ModuleHelloBody {
7877            manifest,
7878            protocol_ver: PROTOCOL_VERSION,
7879            control_ops,
7880            launch_nonce: None,
7881        })
7882        .unwrap();
7883        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7884    }
7885
7886    fn capability_grammar_hello_frame(
7887        capabilities: Value,
7888        runtime_computed: Option<Value>,
7889        corr: u64,
7890    ) -> Frame {
7891        let mut body = serde_json::to_value(ModuleHelloBody {
7892            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7893            protocol_ver: PROTOCOL_VERSION,
7894            control_ops: None,
7895            launch_nonce: None,
7896        })
7897        .expect("HELLO body serializes");
7898        body["manifest"]["capabilities"] = capabilities;
7899        if let Some(runtime_computed) = runtime_computed {
7900            body["runtime_computed"] = runtime_computed;
7901        }
7902        Frame::build(
7903            FrameType::Hello,
7904            control_flags(),
7905            0,
7906            0,
7907            corr,
7908            serde_json::to_vec(&body).expect("HELLO body reserializes"),
7909        )
7910        .expect("HELLO frame builds")
7911    }
7912
7913    fn channel_request(channel: u16, corr: u64) -> Frame {
7914        Frame::build(
7915            FrameType::Request,
7916            Flags::new(true, Priority::Interactive, false),
7917            channel,
7918            0,
7919            corr,
7920            b"opaque".to_vec(),
7921        )
7922        .unwrap()
7923    }
7924
7925    fn route_ctx(
7926        connection_id: ConnectionId,
7927    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7928        let (tx, rx) = mpsc::channel(8);
7929        (
7930            RouteCtx {
7931                connection_id,
7932                egress: FrameSink::new(tx),
7933            },
7934            rx,
7935        )
7936    }
7937
7938    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7939        serde_json::from_slice(&frame.body).unwrap()
7940    }
7941
7942    /// Register a module over a connection that has a sink and return the
7943    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
7944    /// module's own sink rather than returning it as a reply, so the ack is
7945    /// taken off `rx` here and whatever the test reads next is what followed it.
7946    async fn hello_via_sink(
7947        handler: &ControlHandler,
7948        ctx: &RouteCtx,
7949        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7950        hello: Frame,
7951    ) -> Frame {
7952        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7953        assert!(
7954            replies.is_empty(),
7955            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7956        );
7957        let ack = rx
7958            .try_recv()
7959            .expect("HELLO_ACK is queued on the module sink")
7960            .frame;
7961        assert_eq!(ack.header.ty, FrameType::HelloAck);
7962        ack
7963    }
7964
7965    fn parse_error(frame: &Frame) -> Value {
7966        serde_json::from_slice(&frame.body).unwrap()
7967    }
7968
7969    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7970        serde_json::from_slice(&frame.body).unwrap()
7971    }
7972
7973    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7974        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7975            route_channel,
7976            route_epoch: 0,
7977            kind,
7978        })
7979        .unwrap();
7980        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7981    }
7982
7983    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7984        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7985            module_id: module_id.to_string(),
7986        })
7987        .unwrap();
7988        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7989    }
7990
7991    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7992        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7993    }
7994
7995    fn route_open_frame_with_consumer_capabilities(
7996        corr: u64,
7997        module_id: &str,
7998        project_root: ScratchDir,
7999        consumer_capabilities: Option<Vec<String>>,
8000    ) -> Frame {
8001        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8002            target: RouteTarget::ToolProvider {
8003                module_id: module_id.to_string(),
8004            },
8005            identity: BindIdentity::new(
8006                project_root.path().to_path_buf(),
8007                "unit".to_string(),
8008                "session".to_string(),
8009            ),
8010            consumer_identity: None,
8011            consumer_capabilities,
8012            role_versions: None,
8013            admission_facts: None,
8014            scope: None,
8015        })
8016        .unwrap();
8017        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8018    }
8019
8020    fn route_open_frame_with_role_versions(
8021        corr: u64,
8022        module_id: &str,
8023        project_root: ScratchDir,
8024        role_versions: Option<BTreeMap<String, String>>,
8025    ) -> Frame {
8026        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8027            target: RouteTarget::ToolProvider {
8028                module_id: module_id.to_string(),
8029            },
8030            identity: BindIdentity::new(
8031                project_root.path().to_path_buf(),
8032                "unit".to_string(),
8033                format!("session-{corr}"),
8034            ),
8035            consumer_identity: None,
8036            consumer_capabilities: None,
8037            role_versions,
8038            admission_facts: None,
8039            scope: None,
8040        })
8041        .unwrap();
8042        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8043    }
8044
8045    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8046        entries
8047            .iter()
8048            .map(|(role, version)| (role.to_string(), version.to_string()))
8049            .collect()
8050    }
8051
8052    fn route_open_frame_with_admission_facts(
8053        corr: u64,
8054        module_id: &str,
8055        project_root: ScratchDir,
8056        consumer_identity: Option<subc_control::ConsumerIdentity>,
8057        facts: Option<Value>,
8058    ) -> Frame {
8059        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8060            target: RouteTarget::ToolProvider {
8061                module_id: module_id.to_string(),
8062            },
8063            identity: BindIdentity::new(
8064                project_root.path().to_path_buf(),
8065                "unit".to_string(),
8066                format!("session-{corr}"),
8067            ),
8068            consumer_identity,
8069            consumer_capabilities: None,
8070            role_versions: None,
8071            admission_facts: facts,
8072            scope: None,
8073        })
8074        .unwrap();
8075        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8076    }
8077
8078    #[derive(Clone, Default)]
8079    struct EventCapture {
8080        events: Arc<Mutex<Vec<CapturedEvent>>>,
8081    }
8082
8083    #[derive(Clone, Debug)]
8084    struct CapturedEvent {
8085        target: String,
8086        level: tracing::Level,
8087        fields: BTreeMap<String, String>,
8088    }
8089
8090    impl EventCapture {
8091        fn events(&self) -> Vec<CapturedEvent> {
8092            self.events.lock().unwrap().clone()
8093        }
8094    }
8095
8096    impl<S> Layer<S> for EventCapture
8097    where
8098        S: Subscriber,
8099    {
8100        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8101            let mut visitor = EventFieldVisitor::default();
8102            event.record(&mut visitor);
8103            self.events.lock().unwrap().push(CapturedEvent {
8104                target: event.metadata().target().to_string(),
8105                level: *event.metadata().level(),
8106                fields: visitor.fields,
8107            });
8108        }
8109    }
8110
8111    #[derive(Default)]
8112    struct EventFieldVisitor {
8113        fields: BTreeMap<String, String>,
8114    }
8115
8116    impl Visit for EventFieldVisitor {
8117        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8118            self.fields
8119                .insert(field.name().to_string(), format!("{value:?}"));
8120        }
8121    }
8122
8123    fn health_response(corr: u64, status: HealthStatus) -> Frame {
8124        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8125            status,
8126            detail: Some("warming".to_string()),
8127            metrics: Some(json!({"queue_depth": 3})),
8128        })
8129        .unwrap();
8130        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8131    }
8132
8133    fn route_bind_ack(corr: u64) -> Frame {
8134        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8135        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8136    }
8137
8138    fn unique_project_root(label: &str) -> ScratchDir {
8139        ScratchDir::new(label)
8140    }
8141
8142    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8143        match parse_route_poll(frame) {
8144            ClientControlResponse::RoutePoll {
8145                status: None,
8146                live: Some(live),
8147                ..
8148            } => assert_eq!(live, expected_live),
8149            other => panic!("unexpected route.poll response: {other:?}"),
8150        }
8151    }
8152
8153    fn bind_liveness_route(
8154        registry: &Registry,
8155        forwarding: &ForwardingTable,
8156        module_id: &str,
8157    ) -> (RouteCtx, u16, u32) {
8158        let module_connection = ConnectionId::new(101);
8159        let client_connection = ConnectionId::new(202);
8160        let registration = registry
8161            .register_with_control_ops(
8162                manifest(module_id, PROTOCOL_VERSION),
8163                PROTOCOL_VERSION,
8164                module_connection,
8165                module_baseline_control_ops(),
8166            )
8167            .unwrap();
8168        let (module_tx, _module_rx) = mpsc::channel(8);
8169        let endpoint = forwarding
8170            .register_module_connection(
8171                module_connection,
8172                module_id.to_string(),
8173                PROTOCOL_VERSION,
8174                manifest_concurrency(&registration.manifest),
8175                FrameSink::new(module_tx),
8176            )
8177            .unwrap();
8178        let (client_ctx, _client_rx) = route_ctx(client_connection);
8179        let pending = forwarding
8180            .begin_route_bind_relay_for_test(
8181                client_connection,
8182                client_ctx.egress.clone(),
8183                1,
8184                module_id,
8185            )
8186            .unwrap();
8187        assert_eq!(pending.endpoint, endpoint);
8188        let route_channel = pending.client_channel;
8189        let route_epoch = pending.client_epoch;
8190        forwarding
8191            .complete_pending_relay(
8192                module_connection,
8193                pending.corr,
8194                RouteBindRelayOutcome::Accepted,
8195            )
8196            .unwrap();
8197        (client_ctx, route_channel, route_epoch)
8198    }
8199
8200    struct FakeProcessLiveness {
8201        live: Option<bool>,
8202    }
8203
8204    impl ModuleProcessLiveness for FakeProcessLiveness {
8205        fn process_live(&self, _module_id: &str) -> Option<bool> {
8206            self.live
8207        }
8208    }
8209
8210    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8211    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8212    {
8213        let registry = Arc::new(Registry::default());
8214        let supervisor_handle = SupervisorHandle::new();
8215        let supervisor = Supervisor::new_for_test(
8216            Arc::clone(&registry),
8217            RestartPolicy::new(1, Duration::from_millis(10)),
8218        )
8219        .with_handle(supervisor_handle.clone());
8220        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8221        let module = supervisor
8222            .spawn(ModuleSpec {
8223                module_id: "stderr-tail-wire".to_string(),
8224                program: fake_aft_stub_path(),
8225                args: Vec::new(),
8226                env: vec![
8227                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8228                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8229                ],
8230                reserved: false,
8231                reserved_prefixes: Vec::new(),
8232                protocol: ModuleProtocol::Subc,
8233                overlap: Default::default(),
8234            })
8235            .unwrap();
8236
8237        let deadline = Instant::now() + Duration::from_secs(5);
8238        loop {
8239            let tail = module.stderr_tail(None, None);
8240            if tail
8241                .entries
8242                .iter()
8243                .any(|entry| matches!(entry, TailEntry::ProcessStart))
8244                && tail.entries.iter().any(|entry| {
8245                    matches!(
8246                        entry,
8247                        TailEntry::Line {
8248                            truncated: true,
8249                            ..
8250                        }
8251                    )
8252                })
8253            {
8254                break;
8255            }
8256            assert!(
8257                Instant::now() < deadline,
8258                "module did not produce a truncated line and restart boundary: {tail:?}"
8259            );
8260            sleep(Duration::from_millis(10)).await;
8261        }
8262
8263        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8264        let request = ClientControlRequest::SupervisorStderrTail {
8265            module_id: "stderr-tail-wire".to_string(),
8266            max_lines: None,
8267            max_bytes: None,
8268        };
8269        let frame = Frame::build(
8270            FrameType::Request,
8271            control_flags(),
8272            0,
8273            0,
8274            1,
8275            serde_json::to_vec(&request).unwrap(),
8276        )
8277        .unwrap();
8278        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8279        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8280        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8281            serde_json::from_slice(&responses[0].body).unwrap()
8282        else {
8283            panic!("expected supervisor.stderr_tail response");
8284        };
8285
8286        assert!(
8287            tail.entries
8288                .iter()
8289                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8290            "the control response lost the restart boundary"
8291        );
8292        let Some(StderrTailEntry::Line {
8293            text,
8294            truncated,
8295            at_ms,
8296        }) = tail.entries.iter().find(|entry| {
8297            matches!(
8298                entry,
8299                StderrTailEntry::Line {
8300                    truncated: true,
8301                    ..
8302                }
8303            )
8304        })
8305        else {
8306            panic!("the control response lost the truncated line");
8307        };
8308        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8309        assert!(*truncated);
8310        assert!(
8311            at_ms.is_some(),
8312            "the control response lost the line's capture time"
8313        );
8314    }
8315
8316    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
8317    /// read done on the worker thread would stall every other task until it
8318    /// finished; the read must run off the worker so this test's own task keeps
8319    /// running while the read is paused.
8320    #[tokio::test(flavor = "current_thread")]
8321    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8322        let dir = ScratchDir::new("terminals-off-worker");
8323        let journal_path = dir.join("terminals.jsonl");
8324        let registry = Arc::new(Registry::default());
8325        let supervisor_handle = SupervisorHandle::new();
8326        let supervisor =
8327            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8328                .with_handle(supervisor_handle.clone())
8329                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8330        let module = supervisor
8331            .spawn(ModuleSpec {
8332                module_id: "terminal-off-worker".to_string(),
8333                program: fake_aft_stub_path(),
8334                args: Vec::new(),
8335                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8336                reserved: false,
8337                reserved_prefixes: Vec::new(),
8338                protocol: ModuleProtocol::Subc,
8339                overlap: Default::default(),
8340            })
8341            .unwrap();
8342        let deadline = Instant::now() + Duration::from_secs(5);
8343        while module.terminal_history().entries.len() != 2 {
8344            assert!(Instant::now() < deadline, "module did not record two exits");
8345            sleep(Duration::from_millis(10)).await;
8346        }
8347
8348        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8349        let handler =
8350            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
8351        let frame = Frame::build(
8352            FrameType::Request,
8353            control_flags(),
8354            0,
8355            0,
8356            1,
8357            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8358                module_id: "terminal-off-worker".to_string(),
8359            })
8360            .unwrap(),
8361        )
8362        .unwrap();
8363        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8364        let spawned_at = std::time::Instant::now();
8365        let read = tokio::spawn({
8366            let handler = Arc::clone(&handler);
8367            async move { handler.handle_control_frame(&ctx, frame).await }
8368        });
8369        // Waiting for the pause from a blocking thread keeps this task pending,
8370        // so the runtime's single worker is free to run the read task.
8371        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8372            .await
8373            .unwrap()
8374            .expect("the history read reached its pause");
8375        let elapsed = spawned_at.elapsed();
8376        assert!(
8377            elapsed < Duration::from_secs(2) && !read.is_finished(),
8378            "this task could not run while the history read was paused \
8379             (resumed after {elapsed:?}, read finished: {})",
8380            read.is_finished()
8381        );
8382
8383        drop(release);
8384        let responses = read.await.unwrap().unwrap();
8385        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8386        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8387            panic!("expected supervisor.terminals response");
8388        };
8389        assert_eq!(terminals.entries.len(), 2);
8390        assert_eq!(terminals.journal_skipped_lines, 0);
8391        assert_eq!(terminals.journal_read_errors, 0);
8392    }
8393
8394    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8395    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8396        let registry = Arc::new(Registry::default());
8397        let supervisor_handle = SupervisorHandle::new();
8398        let supervisor =
8399            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8400                .with_handle(supervisor_handle.clone());
8401        let module = supervisor
8402            .spawn(ModuleSpec {
8403                module_id: "terminal-golden".to_string(),
8404                program: fake_aft_stub_path(),
8405                args: Vec::new(),
8406                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8407                reserved: false,
8408                reserved_prefixes: Vec::new(),
8409                protocol: ModuleProtocol::Subc,
8410                overlap: Default::default(),
8411            })
8412            .unwrap();
8413
8414        let deadline = Instant::now() + Duration::from_secs(5);
8415        while module.terminal_history().entries.len() != 2 {
8416            assert!(
8417                Instant::now() < deadline,
8418                "module did not retain two terminal exits: {:?}",
8419                module.terminal_history()
8420            );
8421            sleep(Duration::from_millis(10)).await;
8422        }
8423
8424        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8425        let request = ClientControlRequest::SupervisorTerminals {
8426            module_id: "terminal-golden".to_string(),
8427        };
8428        let frame = Frame::build(
8429            FrameType::Request,
8430            control_flags(),
8431            0,
8432            0,
8433            1,
8434            serde_json::to_vec(&request).unwrap(),
8435        )
8436        .unwrap();
8437        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8438        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8439        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8440        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8441            panic!("expected supervisor.terminals response");
8442        };
8443        assert_eq!(terminals.entries.len(), 2);
8444        assert_eq!(terminals.dropped, 0);
8445
8446        let mut rendered = serde_json::to_value(response).unwrap();
8447        // Wall-clock fields are the observation contract, but not stable fixture
8448        // bytes; normalize only them after the real handler has shaped the response.
8449        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8450        for (index, entry) in rendered["entries"]
8451            .as_array_mut()
8452            .expect("terminal response entries array")
8453            .iter_mut()
8454            .enumerate()
8455        {
8456            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8457        }
8458
8459        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8460            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8461        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8462        if std::env::var_os("UPDATE_GOLDEN").is_some() {
8463            std::fs::write(&golden_path, &serialized).unwrap();
8464        }
8465        let expected: Value =
8466            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8467        assert_eq!(rendered, expected);
8468    }
8469
8470    #[test]
8471    fn hello_registers_manifest_and_returns_ack() {
8472        let registry = Arc::new(Registry::default());
8473        let handler = ControlHandler::new(Arc::clone(&registry));
8474        let conn = ConnectionId::new(1);
8475
8476        let responses = handler
8477            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8478            .unwrap();
8479
8480        assert_eq!(responses.len(), 1);
8481        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8482        assert_eq!(responses[0].header.channel, 0);
8483        assert_eq!(responses[0].header.corr, 7);
8484        let ack = parse_ack(&responses[0]);
8485        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8486        assert!(ack
8487            .subc_capabilities
8488            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8489        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8490        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8491        assert!(ack
8492            .subc_ops
8493            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8494        assert!(ack
8495            .subc_ops
8496            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8497
8498        let registration = registry.get_module("aft").unwrap().unwrap();
8499        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8500        assert_eq!(registration.state, ChannelState::Active);
8501        assert_eq!(registration.connection_id, conn);
8502        assert_eq!(registration.control_ops, module_baseline_control_ops());
8503    }
8504
8505    #[test]
8506    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8507        let invalid_identifiers = [
8508            ("case_change", "credentials-Provider/v1"),
8509            ("leading_zero", "credentials-provider/v01"),
8510            ("trailing_hyphen", "credentials-provider-/v1"),
8511            ("consecutive_hyphens", "credentials--provider/v1"),
8512            ("uppercase", "Credentials-provider/v1"),
8513            ("missing_v", "credentials-provider/1"),
8514            ("whitespace", "credentials provider/v1"),
8515            ("zero_version", "credentials-provider/v0"),
8516            ("out_of_range_version", "credentials-provider/v4294967296"),
8517            (
8518                "overlength_name",
8519                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8520            ),
8521        ];
8522        let mut cases = invalid_identifiers
8523            .into_iter()
8524            .map(|(name, identifier)| {
8525                (
8526                    format!("identifier_{name}"),
8527                    "capabilities.provides[0]".to_string(),
8528                    identifier.to_string(),
8529                    json!({ "provides": [identifier] }),
8530                    None,
8531                )
8532            })
8533            .collect::<Vec<_>>();
8534        cases.extend([
8535            (
8536                "unknown_need".to_string(),
8537                "capabilities.requires[0].need".to_string(),
8538                "deferred".to_string(),
8539                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8540                None,
8541            ),
8542            (
8543                "duplicate_provides".to_string(),
8544                "capabilities.provides[1]".to_string(),
8545                "credentials-provider/v1".to_string(),
8546                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8547                None,
8548            ),
8549            (
8550                "duplicate_must_never_reach".to_string(),
8551                "capabilities.must_never_reach[1]".to_string(),
8552                "credentials-provider/v1".to_string(),
8553                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8554                None,
8555            ),
8556            (
8557                "duplicate_requires_same_need".to_string(),
8558                "capabilities.requires[1]".to_string(),
8559                "credentials-provider/v1".to_string(),
8560                json!({ "requires": [
8561                    { "capability": "credentials-provider/v1", "need": "required" },
8562                    { "capability": "credentials-provider/v1", "need": "required" }
8563                ] }),
8564                None,
8565            ),
8566            (
8567                "duplicate_requires_conflicting_need".to_string(),
8568                "capabilities.requires[1]".to_string(),
8569                "credentials-provider/v1".to_string(),
8570                json!({ "requires": [
8571                    { "capability": "credentials-provider/v1", "need": "required" },
8572                    { "capability": "credentials-provider/v1", "need": "optional" }
8573                ] }),
8574                None,
8575            ),
8576            (
8577                "capabilities_root_pointer".to_string(),
8578                "runtime_computed[0]".to_string(),
8579                "/capabilities".to_string(),
8580                json!({}),
8581                Some(json!(["/capabilities"])),
8582            ),
8583            (
8584                "capabilities_descendant_pointer".to_string(),
8585                "runtime_computed[0]".to_string(),
8586                "/capabilities/provides".to_string(),
8587                json!({}),
8588                Some(json!(["/capabilities/provides"])),
8589            ),
8590            (
8591                "malformed_pointer_without_leading_slash".to_string(),
8592                "runtime_computed[0]".to_string(),
8593                "capabilities".to_string(),
8594                json!({}),
8595                Some(json!(["capabilities"])),
8596            ),
8597            (
8598                "malformed_pointer_escape".to_string(),
8599                "runtime_computed[0]".to_string(),
8600                "/roles/~2/tools".to_string(),
8601                json!({}),
8602                Some(json!(["/roles/~2/tools"])),
8603            ),
8604            (
8605                "unknown_capabilities_field".to_string(),
8606                "capabilities.future".to_string(),
8607                "<array>".to_string(),
8608                json!({ "future": [] }),
8609                None,
8610            ),
8611        ]);
8612
8613        for (index, (name, field, value, capabilities, runtime_computed)) in
8614            cases.into_iter().enumerate()
8615        {
8616            let registry = Arc::new(Registry::default());
8617            let handler = ControlHandler::new(Arc::clone(&registry));
8618            let response = handler
8619                .handle_control(
8620                    ConnectionId::new((index + 1) as u64),
8621                    capability_grammar_hello_frame(
8622                        capabilities,
8623                        runtime_computed,
8624                        index as u64 + 1,
8625                    ),
8626                )
8627                .expect("invalid HELLO returns a refusal");
8628
8629            assert_eq!(response.len(), 1, "{name} must emit one refusal");
8630            let error = parse_error(&response[0]);
8631            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8632            let message = error["message"]
8633                .as_str()
8634                .expect("error message is a string");
8635            assert!(
8636                message.contains(&field),
8637                "{name}: field missing from {message}"
8638            );
8639            assert!(
8640                message.contains(&value),
8641                "{name}: value missing from {message}"
8642            );
8643            assert_eq!(
8644                registry
8645                    .active_registration_count()
8646                    .expect("registry reads"),
8647                0,
8648                "{name}: refused HELLO must not create a catalog entry"
8649            );
8650        }
8651    }
8652
8653    #[test]
8654    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8655        let registry = Arc::new(Registry::default());
8656        let handler = ControlHandler::new(Arc::clone(&registry));
8657        let capabilities = json!({
8658            "provides": ["credentials-provider/v1"],
8659            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8660            "must_never_reach": ["federation-transport/v1"]
8661        });
8662        let response = handler
8663            .handle_control(
8664                ConnectionId::new(99),
8665                capability_grammar_hello_frame(
8666                    capabilities.clone(),
8667                    Some(json!(["/roles/0/tools"])),
8668                    99,
8669                ),
8670            )
8671            .expect("valid HELLO registers");
8672        assert_eq!(response[0].header.ty, FrameType::HelloAck);
8673
8674        let request = Frame::build(
8675            FrameType::Request,
8676            control_flags(),
8677            0,
8678            0,
8679            100,
8680            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8681                .expect("catalog request serializes"),
8682        )
8683        .expect("catalog request frame builds");
8684        let response = handler
8685            .handle_catalog_list(request, None)
8686            .expect("catalog list succeeds");
8687        let ClientControlResponse::CatalogList { modules, .. } =
8688            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8689        else {
8690            panic!("catalog request must return catalog.list");
8691        };
8692        assert_eq!(modules.len(), 1);
8693        assert_eq!(
8694            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8695            capabilities
8696        );
8697    }
8698
8699    #[test]
8700    fn catalog_list_mirrors_management_operation_description() {
8701        let registry = Arc::new(Registry::default());
8702        let handler = ControlHandler::new(Arc::clone(&registry));
8703        let description = "List managed records and return their identifiers and metadata.";
8704        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8705        manifest.provides = vec![ProviderRole::ManagementSurface {
8706            operations: vec![ManagementOperation {
8707                name: "records.list".to_string(),
8708                kind: ManagementOperationKind::Query,
8709                description: Some(description.to_string()),
8710            }],
8711            config_schema: json!({"type": "object"}),
8712            observability: vec![ObservabilitySurface {
8713                name: "records.stats".to_string(),
8714                kind: ObservabilityKind::Snapshot,
8715            }],
8716            identity_scope: vec![IdentityScope::Project],
8717            concurrency: Concurrency::ModuleManaged,
8718        }];
8719        registry
8720            .register_with_control_ops(
8721                manifest,
8722                PROTOCOL_VERSION,
8723                ConnectionId::new(99),
8724                Vec::new(),
8725            )
8726            .expect("described management manifest registers");
8727
8728        let request = Frame::build(
8729            FrameType::Request,
8730            control_flags(),
8731            0,
8732            0,
8733            100,
8734            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8735                .expect("catalog request serializes"),
8736        )
8737        .expect("catalog request frame builds");
8738        let response = handler
8739            .handle_catalog_list(request, None)
8740            .expect("catalog list succeeds");
8741        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8742        assert_eq!(
8743            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8744            "catalog.list must preserve the declared operation description verbatim"
8745        );
8746    }
8747
8748    #[test]
8749    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8750        let registry = Arc::new(Registry::default());
8751        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
8752            [("vault".to_string(), true), ("squatter".to_string(), true)],
8753            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8754        );
8755        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8756        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8757            provides: vec!["credentials-provider/v1".to_string()],
8758            requires: Vec::new(),
8759            must_never_reach: Vec::new(),
8760        });
8761        let frame = Frame::build(
8762            FrameType::Hello,
8763            control_flags(),
8764            0,
8765            0,
8766            77,
8767            serde_json::to_vec(&ModuleHelloBody {
8768                manifest: squatter,
8769                protocol_ver: PROTOCOL_VERSION,
8770                control_ops: None,
8771                launch_nonce: None,
8772            })
8773            .expect("HELLO serializes"),
8774        )
8775        .expect("HELLO frame builds");
8776        let response = handler
8777            .handle_control(ConnectionId::new(77), frame)
8778            .expect("reserved claim receives a typed refusal");
8779        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8780        assert_eq!(
8781            registry
8782                .active_registration_count()
8783                .expect("registry reads"),
8784            0,
8785            "a reserved capability refusal must not leave a catalog entry"
8786        );
8787    }
8788
8789    #[test]
8790    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8791        for settlement in ["timeout", "inconclusive", "drop"] {
8792            let breakers = RouteBindBreakers::default();
8793            let RouteBindAdmission::Admitted {
8794                guard: mut old,
8795                probe: false,
8796            } = breakers.admit("prov")
8797            else {
8798                panic!("ordinary relay admitted")
8799            };
8800            let RouteBindAdmission::Admitted {
8801                guard: mut opener, ..
8802            } = breakers.admit("prov")
8803            else {
8804                panic!("second relay admitted")
8805            };
8806            assert!(
8807                !opener
8808                    .record_timeout(1, Duration::ZERO)
8809                    .unwrap()
8810                    .reopened_after_probe
8811            );
8812            let RouteBindAdmission::Admitted {
8813                guard: mut probe,
8814                probe: true,
8815            } = breakers.admit("prov")
8816            else {
8817                panic!("one cooldown probe admitted")
8818            };
8819            match settlement {
8820                "timeout" => assert!(
8821                    !old.record_timeout(1, Duration::ZERO)
8822                        .unwrap()
8823                        .reopened_after_probe
8824                ),
8825                "inconclusive" => old.record_inconclusive(),
8826                "drop" => drop(old),
8827                _ => unreachable!(),
8828            }
8829            assert!(
8830                matches!(
8831                    breakers.admit("prov"),
8832                    RouteBindAdmission::Refused {
8833                        probe_in_flight: true,
8834                        ..
8835                    }
8836                ),
8837                "{settlement} of a pre-open relay cannot release the real probe"
8838            );
8839            assert!(
8840                probe
8841                    .record_timeout(1, Duration::ZERO)
8842                    .unwrap()
8843                    .reopened_after_probe
8844            );
8845            assert!(matches!(
8846                breakers.admit("prov"),
8847                RouteBindAdmission::Admitted { probe: true, .. }
8848            ));
8849        }
8850        let breakers = RouteBindBreakers::default();
8851        let admit = || match breakers.admit("prov") {
8852            RouteBindAdmission::Admitted { guard, .. } => guard,
8853            _ => panic!("relay admitted"),
8854        };
8855        admit().record_timeout(1, Duration::ZERO);
8856        let mut old_probe = admit();
8857        breakers.reset_for_new_module_connection("prov");
8858        admit().record_timeout(1, Duration::ZERO);
8859        let _new_probe = admit();
8860        old_probe.record_inconclusive();
8861        assert!(matches!(
8862            breakers.admit("prov"),
8863            RouteBindAdmission::Refused {
8864                probe_in_flight: true,
8865                ..
8866            }
8867        ));
8868    }
8869
8870    #[tokio::test]
8871    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8872        for candidate in [false, true] {
8873            let registry = Arc::new(Registry::default());
8874            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
8875                [("vault".to_string(), true), ("squatter".to_string(), true)],
8876                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8877            );
8878            let conn = ConnectionId::new(77);
8879            let (ctx, mut rx) = route_ctx(conn);
8880            let initial = capability_manifest("squatter", &[], &[]);
8881            if candidate {
8882                registry
8883                    .register_candidate_with_control_ops(
8884                        initial.clone(),
8885                        PROTOCOL_VERSION,
8886                        conn,
8887                        module_baseline_control_ops(),
8888                    )
8889                    .unwrap();
8890                handler
8891                    .forwarding
8892                    .register_candidate_module_connection(
8893                        conn,
8894                        "squatter".to_string(),
8895                        PROTOCOL_VERSION,
8896                        manifest_concurrency(&initial),
8897                        ctx.egress.clone(),
8898                    )
8899                    .unwrap();
8900            } else {
8901                hello_via_sink(
8902                    &handler,
8903                    &ctx,
8904                    &mut rx,
8905                    hello_frame_with_manifest(initial.clone(), 1),
8906                )
8907                .await;
8908            }
8909            let response = handler
8910                .handle_control_frame(
8911                    &ctx,
8912                    catalog_update_with_capabilities_frame(
8913                        2,
8914                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
8915                            .capabilities
8916                            .unwrap(),
8917                    ),
8918                )
8919                .await
8920                .unwrap();
8921            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8922            assert_eq!(
8923                registry
8924                    .get_module_by_connection(conn)
8925                    .unwrap()
8926                    .unwrap()
8927                    .manifest,
8928                initial
8929            );
8930        }
8931    }
8932
8933    #[test]
8934    fn server_describe_surfaces_required_capability_verdict_fields() {
8935        let registry = Arc::new(Registry::default());
8936        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
8937            [
8938                ("consumer".to_string(), true),
8939                ("provider".to_string(), false),
8940            ],
8941            BTreeMap::new(),
8942        );
8943        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8944        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8945            provides: Vec::new(),
8946            requires: vec![subc_protocol::manifest::CapabilityRequirement {
8947                capability: "credentials-provider/v1".to_string(),
8948                need: subc_protocol::manifest::CapabilityNeed::Required,
8949            }],
8950            must_never_reach: Vec::new(),
8951        });
8952        let hello = Frame::build(
8953            FrameType::Hello,
8954            control_flags(),
8955            0,
8956            0,
8957            78,
8958            serde_json::to_vec(&ModuleHelloBody {
8959                manifest: consumer,
8960                protocol_ver: PROTOCOL_VERSION,
8961                control_ops: None,
8962                launch_nonce: None,
8963            })
8964            .expect("HELLO serializes"),
8965        )
8966        .expect("HELLO frame builds");
8967        handler
8968            .handle_control(ConnectionId::new(78), hello)
8969            .expect("consumer registers");
8970        let describe = Frame::build(
8971            FrameType::Request,
8972            control_flags(),
8973            0,
8974            0,
8975            79,
8976            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8977                .expect("request serializes"),
8978        )
8979        .expect("describe frame builds");
8980        let response = handler
8981            .handle_server_describe(describe)
8982            .expect("server.describe succeeds");
8983        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8984        let requirement = &rendered["capability_requirements"][0];
8985        assert_eq!(requirement["consumer"], "consumer");
8986        assert_eq!(requirement["verdict"], "never_provided");
8987        assert_eq!(requirement["episode_seq"], 1);
8988        assert_eq!(requirement["config_satisfiable"], false);
8989        assert_eq!(requirement["runtime_available"], false);
8990        assert!(requirement["detail"]
8991            .as_str()
8992            .expect("detail string")
8993            .contains("credentials-provider/v1"));
8994    }
8995
8996    #[test]
8997    fn catalog_list_omits_capabilities_for_legacy_manifest() {
8998        let registry = Arc::new(Registry::default());
8999        let handler = ControlHandler::new(Arc::clone(&registry));
9000        let hello = handler
9001            .handle_control(
9002                ConnectionId::new(101),
9003                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9004            )
9005            .expect("legacy HELLO registers");
9006        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9007
9008        let request = Frame::build(
9009            FrameType::Request,
9010            control_flags(),
9011            0,
9012            0,
9013            102,
9014            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9015                .expect("catalog request serializes"),
9016        )
9017        .expect("catalog request frame builds");
9018        let response = handler
9019            .handle_catalog_list(request, None)
9020            .expect("catalog list succeeds");
9021        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9022        assert!(
9023            body["modules"][0].get("capabilities").is_none(),
9024            "legacy manifest must retain an absent capabilities field on catalog.list"
9025        );
9026    }
9027
9028    #[test]
9029    fn hello_ack_omits_storage_when_no_storage_config() {
9030        let registry = Arc::new(Registry::default());
9031        let handler = ControlHandler::new(Arc::clone(&registry));
9032        let responses = handler
9033            .handle_control(
9034                ConnectionId::new(1),
9035                hello_frame("aft", PROTOCOL_VERSION, 7),
9036            )
9037            .unwrap();
9038        let ack = parse_ack(&responses[0]);
9039        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9040        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9041    }
9042
9043    #[tokio::test]
9044    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9045        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9046        let registry = Arc::new(Registry::default());
9047        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
9048        let responses = handler
9049            .handle_control(
9050                ConnectionId::new(1),
9051                hello_frame("aft", PROTOCOL_VERSION, 7),
9052            )
9053            .unwrap();
9054        let ack = parse_ack(&responses[0]);
9055        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9056
9057        let described = handler
9058            .handle_control_frame(
9059                &route_ctx(ConnectionId::new(2)).0,
9060                Frame::build(
9061                    FrameType::Request,
9062                    control_flags(),
9063                    0,
9064                    0,
9065                    9,
9066                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9067                )
9068                .unwrap(),
9069            )
9070            .await
9071            .unwrap();
9072        let ClientControlResponse::ServerDescribe { machine_id, .. } =
9073            serde_json::from_slice(&described[0].body).unwrap()
9074        else {
9075            panic!("server.describe answered with another shape");
9076        };
9077        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9078    }
9079
9080    #[test]
9081    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9082        // With a central sqlite storage policy, each registering module gets its
9083        // own resolved descriptor in HELLO_ACK, keyed by its module id.
9084        let registry = Arc::new(Registry::default());
9085        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
9086            crate::daemon_config::StorageConfig::Sqlite {
9087                data_home: std::path::PathBuf::from("/data"),
9088            },
9089        ));
9090
9091        let responses = handler
9092            .handle_control(
9093                ConnectionId::new(1),
9094                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9095            )
9096            .unwrap();
9097        let ack = parse_ack(&responses[0]);
9098        assert_eq!(
9099            ack.storage,
9100            Some(serde_json::json!({
9101                "module_id": "alfonso-routing",
9102                "storage_namespace": "default",
9103                "isolation": { "kind": "module" },
9104                "backend": {
9105                    "backend": "sqlite",
9106                    "path": "/data/cortexkit/alfonso-routing/store.db"
9107                }
9108            })),
9109            "the delivered descriptor is the module's own sqlite store path"
9110        );
9111    }
9112
9113    #[test]
9114    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9115        let registry = Arc::new(Registry::default());
9116        let handler = ControlHandler::new(Arc::clone(&registry));
9117        let conn = ConnectionId::new(1);
9118        let responses = handler
9119            .handle_control(
9120                conn,
9121                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9122            )
9123            .unwrap();
9124        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9125        let registration = registry.get_module("aft").unwrap().unwrap();
9126        assert_eq!(registration.control_ops, module_baseline_control_ops());
9127
9128        let frame =
9129            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9130        assert!(handler
9131            .guard_module_control_op(&frame, "aft", "route.bind")
9132            .unwrap()
9133            .is_none());
9134        let error = handler
9135            .guard_module_control_op(&frame, "aft", "test.synthetic")
9136            .unwrap()
9137            .expect("synthetic ungranted op should be rejected");
9138        assert_eq!(error.header.ty, FrameType::Error);
9139        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9140    }
9141
9142    #[test]
9143    fn hello_control_ops_some_adds_optional_grants() {
9144        let registry = Arc::new(Registry::default());
9145        let handler = ControlHandler::new(Arc::clone(&registry));
9146        handler
9147            .handle_control(
9148                ConnectionId::new(1),
9149                hello_frame_with_control_ops(
9150                    "aft",
9151                    PROTOCOL_VERSION,
9152                    7,
9153                    Some(vec![
9154                        "future.synthetic".to_string(),
9155                        "route.bind".to_string(),
9156                    ]),
9157                ),
9158            )
9159            .unwrap();
9160        let registration = registry.get_module("aft").unwrap().unwrap();
9161        assert_eq!(
9162            registration.control_ops,
9163            vec![
9164                "route.bind".to_string(),
9165                "route.status".to_string(),
9166                "future.synthetic".to_string(),
9167            ]
9168        );
9169        let frame =
9170            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9171        assert!(handler
9172            .guard_module_control_op(&frame, "aft", "future.synthetic")
9173            .unwrap()
9174            .is_none());
9175    }
9176
9177    #[tokio::test]
9178    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9179        let registry = Arc::new(Registry::default());
9180        let forwarding = Arc::new(ForwardingTable::default());
9181        let handler =
9182            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9183        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9184        hello_via_sink(
9185            &handler,
9186            &module_ctx,
9187            &mut module_rx,
9188            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9189        )
9190        .await;
9191
9192        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9193        let responses = handler
9194            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9195            .await
9196            .unwrap();
9197        assert_eq!(responses.len(), 1);
9198        assert_eq!(responses[0].header.ty, FrameType::Error);
9199        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9200        assert!(module_rx.try_recv().is_err());
9201    }
9202
9203    #[tokio::test]
9204    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9205        let registry = Arc::new(Registry::default());
9206        let forwarding = Arc::new(ForwardingTable::default());
9207        let handler =
9208            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9209        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9210        hello_via_sink(
9211            &handler,
9212            &module_ctx,
9213            &mut module_rx,
9214            hello_frame_with_control_ops(
9215                "aft",
9216                PROTOCOL_VERSION,
9217                7,
9218                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9219            ),
9220        )
9221        .await;
9222
9223        let project_root = unique_project_root("demux");
9224        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9225        let route_handler = handler.clone();
9226        let route_task = tokio::spawn(async move {
9227            route_handler
9228                .handle_control_frame(
9229                    &route_client_ctx,
9230                    route_open_frame(100, "aft", project_root),
9231                )
9232                .await
9233                .unwrap()
9234        });
9235        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9236            .await
9237            .unwrap()
9238            .unwrap();
9239        assert!(matches!(
9240            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9241            ModuleControlRequest::RouteBind { .. }
9242        ));
9243
9244        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9245        let health_handler = handler.clone();
9246        let health_task = tokio::spawn(async move {
9247            health_handler
9248                .handle_control_frame(
9249                    &health_client_ctx,
9250                    supervisor_health_probe_frame(101, "aft"),
9251                )
9252                .await
9253                .unwrap()
9254        });
9255        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9256            .await
9257            .unwrap()
9258            .unwrap();
9259        assert_eq!(
9260            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9261            ModuleControlRequest::HealthCheck {}
9262        );
9263
9264        handler
9265            .handle_control_frame(
9266                &module_ctx,
9267                health_response(health_frame.header.corr, HealthStatus::Degraded),
9268            )
9269            .await
9270            .unwrap();
9271        let health_response = health_task.await.unwrap();
9272        assert_eq!(health_response.len(), 1);
9273        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9274            ClientControlResponse::SupervisorHealthProbe {
9275                module_id,
9276                status,
9277                detail,
9278                metrics,
9279            } => {
9280                assert_eq!(module_id, "aft");
9281                assert_eq!(status, HealthStatus::Degraded);
9282                assert_eq!(detail.as_deref(), Some("warming"));
9283                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9284            }
9285            other => panic!("unexpected health response: {other:?}"),
9286        }
9287
9288        handler
9289            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9290            .await
9291            .unwrap();
9292        let route_response = route_task.await.unwrap();
9293        assert!(route_response.is_empty());
9294        let published = route_client_rx.recv().await.unwrap();
9295        assert!(matches!(
9296            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9297            ClientControlResponse::RouteOpen { .. }
9298        ));
9299    }
9300
9301    /// Start one `route.open` on `client_connection` and return its still-running
9302    /// handler task together with the `route.bind` the module received for it.
9303    /// The handler blocks until the module answers, so it has to run as a task
9304    /// while the test drives the module side.
9305    async fn relay_route_open(
9306        handler: &ControlHandler,
9307        client_connection: ConnectionId,
9308        client_egress: &FrameSink,
9309        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9310        corr: u64,
9311        module_id: &str,
9312        project_root_label: &str,
9313    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9314        let ctx = RouteCtx {
9315            connection_id: client_connection,
9316            egress: client_egress.clone(),
9317        };
9318        let handler = handler.clone();
9319        let project_root = unique_project_root(project_root_label);
9320        let module_id = module_id.to_string();
9321        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9322        let task = tokio::spawn(async move {
9323            let _guard = tracing::dispatcher::set_default(&dispatch);
9324            handler
9325                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9326                .await
9327                .unwrap()
9328        });
9329        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9330            .await
9331            .expect("module receives the relayed route.bind")
9332            .expect("module egress is open");
9333        (task, bind.frame)
9334    }
9335
9336    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9337        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9338            ModuleControlRequest::RouteBind {
9339                route_channel,
9340                epoch,
9341                ..
9342            } => (route_channel, epoch),
9343            other => panic!("expected a route.bind request, got {other:?}"),
9344        }
9345    }
9346
9347    fn published_route(frame: &Frame) -> (u16, u32) {
9348        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9349            ClientControlResponse::RouteOpen {
9350                route_channel,
9351                route_epoch,
9352            } => (route_channel, route_epoch),
9353            other => panic!("expected a route.open response, got {other:?}"),
9354        }
9355    }
9356
9357    /// Reproduction of a production outage. A client had `route.open`s in
9358    /// flight to a module and was already marked closing -- its egress had refused a
9359    /// module frame, so the daemon asked its connection to end -- while its sink
9360    /// was still open. When the module acked those binds, the daemon refused to
9361    /// commit a route for a closing client, and that refusal was returned from
9362    /// the MODULE connection's frame handler, where a router error that has no
9363    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
9364    /// the supervisor correctly did not respawn a clean exit, and every seat lost
9365    /// its tools for hours -- one client's teardown took down a connection
9366    /// carrying ~170 other routes.
9367    ///
9368    /// The window is opened here by calling the production path that opens it
9369    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
9370    /// state that matters is "in `closing_connections`, sink still open, relay
9371    /// still pending", and it lasts only from the close request until the
9372    /// connection loop reacts to it; a socket-level test can flood a client into
9373    /// that escalation but cannot pin the module's ack inside the window. Closing
9374    /// the socket instead takes the other path entirely -- connection teardown
9375    /// removes the pending relay under the same lock, so the ack finds nothing.
9376    #[tokio::test]
9377    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9378        let registry = Arc::new(Registry::default());
9379        let forwarding = Arc::new(ForwardingTable::default());
9380        let handler =
9381            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9382
9383        let module_connection = ConnectionId::new(30);
9384        let (module_ctx, mut module_rx) = route_ctx(module_connection);
9385        hello_via_sink(
9386            &handler,
9387            &module_ctx,
9388            &mut module_rx,
9389            hello_frame("aft", PROTOCOL_VERSION, 7),
9390        )
9391        .await;
9392
9393        let dying_client = ConnectionId::new(31);
9394        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9395
9396        // A published route on the dying client. The escalation below only marks
9397        // a connection closing for a route it has already published.
9398        let (first_task, first_bind) = relay_route_open(
9399            &handler,
9400            dying_client,
9401            &dying_ctx.egress,
9402            &mut module_rx,
9403            100,
9404            "aft",
9405            "closing-first",
9406        )
9407        .await;
9408        handler
9409            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9410            .await
9411            .unwrap();
9412        assert!(first_task.await.unwrap().is_empty());
9413        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9414
9415        // A second route.open from the same client, relayed and awaiting its ack.
9416        let (second_task, second_bind) = relay_route_open(
9417            &handler,
9418            dying_client,
9419            &dying_ctx.egress,
9420            &mut module_rx,
9421            101,
9422            "aft",
9423            "closing-second",
9424        )
9425        .await;
9426        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9427
9428        // The window: the client is closing, its sink is still open, and its
9429        // second bind is still pending.
9430        assert!(forwarding
9431            .escalate_client_delivery_failure(
9432                dying_client,
9433                first_channel,
9434                first_epoch,
9435                CloseReason::new(
9436                    "module_to_client_delivery_failed",
9437                    "client egress refused a module frame",
9438                ),
9439                crate::forwarding::UndeliveredFrame {
9440                    module_id: None,
9441                    sink: &dying_ctx.egress,
9442                },
9443            )
9444            .unwrap());
9445        assert!(!dying_ctx.egress.is_closed());
9446
9447        // The frame that used to end the module connection.
9448        let ack = handler
9449            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9450            .await;
9451        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9452        if module_loop_error.is_some() {
9453            // What the server's connection loop does with a router error that has
9454            // no ERROR-frame translation: end the connection, which releases the
9455            // module's registration and every route on it.
9456            handler.cleanup_connection(module_connection).unwrap();
9457        }
9458        // Read the module's next frame before opening the co-tenant's route, so
9459        // the GOODBYE assertion below is about THIS ack and not about later
9460        // traffic. `None` means the module was told nothing.
9461        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9462            .await
9463            .ok()
9464            .flatten();
9465
9466        // 1. The module connection is still registered.
9467        assert!(
9468            registry
9469                .get_module_by_connection(module_connection)
9470                .unwrap()
9471                .is_some(),
9472            "one client's closing connection ended the shared module connection: \
9473             {module_loop_error:?}"
9474        );
9475        // ...and still serving: another client can open and use a route on it.
9476        let cotenant = ConnectionId::new(32);
9477        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9478        let (cotenant_task, cotenant_bind) = relay_route_open(
9479            &handler,
9480            cotenant,
9481            &cotenant_ctx.egress,
9482            &mut module_rx,
9483            102,
9484            "aft",
9485            "closing-cotenant",
9486        )
9487        .await;
9488        handler
9489            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9490            .await
9491            .unwrap();
9492        assert!(cotenant_task.await.unwrap().is_empty());
9493        let (cotenant_channel, cotenant_epoch) =
9494            published_route(&cotenant_rx.recv().await.unwrap());
9495        assert!(matches!(
9496            forwarding
9497                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9498                .unwrap(),
9499            DataRoute::Client(DataRouteState::Bound(_))
9500        ));
9501
9502        // 2. The module was told to drop the binding it created for the route
9503        //    that will never be published.
9504        let goodbye = post_ack_module_frame
9505            .expect("module receives a GOODBYE for the abandoned route channel");
9506        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9507        assert_eq!(goodbye.header.channel, abandoned_channel);
9508        assert_eq!(goodbye.header.epoch, abandoned_epoch);
9509
9510        // 3. The dying client received nothing: no route was ever published to
9511        //    it. Its route.open is answered as unavailable, which the connection
9512        //    loop would write to a socket that is already going away.
9513        assert!(dying_rx.try_recv().is_err());
9514        let second_response = second_task.await.unwrap();
9515        assert_eq!(second_response.len(), 1);
9516        assert_eq!(
9517            parse_error(&second_response[0])["code"],
9518            "target_unavailable"
9519        );
9520    }
9521
9522    /// One `catalog.list` as a client sees it: the generation and the listed
9523    /// module ids with their readiness.
9524    async fn catalog_snapshot(
9525        handler: &ControlHandler,
9526        client_ctx: &RouteCtx,
9527        corr: u64,
9528    ) -> (u64, Vec<(String, bool)>) {
9529        let body =
9530            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9531        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9532        let reply = handler
9533            .handle_control_frame(client_ctx, request)
9534            .await
9535            .unwrap()
9536            .pop()
9537            .unwrap();
9538        let ClientControlResponse::CatalogList {
9539            generation,
9540            modules,
9541            ..
9542        } = serde_json::from_slice(&reply.body).unwrap()
9543        else {
9544            panic!("not a catalog.list reply");
9545        };
9546        (
9547            generation,
9548            modules
9549                .into_iter()
9550                .map(|entry| (entry.module_id, entry.ready))
9551                .collect(),
9552        )
9553    }
9554
9555    /// A handler whose HELLO for module "aft" on connection 40 is parked
9556    /// between reserving its registry entry and installing its forwarding
9557    /// endpoint. Returns the handler, the two-party barrier (wait once to know
9558    /// the HELLO is parked, once more to release it), the HELLO's thread, and
9559    /// the module's context and receiver.
9560    fn hello_held_before_forwarding() -> (
9561        ControlHandler,
9562        Arc<std::sync::Barrier>,
9563        std::thread::JoinHandle<Vec<Frame>>,
9564        RouteCtx,
9565        mpsc::Receiver<crate::router::OutboundFrame>,
9566    ) {
9567        let hold = Arc::new(std::sync::Barrier::new(2));
9568        let handler = ControlHandler::with_forwarding(
9569            Arc::new(Registry::default()),
9570            Arc::new(ForwardingTable::default()),
9571        )
9572        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9573        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9574        let hello = {
9575            let handler = handler.clone();
9576            let module_ctx = module_ctx.clone();
9577            std::thread::spawn(move || {
9578                handler
9579                    .handle_hello(
9580                        module_ctx.connection_id,
9581                        Some(module_ctx.egress.clone()),
9582                        hello_frame("aft", PROTOCOL_VERSION, 7),
9583                    )
9584                    .unwrap()
9585            })
9586        };
9587        (handler, hold, hello, module_ctx, module_rx)
9588    }
9589
9590    /// While a HELLO has reserved its registry entry but its forwarding
9591    /// endpoint is not installed yet, `route.open` to the module can only be
9592    /// refused. `catalog.list` must not list the module in that window: a
9593    /// client that opens a route as soon as it sees a module listed and ready
9594    /// would be refused for a module the daemon had just advertised. The
9595    /// refusal in the window stays the retryable `target_unavailable`, never
9596    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
9597    /// Once the HELLO completes, the module is listed and a route to it opens.
9598    #[tokio::test]
9599    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
9600        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
9601        let client = ConnectionId::new(41);
9602        let (client_ctx, mut client_rx) = route_ctx(client);
9603
9604        hold.wait();
9605        // Parked: the registry holds "aft", forwarding has no endpoint for it.
9606        assert!(handler.registry.get_module("aft").unwrap().is_some());
9607        assert!(!handler
9608            .forwarding
9609            .has_live_module_connection("aft")
9610            .unwrap());
9611        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
9612        assert!(
9613            listed.is_empty(),
9614            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
9615        );
9616        let refused = handler
9617            .handle_control_frame(
9618                &client_ctx,
9619                route_open_frame(301, "aft", unique_project_root("listed-window")),
9620            )
9621            .await
9622            .unwrap();
9623        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
9624        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
9625        let duplicate = handler
9626            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
9627            .await
9628            .unwrap();
9629        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9630
9631        hold.wait();
9632        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9633        let ack = module_rx.try_recv().unwrap().frame;
9634        assert_eq!(ack.header.ty, FrameType::HelloAck);
9635        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9636        assert_eq!(listed, vec![("aft".to_string(), true)]);
9637
9638        let (open_task, bind) = relay_route_open(
9639            &handler,
9640            client,
9641            &client_ctx.egress,
9642            &mut module_rx,
9643            303,
9644            "aft",
9645            "listed-after",
9646        )
9647        .await;
9648        assert!(handler
9649            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9650            .await
9651            .unwrap()
9652            .is_empty());
9653        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
9654        published_route(&client_rx.try_recv().unwrap().frame);
9655    }
9656
9657    /// A catalog watcher keys on the generation: when it changes, it re-reads
9658    /// the listing. So the generation must advance exactly when the module
9659    /// appears. If it advanced while the module was still hidden, a watcher
9660    /// would read the new generation without the module, and the module would
9661    /// later appear under that same generation with nothing telling the
9662    /// watcher to look again.
9663    #[tokio::test]
9664    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
9665        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
9666        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9667        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9668
9669        hold.wait();
9670        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
9671        assert!(listed.is_empty());
9672        assert_eq!(
9673            during, before,
9674            "the generation moved while the module was not listed"
9675        );
9676
9677        hold.wait();
9678        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9679        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9680        assert_eq!(listed, vec![("aft".to_string(), true)]);
9681        assert_eq!(
9682            after,
9683            before + 1,
9684            "the module appeared without exactly one generation bump"
9685        );
9686    }
9687
9688    /// A HELLO whose forwarding endpoint cannot be installed (here the
9689    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
9690    /// refused and leaves nothing behind: no registry entry, no catalog entry,
9691    /// and the catalog generation where it was.
9692    #[tokio::test]
9693    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
9694        let registry = Arc::new(Registry::default());
9695        let handler = ControlHandler::with_forwarding(
9696            Arc::clone(&registry),
9697            Arc::new(ForwardingTable::default()),
9698        );
9699        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9700        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9701        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9702        drop(module_rx);
9703
9704        let replies = handler
9705            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
9706            .await
9707            .unwrap();
9708        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
9709        assert!(registry.get_module("aft").unwrap().is_none());
9710        assert!(registry
9711            .get_module_by_connection(module_ctx.connection_id)
9712            .unwrap()
9713            .is_none());
9714        assert_eq!(
9715            catalog_snapshot(&handler, &client_ctx, 301).await,
9716            (before, Vec::new())
9717        );
9718    }
9719
9720    /// A handler whose `route.open` parks after queueing the route.bind and
9721    /// before marking it enqueued, with module "aft" registered on connection
9722    /// 40. Returns the two-party barrier that parks and releases it.
9723    async fn route_open_held_after_bind_send() -> (
9724        ControlHandler,
9725        Arc<tokio::sync::Barrier>,
9726        RouteCtx,
9727        mpsc::Receiver<crate::router::OutboundFrame>,
9728    ) {
9729        let hold = Arc::new(tokio::sync::Barrier::new(2));
9730        let handler = ControlHandler::with_forwarding(
9731            Arc::new(Registry::default()),
9732            Arc::new(ForwardingTable::default()),
9733        )
9734        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
9735        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9736        hello_via_sink(
9737            &handler,
9738            &module_ctx,
9739            &mut module_rx,
9740            hello_frame("aft", PROTOCOL_VERSION, 7),
9741        )
9742        .await;
9743        (handler, hold, module_ctx, module_rx)
9744    }
9745
9746    /// A fast module acks the route.bind, and the reader commits the route,
9747    /// before `route.open` has marked the relay enqueued. The daemon used to
9748    /// read the missing relay as an abandoned bind and send the module a
9749    /// GOODBYE for the route it had just committed: the client was told its
9750    /// route was open, the module unbound the channel, and the client's first
9751    /// request was dropped on the module side as unbound.
9752    #[tokio::test]
9753    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
9754        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9755        let client = ConnectionId::new(41);
9756        let (client_ctx, mut client_rx) = route_ctx(client);
9757
9758        let (open_task, bind) = relay_route_open(
9759            &handler,
9760            client,
9761            &client_ctx.egress,
9762            &mut module_rx,
9763            200,
9764            "aft",
9765            "acked-before-mark",
9766        )
9767        .await;
9768        let (module_channel, module_epoch) = route_bind_channel(&bind);
9769        // route.open is now parked between queueing the bind and marking it.
9770        hold.wait().await;
9771
9772        // The module answers inside that window, and the route commits.
9773        assert!(handler
9774            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9775            .await
9776            .unwrap()
9777            .is_empty());
9778        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
9779
9780        hold.wait().await;
9781        assert!(
9782            open_task.await.unwrap().is_empty(),
9783            "route.open answers accepted"
9784        );
9785
9786        // Every frame route.open could have sent is already queued, because it
9787        // has returned; the module must have been sent nothing.
9788        match module_rx.try_recv() {
9789            Err(mpsc::error::TryRecvError::Empty) => {}
9790            Ok(outbound) => panic!(
9791                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
9792                outbound.frame.header.ty,
9793                outbound.frame.header.channel,
9794                outbound.frame.header.epoch
9795            ),
9796            Err(err) => panic!("module egress closed: {err}"),
9797        }
9798
9799        // The client's first request on the route is the next thing the module
9800        // reads, on the binding it just created.
9801        let router = Router::with_control_handler(Arc::new(handler.clone()));
9802        let request = Frame::build(
9803            FrameType::Request,
9804            Flags::new(true, Priority::Interactive, false),
9805            client_channel,
9806            client_epoch,
9807            201,
9808            b"first request".to_vec(),
9809        )
9810        .unwrap();
9811        router
9812            .route_for_connection(&client_ctx, request)
9813            .await
9814            .unwrap();
9815        let delivered = module_rx.try_recv().unwrap().frame;
9816        assert_eq!(delivered.header.ty, FrameType::Request);
9817        assert_eq!(
9818            (delivered.header.channel, delivered.header.epoch),
9819            (module_channel, module_epoch)
9820        );
9821        assert_eq!(delivered.body, b"first request");
9822    }
9823
9824    /// The client goes away while `route.open` is still queueing the bind, so
9825    /// the relay is dropped with no module reply. The teardown cannot send the
9826    /// GOODBYE itself (it could reach the module ahead of the bind), so
9827    /// `route.open` must send it once the bind is queued; otherwise the module
9828    /// keeps a binding the daemon will never commit.
9829    #[tokio::test]
9830    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
9831        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9832        let client = ConnectionId::new(42);
9833        let (client_ctx, _client_rx) = route_ctx(client);
9834
9835        let (open_task, bind) = relay_route_open(
9836            &handler,
9837            client,
9838            &client_ctx.egress,
9839            &mut module_rx,
9840            300,
9841            "aft",
9842            "released-before-mark",
9843        )
9844        .await;
9845        let (module_channel, module_epoch) = route_bind_channel(&bind);
9846        hold.wait().await;
9847
9848        handler.cleanup_connection(client).unwrap();
9849        assert!(
9850            module_rx.try_recv().is_err(),
9851            "client teardown must not GOODBYE a bind route.open is still queueing"
9852        );
9853
9854        hold.wait().await;
9855        let response = open_task.await.unwrap();
9856        assert_eq!(response.len(), 1);
9857        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9858
9859        let goodbye = module_rx
9860            .try_recv()
9861            .expect("module is told to drop the binding the daemon abandoned")
9862            .frame;
9863        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9864        assert_eq!(
9865            (goodbye.header.channel, goodbye.header.epoch),
9866            (module_channel, module_epoch)
9867        );
9868        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9869    }
9870
9871    /// The module acks inside the same window, but the client's egress closed
9872    /// first, so the reader refuses to commit. The ack proves the module holds
9873    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
9874    /// send a second one.
9875    #[tokio::test]
9876    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
9877        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9878        let client = ConnectionId::new(43);
9879        let (client_ctx, client_rx) = route_ctx(client);
9880
9881        let (open_task, bind) = relay_route_open(
9882            &handler,
9883            client,
9884            &client_ctx.egress,
9885            &mut module_rx,
9886            400,
9887            "aft",
9888            "gone-client-before-mark",
9889        )
9890        .await;
9891        let (module_channel, module_epoch) = route_bind_channel(&bind);
9892        hold.wait().await;
9893
9894        drop(client_rx);
9895        assert!(client_ctx.egress.is_closed());
9896        handler
9897            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9898            .await
9899            .unwrap();
9900
9901        hold.wait().await;
9902        let response = open_task.await.unwrap();
9903        assert_eq!(response.len(), 1);
9904        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9905
9906        let goodbye = module_rx
9907            .try_recv()
9908            .expect("module is told to drop the binding it acked")
9909            .frame;
9910        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9911        assert_eq!(
9912            (goodbye.header.channel, goodbye.header.epoch),
9913            (module_channel, module_epoch)
9914        );
9915        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9916        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
9917    }
9918
9919    /// The fence at the module-loop boundary, stated as its own contract: which
9920    /// forwarding failures are allowed to end the module connection that is being
9921    /// served. A `ConnectionClosing` naming some client is about that client, and
9922    /// a module connection is shared; the same error naming the module's own
9923    /// connection is about this connection and must stay fatal, as must failures
9924    /// that are about the forwarding table itself.
9925    #[test]
9926    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9927        let handler = ControlHandler::default();
9928        let module_connection = ConnectionId::new(30);
9929        let client_connection = ConnectionId::new(31);
9930
9931        handler
9932            .refuse_to_end_module_connection_for_a_client(
9933                module_connection,
9934                77,
9935                ForwardingError::ConnectionClosing {
9936                    connection_id: client_connection,
9937                },
9938            )
9939            .expect("a closing client must never end the module connection");
9940
9941        assert!(matches!(
9942            handler.refuse_to_end_module_connection_for_a_client(
9943                module_connection,
9944                78,
9945                ForwardingError::ConnectionClosing {
9946                    connection_id: module_connection,
9947                },
9948            ),
9949            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9950                connection_id
9951            })) if connection_id == module_connection
9952        ));
9953        assert!(matches!(
9954            handler.refuse_to_end_module_connection_for_a_client(
9955                module_connection,
9956                79,
9957                ForwardingError::Poisoned,
9958            ),
9959            Err(RouterError::Forwarding(ForwardingError::Poisoned))
9960        ));
9961        assert!(matches!(
9962            handler.refuse_to_end_module_connection_for_a_client(
9963                module_connection,
9964                80,
9965                ForwardingError::StaleModuleEndpoint,
9966            ),
9967            Err(RouterError::Forwarding(
9968                ForwardingError::StaleModuleEndpoint
9969            ))
9970        ));
9971    }
9972
9973    /// The spawn-attestation guard is what stops a connected module from claiming
9974    /// another module's identity and being stamped `Reserved` for it. Every other
9975    /// test that supplies a consumer_identity supplies a CORRECT one, because a
9976    /// correct one is what the rest of the flow needs -- so the guard's rejection
9977    /// branch was never the subject of an assertion, only its acceptance branch.
9978    ///
9979    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
9980    /// whole subc-core library suite green; only the forwarding integration tests
9981    /// notice, and they notice for unrelated reasons. This test exists so the
9982    /// refusal itself is asserted where the guard lives: it fails if the identity
9983    /// check stops refusing, which is the direction that matters, since a guard
9984    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
9985    #[tokio::test]
9986    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9987        let registry = Arc::new(Registry::default());
9988        let forwarding = Arc::new(ForwardingTable::default());
9989        let supervisor = SupervisorHandle::new();
9990        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9991        let handler =
9992            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9993                .with_supervisor(supervisor);
9994
9995        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9996        hello_via_sink(
9997            &handler,
9998            &target_ctx,
9999            &mut target_rx,
10000            hello_frame("target", PROTOCOL_VERSION, 1),
10001        )
10002        .await;
10003
10004        // A real supervised module id presenting the wrong nonce. This is the
10005        // impersonation case: the attacker knows a privileged module_id, which is
10006        // public, and guesses at the nonce, which is not.
10007        let wrong_nonce = handler
10008            .handle_control_frame(
10009                &route_ctx(ConnectionId::new(91)).0,
10010                route_open_frame_with_admission_facts(
10011                    20,
10012                    "target",
10013                    unique_project_root("admission-facts"),
10014                    Some(subc_control::ConsumerIdentity {
10015                        module_id: "fed".to_string(),
10016                        launch_nonce: "not-the-real-nonce".to_string(),
10017                    }),
10018                    None,
10019                ),
10020            )
10021            .await
10022            .unwrap();
10023        assert_eq!(
10024            parse_error(&wrong_nonce[0])["code"],
10025            "bad_consumer_identity",
10026            "a mismatched launch nonce must be refused, not stamped Reserved"
10027        );
10028
10029        // A module id the supervisor never spawned at all, so no nonce exists to
10030        // compare against. An implementation that treats "no record" as "nothing
10031        // to check" fails open here while passing the case above.
10032        let never_spawned = handler
10033            .handle_control_frame(
10034                &route_ctx(ConnectionId::new(92)).0,
10035                route_open_frame_with_admission_facts(
10036                    21,
10037                    "target",
10038                    unique_project_root("admission-facts"),
10039                    Some(subc_control::ConsumerIdentity {
10040                        module_id: "never-spawned".to_string(),
10041                        launch_nonce: "any-nonce".to_string(),
10042                    }),
10043                    None,
10044                ),
10045            )
10046            .await
10047            .unwrap();
10048        assert_eq!(
10049            parse_error(&never_spawned[0])["code"],
10050            "bad_consumer_identity",
10051            "an unspawned module_id must be refused rather than accepted for lack of a record"
10052        );
10053    }
10054
10055    /// The refusal test above proves the guard says NO. Nothing proved it can say
10056    /// YES, and the difference is not academic: replacing the whole authorization
10057    /// with `false` -- admitting no consumer identity at all, revoking Reserved
10058    /// standing for every supervised module in the fleet -- leaves 110 of the 111
10059    /// library tests GREEN. The one that notices does so by HANGING, because it
10060    /// waits for a bind that can no longer happen.
10061    ///
10062    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
10063    /// or flaky test, invites a RETRY rather than an investigation, and the retry
10064    /// hangs too and gets blamed on the runner. So a total revocation of the
10065    /// daemon's trust grant would have shipped behind a symptom nobody attributes
10066    /// to code.
10067    ///
10068    /// The bias is structural rather than accidental. A REFUSAL looks like a
10069    /// failure someone writes a test for; a GRANT looks like the happy path. Every
10070    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
10071    /// suite for that reason, and this one is the purest case in the daemon.
10072    ///
10073    /// This test asserts the EFFECT rather than the absence of an error: the module
10074    /// receives a RouteBind and it carries `Reserved` naming the attested module.
10075    /// A guard that admitted nobody would produce no bind at all; one that admitted
10076    /// everybody would stamp the wrong principal, which the refusal test catches.
10077    #[tokio::test]
10078    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10079        let registry = Arc::new(Registry::default());
10080        let forwarding = Arc::new(ForwardingTable::default());
10081        let supervisor = SupervisorHandle::new();
10082        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10083        let handler =
10084            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10085                .with_supervisor(supervisor);
10086
10087        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10088        hello_via_sink(
10089            &handler,
10090            &target_ctx,
10091            &mut target_rx,
10092            hello_frame("target", PROTOCOL_VERSION, 1),
10093        )
10094        .await;
10095
10096        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10097        let route_handler = handler.clone();
10098        let route_task = tokio::spawn(async move {
10099            route_handler
10100                .handle_control_frame(
10101                    &client_ctx,
10102                    route_open_frame_with_admission_facts(
10103                        30,
10104                        "target",
10105                        unique_project_root("admission-facts"),
10106                        Some(subc_control::ConsumerIdentity {
10107                            module_id: "fed".to_string(),
10108                            launch_nonce: "fed-nonce".to_string(),
10109                        }),
10110                        None,
10111                    ),
10112                )
10113                .await
10114                .unwrap()
10115        });
10116
10117        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
10118        // the very mutation it exists to catch -- a guard that admits nobody -- no
10119        // bind is ever sent, so it HUNG rather than failing. That reproduces the
10120        // exact defect being fixed: a total revocation detected only as a stalled
10121        // suite, which reads as flakiness and invites a retry. An acceptance test
10122        // that waits for an effect must bound the wait, or a red becomes a hang.
10123        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10124            .await
10125            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10126            .expect("module control channel closed before route.bind");
10127        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10128        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10129            panic!("expected route.bind")
10130        };
10131        assert_eq!(
10132            principal,
10133            Some(Principal::Reserved {
10134                module_id: "fed".to_string()
10135            }),
10136            "a correctly attested consumer must be stamped Reserved for its own id"
10137        );
10138
10139        handler
10140            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10141            .await
10142            .unwrap();
10143        assert!(route_task.await.unwrap().is_empty());
10144        assert!(
10145            matches!(
10146                serde_json::from_slice::<ClientControlResponse>(
10147                    &client_rx.recv().await.unwrap().body
10148                )
10149                .unwrap(),
10150                ClientControlResponse::RouteOpen { .. }
10151            ),
10152            "the route must actually open, not merely avoid an error"
10153        );
10154    }
10155
10156    #[tokio::test(start_paused = true)]
10157    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10158        let registry = Arc::new(Registry::default());
10159        let forwarding = Arc::new(ForwardingTable::default());
10160        let supervisor = SupervisorHandle::new();
10161        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10162        let handler =
10163            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10164                .with_supervisor(supervisor);
10165
10166        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10167        hello_via_sink(
10168            &handler,
10169            &target_ctx,
10170            &mut target_rx,
10171            hello_frame("target", PROTOCOL_VERSION, 1),
10172        )
10173        .await;
10174
10175        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10176        let direct_handler = handler.clone();
10177        let direct_open = tokio::spawn(async move {
10178            direct_handler
10179                .handle_control_frame(
10180                    &direct_ctx,
10181                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
10182                )
10183                .await
10184                .unwrap()
10185        });
10186        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10187            .await
10188            .expect("no direct route.bind within 5s")
10189            .expect("target control channel closed before direct route.bind");
10190        handler
10191            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10192            .await
10193            .unwrap();
10194        assert!(direct_open.await.unwrap().is_empty());
10195        let _ = direct_rx.recv().await.unwrap();
10196
10197        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10198        let reserved_handler = handler.clone();
10199        let reserved_open = tokio::spawn(async move {
10200            reserved_handler
10201                .handle_control_frame(
10202                    &reserved_ctx,
10203                    route_open_frame_with_admission_facts(
10204                        3,
10205                        "target",
10206                        unique_project_root("admission-facts"),
10207                        Some(ConsumerIdentity {
10208                            module_id: "fed".to_string(),
10209                            launch_nonce: "fed-nonce".to_string(),
10210                        }),
10211                        None,
10212                    ),
10213                )
10214                .await
10215                .unwrap()
10216        });
10217        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10218            .await
10219            .expect("no reserved route.bind within 5s")
10220            .expect("target control channel closed before reserved route.bind");
10221        handler
10222            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10223            .await
10224            .unwrap();
10225        assert!(reserved_open.await.unwrap().is_empty());
10226        let _ = reserved_rx.recv().await.unwrap();
10227
10228        forwarding
10229            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10230            .unwrap();
10231        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10232        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10233            module_id: Some("target".to_string()),
10234        })
10235        .unwrap();
10236        let census_frame =
10237            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10238        let response = handler
10239            .handle_control_frame(&census_ctx, census_frame)
10240            .await
10241            .unwrap()
10242            .pop()
10243            .unwrap();
10244        let actual: Value = serde_json::from_slice(&response.body).unwrap();
10245        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10246        assert!(matches!(
10247            decoded,
10248            ClientControlResponse::SupervisorRoutes { .. }
10249        ));
10250        let routes = actual["modules"][0]["routes"].as_array().unwrap();
10251        assert_eq!(routes.len(), 2);
10252        assert!(routes.iter().all(|route| route["draining"] == true));
10253        // The census carries WHY: the reason the drain was begun with, in the
10254        // route.closing vocabulary, on every draining route this drain marked.
10255        assert!(
10256            routes.iter().all(|route| route["drain_reason"] == "reload"),
10257            "draining routes must name the drain's reason: {routes:?}"
10258        );
10259        assert!(routes.iter().any(|route| {
10260            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10261        }));
10262        assert!(routes.iter().any(|route| {
10263            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10264        }));
10265
10266        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10267            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10268        if std::env::var_os("UPDATE_GOLDEN").is_some() {
10269            std::fs::write(
10270                &golden_path,
10271                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10272            )
10273            .unwrap();
10274        }
10275        let expected: Value =
10276            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10277        assert_eq!(actual, expected);
10278    }
10279
10280    async fn query_live_roots(
10281        handler: &ControlHandler,
10282        module_ctx: &RouteCtx,
10283    ) -> ModuleControlResponseToModule {
10284        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10285        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10286        let response = handler
10287            .handle_control_frame(module_ctx, frame)
10288            .await
10289            .unwrap()
10290            .pop()
10291            .unwrap();
10292        serde_json::from_slice(&response.body).unwrap()
10293    }
10294
10295    #[tokio::test(start_paused = true)]
10296    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10297        let registry = Arc::new(Registry::default());
10298        let forwarding = Arc::new(ForwardingTable::default());
10299        let handler = ControlHandler::with_forwarding(registry, forwarding);
10300        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10301        hello_via_sink(
10302            &handler,
10303            &target_ctx,
10304            &mut target_rx,
10305            hello_frame("target", PROTOCOL_VERSION, 1),
10306        )
10307        .await;
10308        let root = unique_project_root("live-roots-known");
10309        let path = ProjectRootId::from_path_allowing_missing(root.path())
10310            .unwrap()
10311            .as_path()
10312            .to_path_buf();
10313        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10314        let open_handler = handler.clone();
10315        let opened = tokio::spawn(async move {
10316            open_handler
10317                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10318                .await
10319                .unwrap()
10320        });
10321        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10322            .await
10323            .unwrap()
10324            .unwrap();
10325        handler
10326            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10327            .await
10328            .unwrap();
10329        assert!(opened.await.unwrap().is_empty());
10330        let _ = client_rx.recv().await.unwrap();
10331
10332        let root = unique_project_root("live-roots-pending");
10333        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10334            .unwrap()
10335            .as_path()
10336            .to_path_buf();
10337        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10338        let open_handler = handler.clone();
10339        let pending = tokio::spawn(async move {
10340            open_handler
10341                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10342                .await
10343                .unwrap()
10344        });
10345        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10346            .await
10347            .unwrap()
10348            .unwrap();
10349        let actual = query_live_roots(&handler, &target_ctx).await;
10350        let ModuleControlResponseToModule::LiveRoots {
10351            roots,
10352            unknown_root_bindings,
10353            total_bindings,
10354        } = actual
10355        else {
10356            panic!("expected live roots")
10357        };
10358        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10359        assert_eq!(unknown_root_bindings, 0);
10360        assert_eq!(
10361            roots.len(),
10362            2,
10363            "root-known arm must retain each canonical root"
10364        );
10365        assert_eq!(
10366            total_bindings,
10367            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10368        );
10369        let counts = roots
10370            .iter()
10371            .map(|root| (root.project_root.clone(), root.bound, root.pending))
10372            .collect::<Vec<_>>();
10373        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10374        expected.sort_by(|a, b| a.0.cmp(&b.0));
10375        assert_eq!(
10376            counts, expected,
10377            "roots must sort by path and count pending separately"
10378        );
10379        handler
10380            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10381            .await
10382            .unwrap();
10383        assert!(pending.await.unwrap().is_empty());
10384    }
10385
10386    #[tokio::test(start_paused = true)]
10387    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10388        let forwarding = Arc::new(ForwardingTable::default());
10389        let handler =
10390            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10391        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10392        hello_via_sink(
10393            &handler,
10394            &target_ctx,
10395            &mut target_rx,
10396            hello_frame("target", PROTOCOL_VERSION, 1),
10397        )
10398        .await;
10399        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10400        let pending = forwarding
10401            .begin_route_bind_relay_for_test(
10402                client_ctx.connection_id,
10403                client_ctx.egress.clone(),
10404                2,
10405                "target",
10406            )
10407            .unwrap();
10408        forwarding
10409            .complete_pending_relay(
10410                target_ctx.connection_id,
10411                pending.corr,
10412                RouteBindRelayOutcome::Accepted,
10413            )
10414            .unwrap();
10415        let actual = query_live_roots(&handler, &target_ctx).await;
10416        let ModuleControlResponseToModule::LiveRoots {
10417            roots,
10418            unknown_root_bindings,
10419            total_bindings,
10420        } = actual
10421        else {
10422            panic!("expected live roots")
10423        };
10424        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10425        assert_eq!(
10426            unknown_root_bindings, 1,
10427            "unknown-root arm must not read as no bindings"
10428        );
10429        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10430        assert_eq!(
10431            total_bindings,
10432            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10433        );
10434    }
10435
10436    /// A module reads HELLO_ACK as its first frame and exits on anything else,
10437    /// so the ack has to be on its outbound queue before the module is
10438    /// routable. The connection loop writes a handler's replies only after the
10439    /// handler returns; this test stops in exactly that gap, runs a real
10440    /// route.open from another connection, and only then writes whatever the
10441    /// HELLO handler returned, the way the loop would. If the ack were still a
10442    /// reply, the route.bind request would reach the module first.
10443    #[tokio::test(start_paused = true)]
10444    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10445        let forwarding = Arc::new(ForwardingTable::default());
10446        let handler =
10447            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10448        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10449        let replies = handler
10450            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10451            .await
10452            .unwrap();
10453        let queued_by_hello = module_rx.len();
10454
10455        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10456        let open_handler = handler.clone();
10457        let open = tokio::spawn(async move {
10458            open_handler
10459                .handle_control_frame(
10460                    &client_ctx,
10461                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10462                )
10463                .await
10464                .unwrap()
10465        });
10466        // Let the route.open run until its route.bind is on the module's queue.
10467        let mut spins = 0;
10468        while module_rx.len() == queued_by_hello {
10469            spins += 1;
10470            assert!(spins < 10_000, "route.open never queued a route.bind");
10471            tokio::task::yield_now().await;
10472        }
10473
10474        // Now the connection loop's half: write the HELLO handler's replies.
10475        for reply in replies {
10476            module_ctx.egress.send(reply).await.unwrap();
10477        }
10478
10479        let first = module_rx.recv().await.unwrap().frame;
10480        assert_eq!(
10481            first.header.ty,
10482            FrameType::HelloAck,
10483            "the first frame a registering module reads must be its HELLO_ACK"
10484        );
10485        assert_eq!(first.header.corr, 7);
10486        let second = module_rx.recv().await.unwrap().frame;
10487        assert_eq!(second.header.ty, FrameType::Request);
10488        assert!(
10489            matches!(
10490                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10491                ModuleControlRequest::RouteBind { .. }
10492            ),
10493            "the route.bind follows the ack"
10494        );
10495        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10496
10497        handler
10498            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10499            .await
10500            .unwrap();
10501        assert!(open.await.unwrap().is_empty());
10502        let _ = client_rx.recv().await.unwrap();
10503    }
10504
10505    #[tokio::test(start_paused = true)]
10506    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10507        let handler = ControlHandler::with_forwarding(
10508            Arc::new(Registry::default()),
10509            Arc::new(ForwardingTable::default()),
10510        );
10511        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10512        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10513        hello_via_sink(
10514            &handler,
10515            &first_ctx,
10516            &mut first_rx,
10517            hello_frame("first", PROTOCOL_VERSION, 1),
10518        )
10519        .await;
10520        hello_via_sink(
10521            &handler,
10522            &second_ctx,
10523            &mut second_rx,
10524            hello_frame("second", PROTOCOL_VERSION, 2),
10525        )
10526        .await;
10527        let root = unique_project_root("second-only");
10528        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10529        let cloned = handler.clone();
10530        let open = tokio::spawn(async move {
10531            cloned
10532                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10533                .await
10534                .unwrap()
10535        });
10536        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10537            .await
10538            .unwrap()
10539            .unwrap();
10540        let first = query_live_roots(&handler, &first_ctx).await;
10541        let second = query_live_roots(&handler, &second_ctx).await;
10542        assert!(
10543            matches!(
10544                first,
10545                ModuleControlResponseToModule::LiveRoots {
10546                    total_bindings: 0,
10547                    ..
10548                }
10549            ),
10550            "cross-module scope must not expose another module's roots"
10551        );
10552        assert!(
10553            matches!(
10554                second,
10555                ModuleControlResponseToModule::LiveRoots {
10556                    total_bindings: 1,
10557                    ..
10558                }
10559            ),
10560            "second module must see its pending route"
10561        );
10562        handler
10563            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10564            .await
10565            .unwrap();
10566        assert!(open.await.unwrap().is_empty());
10567    }
10568
10569    #[tokio::test(start_paused = true)]
10570    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10571        let handler = ControlHandler::with_forwarding(
10572            Arc::new(Registry::default()),
10573            Arc::new(ForwardingTable::default()),
10574        );
10575        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10576        hello_via_sink(
10577            &handler,
10578            &target_ctx,
10579            &mut target_rx,
10580            hello_frame("target", PROTOCOL_VERSION, 1),
10581        )
10582        .await;
10583        let actual = query_live_roots(&handler, &target_ctx).await;
10584        let ModuleControlResponseToModule::LiveRoots {
10585            roots,
10586            unknown_root_bindings,
10587            total_bindings,
10588        } = actual
10589        else {
10590            panic!("expected live roots")
10591        };
10592        assert!(roots.is_empty());
10593        assert_eq!(unknown_root_bindings, 0);
10594        assert_eq!(total_bindings, 0);
10595        assert_eq!(
10596            total_bindings,
10597            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10598        );
10599    }
10600
10601    /// Read the vendored fed corpus rather than hand-building a package.
10602    ///
10603    /// A hand-built object encodes what the test author believed the carrier
10604    /// emits. These vectors are what it actually emits, and one of them exists
10605    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
10606    /// ignores additive unknown fields at the traversal emit terminus."
10607    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
10608        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10609            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
10610        let text = std::fs::read_to_string(&path)
10611            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
10612        let vectors: Vec<(String, Value)> = text
10613            .lines()
10614            .filter(|line| !line.trim().is_empty())
10615            .map(|line| {
10616                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
10617                let id = entry["corpus_id"]
10618                    .as_str()
10619                    .expect("every vector carries a corpus_id")
10620                    .to_string();
10621                (id, entry["package"].clone())
10622            })
10623            .collect();
10624        // Pin the count: a corpus that silently shrinks would take its coverage
10625        // with it, and a suite reading N-1 vectors reports the same clean pass
10626        // as one reading N.
10627        assert_eq!(
10628            vectors.len(),
10629            3,
10630            "vendored fed corpus changed size; re-sync from subc-federation"
10631        );
10632
10633        // Pin what makes the corpus DISCRIMINATING, not just present.
10634        //
10635        // The relay test below takes its expected value from the corpus, so the
10636        // corpus supplies the test's power to detect a lossy relay rather than
10637        // its correctness. A relay that dropped unrecognised fields would still
10638        // be caught -- but only by a package carrying fields it does not know.
10639        // Shrink every package to the handful of keys any implementation would
10640        // recognise and the test keeps passing over an input that can no longer
10641        // fail, which is the same clean green as a corpus that shrank away.
10642        //
10643        // So assert the precondition rather than duplicating the packages here:
10644        // at least one vector must carry a field beyond the small common set.
10645        // That is one claim to maintain instead of nine, and it fails loudly if
10646        // a re-sync ever flattens the corpus.
10647        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10648        let richest = vectors
10649            .iter()
10650            .filter_map(|(_, package)| package.as_object())
10651            .map(|object| {
10652                object
10653                    .keys()
10654                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10655                    .count()
10656            })
10657            .max()
10658            .unwrap_or(0);
10659        assert!(
10660            richest >= 2,
10661            "vendored corpus no longer carries a package with unmodelled fields, \
10662             so the relay test can no longer distinguish a verbatim relay from a lossy one"
10663        );
10664
10665        vectors
10666    }
10667
10668    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
10669    ///
10670    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
10671    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
10672    /// three-key object, so a relay that quietly dropped fields it did not
10673    /// recognise would satisfy it. These vectors carry nine keys including ones
10674    /// this crate has no type for, so a typed relay fails here and only here.
10675    #[tokio::test]
10676    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10677        for (corpus_id, package) in fed_admission_facts_vectors() {
10678            let registry = Arc::new(Registry::default());
10679            let forwarding = Arc::new(ForwardingTable::default());
10680            let supervisor = SupervisorHandle::new();
10681            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10682            let handler =
10683                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10684                    .with_supervisor(supervisor)
10685                    .with_admission_facts_config(
10686                        Some("fed".to_string()),
10687                        Some(vec!["target".to_string()]),
10688                    );
10689
10690            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10691            hello_via_sink(
10692                &handler,
10693                &target_ctx,
10694                &mut target_rx,
10695                hello_frame("target", PROTOCOL_VERSION, 1),
10696            )
10697            .await;
10698
10699            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
10700            let route_handler = handler.clone();
10701            let expected = package.clone();
10702            let route_task = tokio::spawn(async move {
10703                route_handler
10704                    .handle_control_frame(
10705                        &client_ctx,
10706                        route_open_frame_with_admission_facts(
10707                            20,
10708                            "target",
10709                            unique_project_root("admission-facts"),
10710                            Some(subc_control::ConsumerIdentity {
10711                                module_id: "fed".to_string(),
10712                                launch_nonce: "fed-nonce".to_string(),
10713                            }),
10714                            Some(package),
10715                        ),
10716                    )
10717                    .await
10718                    .unwrap()
10719            });
10720
10721            let bind_frame = target_rx.recv().await.unwrap();
10722            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10723            let ModuleControlRequest::RouteBind {
10724                admission_facts, ..
10725            } = bind
10726            else {
10727                panic!("{corpus_id}: expected route.bind")
10728            };
10729            assert_eq!(
10730                admission_facts,
10731                Some(expected),
10732                "{corpus_id}: relay must not add, drop or reshape any field"
10733            );
10734
10735            handler
10736                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10737                .await
10738                .unwrap();
10739            route_task.await.unwrap();
10740        }
10741    }
10742
10743    #[tokio::test]
10744    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
10745        let registry = Arc::new(Registry::default());
10746        let forwarding = Arc::new(ForwardingTable::default());
10747        let supervisor = SupervisorHandle::new();
10748        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10749        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
10750        let handler =
10751            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10752                .with_supervisor(supervisor)
10753                .with_admission_facts_config(
10754                    Some("fed".to_string()),
10755                    Some(vec!["target".to_string()]),
10756                );
10757
10758        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
10759        hello_via_sink(
10760            &handler,
10761            &target_ctx,
10762            &mut target_rx,
10763            hello_frame("target", PROTOCOL_VERSION, 1),
10764        )
10765        .await;
10766        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
10767        hello_via_sink(
10768            &handler,
10769            &other_ctx,
10770            &mut other_rx,
10771            hello_frame("other", PROTOCOL_VERSION, 2),
10772        )
10773        .await;
10774
10775        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
10776        let expected_facts = facts.clone();
10777        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
10778        let route_handler = handler.clone();
10779        let route_task = tokio::spawn(async move {
10780            route_handler
10781                .handle_control_frame(
10782                    &client_ctx,
10783                    route_open_frame_with_admission_facts(
10784                        10,
10785                        "target",
10786                        unique_project_root("admission-facts"),
10787                        Some(subc_control::ConsumerIdentity {
10788                            module_id: "fed".to_string(),
10789                            launch_nonce: "fed-nonce".to_string(),
10790                        }),
10791                        Some(facts.clone()),
10792                    ),
10793                )
10794                .await
10795                .unwrap()
10796        });
10797        let bind_frame = target_rx.recv().await.unwrap();
10798        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10799        let ModuleControlRequest::RouteBind {
10800            admission_facts, ..
10801        } = bind
10802        else {
10803            panic!("expected route.bind")
10804        };
10805        assert_eq!(admission_facts, Some(expected_facts));
10806        handler
10807            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10808            .await
10809            .unwrap();
10810        assert!(route_task.await.unwrap().is_empty());
10811        assert!(matches!(
10812            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
10813                .unwrap(),
10814            ClientControlResponse::RouteOpen { .. }
10815        ));
10816
10817        let direct = handler
10818            .handle_control_frame(
10819                &route_ctx(ConnectionId::new(73)).0,
10820                route_open_frame_with_admission_facts(
10821                    11,
10822                    "target",
10823                    unique_project_root("admission-facts"),
10824                    None,
10825                    Some(json!({"x": 1})),
10826                ),
10827            )
10828            .await
10829            .unwrap();
10830        assert_eq!(
10831            parse_error(&direct[0])["code"],
10832            "admission_facts_not_permitted"
10833        );
10834
10835        let different_reserved = handler
10836            .handle_control_frame(
10837                &route_ctx(ConnectionId::new(77)).0,
10838                route_open_frame_with_admission_facts(
10839                    15,
10840                    "target",
10841                    unique_project_root("admission-facts"),
10842                    Some(subc_control::ConsumerIdentity {
10843                        module_id: "other".to_string(),
10844                        launch_nonce: "other-nonce".to_string(),
10845                    }),
10846                    Some(json!({"x": 1})),
10847                ),
10848            )
10849            .await
10850            .unwrap();
10851        assert_eq!(
10852            parse_error(&different_reserved[0])["code"],
10853            "admission_facts_not_permitted"
10854        );
10855
10856        let other_target = handler
10857            .handle_control_frame(
10858                &route_ctx(ConnectionId::new(74)).0,
10859                route_open_frame_with_admission_facts(
10860                    12,
10861                    "other",
10862                    unique_project_root("admission-facts"),
10863                    Some(subc_control::ConsumerIdentity {
10864                        module_id: "fed".to_string(),
10865                        launch_nonce: "fed-nonce".to_string(),
10866                    }),
10867                    Some(json!({"x": 1})),
10868                ),
10869            )
10870            .await
10871            .unwrap();
10872        assert_eq!(
10873            parse_error(&other_target[0])["code"],
10874            "admission_facts_target_not_allowed"
10875        );
10876
10877        let nonexistent = handler
10878            .handle_control_frame(
10879                &route_ctx(ConnectionId::new(75)).0,
10880                route_open_frame_with_admission_facts(
10881                    13,
10882                    "missing",
10883                    unique_project_root("admission-facts"),
10884                    None,
10885                    Some(json!({"x": 1})),
10886                ),
10887            )
10888            .await
10889            .unwrap();
10890        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
10891
10892        let described = handler
10893            .handle_control_frame(
10894                &route_ctx(ConnectionId::new(76)).0,
10895                Frame::build(
10896                    FrameType::Request,
10897                    control_flags(),
10898                    0,
10899                    0,
10900                    14,
10901                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10902                )
10903                .unwrap(),
10904            )
10905            .await
10906            .unwrap();
10907        let ClientControlResponse::ServerDescribe { capabilities, .. } =
10908            serde_json::from_slice(&described[0].body).unwrap()
10909        else {
10910            panic!("expected server.describe response")
10911        };
10912        assert!(capabilities
10913            .iter()
10914            .any(|cap| cap == "admission_facts_relay_v1"));
10915    }
10916
10917    #[tokio::test]
10918    async fn admission_facts_without_configured_carrier_are_rejected() {
10919        let registry = Arc::new(Registry::default());
10920        let forwarding = Arc::new(ForwardingTable::default());
10921        let handler = ControlHandler::with_forwarding(registry, forwarding);
10922        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10923        hello_via_sink(
10924            &handler,
10925            &target_ctx,
10926            &mut target_rx,
10927            hello_frame("target", PROTOCOL_VERSION, 1),
10928        )
10929        .await;
10930
10931        let responses = handler
10932            .handle_control_frame(
10933                &route_ctx(ConnectionId::new(79)).0,
10934                route_open_frame_with_admission_facts(
10935                    16,
10936                    "target",
10937                    unique_project_root("admission-facts"),
10938                    None,
10939                    Some(json!({"x": 1})),
10940                ),
10941            )
10942            .await
10943            .unwrap();
10944        assert_eq!(
10945            parse_error(&responses[0])["code"],
10946            "admission_facts_not_permitted"
10947        );
10948    }
10949
10950    #[tokio::test]
10951    async fn route_open_relays_consumer_capabilities_verbatim() {
10952        let registry = Arc::new(Registry::default());
10953        let forwarding = Arc::new(ForwardingTable::default());
10954        let handler =
10955            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10956        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10957        hello_via_sink(
10958            &handler,
10959            &module_ctx,
10960            &mut module_rx,
10961            hello_frame("aft", PROTOCOL_VERSION, 7),
10962        )
10963        .await;
10964
10965        let expected = vec!["elicitation".to_string(), "roots".to_string()];
10966        let expected_for_request = expected.clone();
10967        let project_root = unique_project_root("consumer-capabilities-present");
10968        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10969        let route_handler = handler.clone();
10970        let route_task = tokio::spawn(async move {
10971            route_handler
10972                .handle_control_frame(
10973                    &client_ctx,
10974                    route_open_frame_with_consumer_capabilities(
10975                        401,
10976                        "aft",
10977                        project_root,
10978                        Some(expected_for_request),
10979                    ),
10980                )
10981                .await
10982                .unwrap()
10983        });
10984        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10985            .await
10986            .unwrap()
10987            .unwrap();
10988        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10989        let ModuleControlRequest::RouteBind {
10990            consumer_capabilities,
10991            ..
10992        } = bind
10993        else {
10994            panic!("expected route.bind request, got {bind:?}");
10995        };
10996        assert_eq!(consumer_capabilities, Some(expected.clone()));
10997
10998        handler
10999            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11000            .await
11001            .unwrap();
11002        let route_response = route_task.await.unwrap();
11003        assert!(route_response.is_empty());
11004        let published = client_rx.recv().await.unwrap();
11005        assert!(matches!(
11006            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11007            ClientControlResponse::RouteOpen { .. }
11008        ));
11009    }
11010
11011    #[tokio::test]
11012    async fn route_open_without_consumer_capabilities_relays_none() {
11013        let registry = Arc::new(Registry::default());
11014        let forwarding = Arc::new(ForwardingTable::default());
11015        let handler =
11016            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11017        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11018        hello_via_sink(
11019            &handler,
11020            &module_ctx,
11021            &mut module_rx,
11022            hello_frame("aft", PROTOCOL_VERSION, 7),
11023        )
11024        .await;
11025
11026        let project_root = unique_project_root("consumer-capabilities-absent");
11027        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11028        let route_handler = handler.clone();
11029        let route_task = tokio::spawn(async move {
11030            route_handler
11031                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11032                .await
11033                .unwrap()
11034        });
11035        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11036            .await
11037            .unwrap()
11038            .unwrap();
11039        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11040        let ModuleControlRequest::RouteBind {
11041            consumer_capabilities,
11042            ..
11043        } = bind
11044        else {
11045            panic!("expected route.bind request, got {bind:?}");
11046        };
11047        assert_eq!(consumer_capabilities, None);
11048
11049        handler
11050            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11051            .await
11052            .unwrap();
11053        let route_response = route_task.await.unwrap();
11054        assert!(route_response.is_empty());
11055        let published = client_rx.recv().await.unwrap();
11056        assert!(matches!(
11057            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11058            ClientControlResponse::RouteOpen { .. }
11059        ));
11060    }
11061
11062    /// Opens a route to a freshly registered `aft` with `sent` as the
11063    /// route.open's role_versions, acks the bind, and returns the role_versions
11064    /// the module's bind carried.
11065    async fn bind_role_versions_for(
11066        sent: Option<BTreeMap<String, String>>,
11067        connection: u64,
11068    ) -> Option<BTreeMap<String, String>> {
11069        let registry = Arc::new(Registry::default());
11070        let forwarding = Arc::new(ForwardingTable::default());
11071        let handler =
11072            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11073        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11074        hello_via_sink(
11075            &handler,
11076            &module_ctx,
11077            &mut module_rx,
11078            hello_frame("aft", PROTOCOL_VERSION, 7),
11079        )
11080        .await;
11081        let project_root = unique_project_root("role-versions");
11082        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11083        let route_handler = handler.clone();
11084        let route_task = tokio::spawn(async move {
11085            route_handler
11086                .handle_control_frame(
11087                    &client_ctx,
11088                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
11089                )
11090                .await
11091                .unwrap()
11092        });
11093        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11094            .await
11095            .expect("a well-formed route.open reaches the module as a bind")
11096            .unwrap();
11097        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11098        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11099            panic!("expected route.bind request, got {bind:?}");
11100        };
11101        handler
11102            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11103            .await
11104            .unwrap();
11105        assert!(route_task.await.unwrap().is_empty());
11106        let published = client_rx.recv().await.unwrap();
11107        assert!(matches!(
11108            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11109            ClientControlResponse::RouteOpen { .. }
11110        ));
11111        role_versions
11112    }
11113
11114    #[tokio::test]
11115    async fn route_open_relays_role_versions_verbatim() {
11116        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11117        assert_eq!(
11118            bind_role_versions_for(Some(sent.clone()), 141).await,
11119            Some(sent)
11120        );
11121    }
11122
11123    /// An empty map declares nothing, so the provider sees no field rather
11124    /// than an empty object it would have to treat as a second "none".
11125    #[tokio::test]
11126    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11127        assert_eq!(bind_role_versions_for(None, 143).await, None);
11128        assert_eq!(
11129            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11130            None
11131        );
11132    }
11133
11134    #[tokio::test]
11135    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11136        let registry = Arc::new(Registry::default());
11137        let forwarding = Arc::new(ForwardingTable::default());
11138        let handler =
11139            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11140        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11141        hello_via_sink(
11142            &handler,
11143            &module_ctx,
11144            &mut module_rx,
11145            hello_frame("aft", PROTOCOL_VERSION, 7),
11146        )
11147        .await;
11148
11149        let nine: BTreeMap<String, String> = (0..9)
11150            .map(|index| (format!("role-{index}"), "v1".to_string()))
11151            .collect();
11152        for (label, malformed) in [
11153            (
11154                "invalid role name",
11155                role_versions(&[("Tool_Provider", "v1")]),
11156            ),
11157            ("invalid version", role_versions(&[("tool-provider", "v0")])),
11158            ("nine entries", nine),
11159        ] {
11160            // A refused open answers at once; one that reached the module
11161            // would wait for its bind ack and trip this timeout.
11162            let responses = tokio::time::timeout(
11163                Duration::from_secs(1),
11164                handler.handle_control_frame(
11165                    &route_ctx(ConnectionId::new(148)).0,
11166                    route_open_frame_with_role_versions(
11167                        404,
11168                        "aft",
11169                        unique_project_root("role-versions-malformed"),
11170                        Some(malformed),
11171                    ),
11172                ),
11173            )
11174            .await
11175            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11176            .unwrap();
11177            assert_eq!(responses.len(), 1, "{label}");
11178            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11179            let error = parse_error(&responses[0]);
11180            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11181            assert_eq!(
11182                error["detail"]["field"], "role_versions",
11183                "{label}: {error}"
11184            );
11185            assert!(
11186                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11187                "{label}: a malformed declaration is terminal"
11188            );
11189            assert!(
11190                module_rx.try_recv().is_err(),
11191                "{label}: the module must never see a bind"
11192            );
11193        }
11194    }
11195
11196    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
11197    /// consumer can tell this daemon forwards the field from one that would
11198    /// drop it.
11199    #[tokio::test]
11200    async fn route_role_versions_capability_is_advertised() {
11201        let handler = ControlHandler::new(Arc::new(Registry::default()));
11202        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11203        let ack = hello_via_sink(
11204            &handler,
11205            &ctx,
11206            &mut rx,
11207            hello_frame("m", PROTOCOL_VERSION, 1),
11208        )
11209        .await;
11210        let ack = parse_ack(&ack);
11211        assert!(
11212            ack.subc_capabilities
11213                .iter()
11214                .any(|c| c == "route-role-versions/v1"),
11215            "{:?}",
11216            ack.subc_capabilities
11217        );
11218
11219        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11220        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11221        let reply = handler
11222            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11223            .await
11224            .unwrap()
11225            .pop()
11226            .unwrap();
11227        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11228            serde_json::from_slice(&reply.body).unwrap()
11229        else {
11230            panic!("not a server.describe reply");
11231        };
11232        assert!(
11233            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11234            "{capabilities:?}"
11235        );
11236    }
11237
11238    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
11239    /// catalog entry without `events` (the module declares none) from a daemon
11240    /// that never reports declared events.
11241    #[tokio::test]
11242    async fn catalog_events_capability_is_advertised() {
11243        let handler = ControlHandler::new(Arc::new(Registry::default()));
11244        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11245        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11246        let reply = handler
11247            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11248            .await
11249            .unwrap()
11250            .pop()
11251            .unwrap();
11252        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11253            serde_json::from_slice(&reply.body).unwrap()
11254        else {
11255            panic!("not a server.describe reply");
11256        };
11257        assert!(
11258            capabilities
11259                .iter()
11260                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11261            "{capabilities:?}"
11262        );
11263    }
11264
11265    #[tokio::test]
11266    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
11267        let handler = ControlHandler::new(Arc::new(Registry::default()));
11268        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
11269        let ack = hello_via_sink(
11270            &handler,
11271            &module,
11272            &mut module_rx,
11273            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
11274        )
11275        .await;
11276        let ack = parse_ack(&ack);
11277        assert!(
11278            ack.subc_capabilities
11279                .iter()
11280                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11281            "{:?}",
11282            ack.subc_capabilities
11283        );
11284        assert!(ack
11285            .subc_capabilities
11286            .iter()
11287            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11288
11289        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
11290        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11291        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11292        let reply = handler
11293            .handle_control_frame(&client, frame)
11294            .await
11295            .unwrap()
11296            .pop()
11297            .unwrap();
11298        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11299            serde_json::from_slice(&reply.body).unwrap()
11300        else {
11301            panic!("not a server.describe reply");
11302        };
11303        assert!(
11304            capabilities
11305                .iter()
11306                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11307            "{capabilities:?}"
11308        );
11309        assert!(capabilities
11310            .iter()
11311            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11312    }
11313
11314    #[tokio::test]
11315    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
11316        let handler = ControlHandler::new(Arc::new(Registry::default()));
11317        let (module, mut rx) = route_ctx(ConnectionId::new(154));
11318        let ack = hello_via_sink(
11319            &handler,
11320            &module,
11321            &mut rx,
11322            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
11323        )
11324        .await;
11325        assert!(parse_ack(&ack)
11326            .subc_capabilities
11327            .iter()
11328            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11329        let (client, _rx) = route_ctx(ConnectionId::new(155));
11330        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11331        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11332        let reply = handler
11333            .handle_control_frame(&client, frame)
11334            .await
11335            .unwrap()
11336            .pop()
11337            .unwrap();
11338        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11339            serde_json::from_slice(&reply.body).unwrap()
11340        else {
11341            panic!("not a server.describe reply")
11342        };
11343        assert!(capabilities
11344            .iter()
11345            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11346    }
11347
11348    #[tokio::test]
11349    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11350        let registry = Arc::new(Registry::default());
11351        let forwarding = Arc::new(ForwardingTable::default());
11352        let handler =
11353            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11354                .with_health_probe_timeout(Duration::from_secs(5));
11355        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11356        hello_via_sink(
11357            &handler,
11358            &module_ctx,
11359            &mut module_rx,
11360            non_routable_hello_frame_with_control_ops(
11361                "mcp",
11362                300,
11363                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11364            ),
11365        )
11366        .await;
11367        assert!(registry
11368            .get_module("mcp")
11369            .unwrap()
11370            .unwrap()
11371            .manifest
11372            .provides
11373            .is_empty());
11374
11375        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11376        let route_response = handler
11377            .handle_control_frame(
11378                &route_client_ctx,
11379                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11380            )
11381            .await
11382            .unwrap();
11383        assert_eq!(route_response[0].header.ty, FrameType::Error);
11384        assert_eq!(
11385            parse_error(&route_response[0])["code"],
11386            "target_unavailable"
11387        );
11388        assert!(parse_error(&route_response[0])["message"]
11389            .as_str()
11390            .unwrap()
11391            .contains("does not provide the requested target"));
11392        assert!(module_rx.try_recv().is_err());
11393
11394        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11395        let health_handler = handler.clone();
11396        let health_task = tokio::spawn(async move {
11397            health_handler
11398                .handle_control_frame(
11399                    &health_client_ctx,
11400                    supervisor_health_probe_frame(302, "mcp"),
11401                )
11402                .await
11403                .unwrap()
11404        });
11405        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11406            .await
11407            .unwrap()
11408            .unwrap();
11409        assert_eq!(
11410            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11411            ModuleControlRequest::HealthCheck {}
11412        );
11413        handler
11414            .handle_control_frame(
11415                &module_ctx,
11416                health_response(health_frame.header.corr, HealthStatus::Ok),
11417            )
11418            .await
11419            .unwrap();
11420        let health_response = health_task.await.unwrap();
11421        assert_eq!(health_response[0].header.ty, FrameType::Response);
11422        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11423            ClientControlResponse::SupervisorHealthProbe {
11424                module_id, status, ..
11425            } => {
11426                assert_eq!(module_id, "mcp");
11427                assert_eq!(status, HealthStatus::Ok);
11428            }
11429            other => panic!("unexpected health response: {other:?}"),
11430        }
11431
11432        // Exercise the forwarding cleanup path directly while leaving the registry
11433        // advertisement in place. If cleanup leaves a stale control sink behind,
11434        // the next probe will enqueue onto it and wait for the long probe timeout
11435        // instead of returning an immediate no-connection error.
11436        forwarding
11437            .cleanup_connection(module_ctx.connection_id)
11438            .unwrap();
11439        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11440        let cleanup_response = tokio::time::timeout(
11441            Duration::from_millis(200),
11442            handler.handle_control_frame(
11443                &cleanup_probe_ctx,
11444                supervisor_health_probe_frame(303, "mcp"),
11445            ),
11446        )
11447        .await
11448        .expect("probe should fail immediately when the control lane is gone")
11449        .unwrap();
11450        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11451        assert_eq!(
11452            parse_error(&cleanup_response[0])["code"],
11453            "target_unavailable"
11454        );
11455        assert!(parse_error(&cleanup_response[0])["message"]
11456            .as_str()
11457            .unwrap()
11458            .contains("no module connection"));
11459
11460        handler
11461            .cleanup_connection(module_ctx.connection_id)
11462            .unwrap();
11463    }
11464
11465    #[tokio::test]
11466    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11467        let registry = Arc::new(Registry::default());
11468        let supervisor_handle = SupervisorHandle::new();
11469        let supervisor =
11470            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
11471                .with_handle(supervisor_handle.clone())
11472                .with_connection_file_path(
11473                    std::env::temp_dir()
11474                        .join(format!("subc-route-open-warming-{}", std::process::id())),
11475                );
11476        let module = supervisor
11477            .supervise_configured(
11478                ModuleSpec {
11479                    module_id: "warming".to_string(),
11480                    program: fake_aft_stub_path(),
11481                    args: Vec::new(),
11482                    env: Vec::new(),
11483                    reserved: false,
11484                    reserved_prefixes: Vec::new(),
11485                    protocol: ModuleProtocol::Subc,
11486                    overlap: Default::default(),
11487                },
11488                true,
11489            )
11490            .unwrap();
11491        assert_eq!(module.state().unwrap(), ModuleState::Running);
11492
11493        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11494        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11495        let response = handler
11496            .handle_control_frame(
11497                &ctx,
11498                route_open_frame(304, "warming", unique_project_root("warming")),
11499            )
11500            .await
11501            .unwrap();
11502        module.stop().await.unwrap();
11503
11504        assert_eq!(response[0].header.ty, FrameType::Error);
11505        let error = parse_error(&response[0]);
11506        assert_eq!(error["code"], "module_warming");
11507        assert!(error["message"]
11508            .as_str()
11509            .unwrap()
11510            .contains("state=running, enabled=true, live=false"));
11511    }
11512
11513    #[test]
11514    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11515        let handler = ControlHandler::new(Arc::new(Registry::default()));
11516        let capture = EventCapture::default();
11517        let _subscriber =
11518            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11519        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11520        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11521        let pending = (0..limit).collect::<Vec<_>>();
11522        let response = handler
11523            .route_open_capacity_refusal(
11524                &ctx,
11525                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11526                "busy",
11527                pending.len(),
11528                limit,
11529            )
11530            .unwrap();
11531        assert_eq!(parse_error(&response)["code"], "target_unavailable");
11532        let event = capture
11533            .events()
11534            .into_iter()
11535            .find(|event| {
11536                event.target == "control"
11537                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11538            })
11539            .expect("connection admission refusal event");
11540        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11541        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11542    }
11543
11544    #[test]
11545    fn route_open_target_cap_logs_admission_reason_and_capacity() {
11546        let handler = ControlHandler::new(Arc::new(Registry::default()));
11547        let capture = EventCapture::default();
11548        let _subscriber =
11549            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11550        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
11551        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
11552        let guards = (0..limit)
11553            .map(|_| {
11554                handler
11555                    .route_bind_concurrency
11556                    .try_admit("busy", limit)
11557                    .unwrap()
11558            })
11559            .collect::<Vec<_>>();
11560        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
11561            Err(in_flight) => in_flight,
11562            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
11563        };
11564        let response = handler
11565            .route_open_target_capacity_refusal(
11566                &ctx,
11567                &route_open_frame(397, "busy", unique_project_root("target-cap")),
11568                "busy",
11569                in_flight,
11570            )
11571            .unwrap();
11572        assert_eq!(parse_error(&response)["code"], "target_unavailable");
11573        let event = capture
11574            .events()
11575            .into_iter()
11576            .find(|event| {
11577                event.target == "control"
11578                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
11579            })
11580            .expect("target admission refusal event");
11581        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11582        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11583        drop(guards);
11584    }
11585
11586    #[tokio::test(flavor = "current_thread")]
11587    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
11588        let handler = ControlHandler::new(Arc::new(Registry::default()));
11589        let capture = EventCapture::default();
11590        let _subscriber =
11591            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11592        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
11593        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11594
11595        let _ = handler
11596            .handle_client_control_request(
11597                &ctx,
11598                frame,
11599                ClientControlRequest::SupervisorSetEnabled {
11600                    module_id: "broca".to_string(),
11601                    enabled: false,
11602                },
11603                &mut None,
11604            )
11605            .await
11606            .unwrap();
11607
11608        let events = capture
11609            .events()
11610            .into_iter()
11611            .filter(|event| {
11612                event.target == "control"
11613                    && event.fields.get("message").map(String::as_str)
11614                        == Some("supervisor request received")
11615            })
11616            .collect::<Vec<_>>();
11617        assert_eq!(events.len(), 1, "one supervisor request log line");
11618        let event = &events[0];
11619        assert_eq!(event.level, tracing::Level::INFO);
11620        assert_eq!(
11621            event.fields.get("op"),
11622            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
11623        );
11624        assert_eq!(
11625            event.fields.get("module_id"),
11626            Some(&"\"broca\"".to_string())
11627        );
11628        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
11629        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
11630        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
11631    }
11632
11633    #[tokio::test(flavor = "current_thread")]
11634    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
11635        let registry = Arc::new(Registry::default());
11636        let module_connection = ConnectionId::new(303);
11637        registry
11638            .register_with_control_ops(
11639                manifest("aft", PROTOCOL_VERSION),
11640                PROTOCOL_VERSION,
11641                module_connection,
11642                module_baseline_control_ops(),
11643            )
11644            .unwrap();
11645        let handler = ControlHandler::new(registry);
11646        let capture = EventCapture::default();
11647        let _subscriber =
11648            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11649        let (ctx, _rx) = route_ctx(module_connection);
11650        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11651
11652        let _ = handler
11653            .handle_client_control_request(
11654                &ctx,
11655                frame,
11656                ClientControlRequest::SupervisorSetEnabled {
11657                    module_id: "broca".to_string(),
11658                    enabled: false,
11659                },
11660                &mut None,
11661            )
11662            .await
11663            .unwrap();
11664
11665        let callers = capture
11666            .events()
11667            .into_iter()
11668            .filter(|event| {
11669                event.fields.get("message").map(String::as_str)
11670                    == Some("supervisor request received")
11671            })
11672            .map(|event| event.fields.get("caller").cloned())
11673            .collect::<Vec<_>>();
11674        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
11675    }
11676
11677    #[tokio::test(flavor = "current_thread")]
11678    async fn supervisor_list_does_not_log_request_received() {
11679        let handler = ControlHandler::new(Arc::new(Registry::default()));
11680        let capture = EventCapture::default();
11681        let _subscriber =
11682            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11683        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
11684        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
11685
11686        handler
11687            .handle_client_control_request(
11688                &ctx,
11689                frame,
11690                ClientControlRequest::SupervisorList {},
11691                &mut None,
11692            )
11693            .await
11694            .unwrap();
11695
11696        assert!(capture.events().into_iter().all(|event| {
11697            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11698        }));
11699    }
11700
11701    #[tokio::test(flavor = "current_thread")]
11702    async fn supervisor_rescan_preview_does_not_log_request_received() {
11703        let handler = ControlHandler::new(Arc::new(Registry::default()));
11704        let capture = EventCapture::default();
11705        let _subscriber =
11706            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11707        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
11708        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
11709
11710        handler
11711            .handle_client_control_request(
11712                &ctx,
11713                frame,
11714                ClientControlRequest::SupervisorRescan { preview: true },
11715                &mut None,
11716            )
11717            .await
11718            .unwrap();
11719
11720        assert!(capture.events().into_iter().all(|event| {
11721            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11722        }));
11723    }
11724
11725    struct ApproveOperator;
11726    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
11727        fn prompt(
11728            &self,
11729            _: &str,
11730            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
11731        ) -> crate::operator_confirm::ProviderResult {
11732            crate::operator_confirm::ProviderResult::Approved
11733        }
11734    }
11735
11736    #[tokio::test]
11737    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
11738        assert_eq!(
11739            ControlHandler::default().privileged_operator_prompt,
11740            PrivilegedOperatorPrompt::Log
11741        );
11742        for (initial, changed, running) in [
11743            ("log", "enforce", PrivilegedOperatorPrompt::Log),
11744            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
11745        ] {
11746            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
11747            let path = dir.join("subc.jsonc");
11748            std::fs::write(
11749                &path,
11750                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
11751            )
11752            .unwrap();
11753            let config = crate::daemon_config::load(&path).unwrap().unwrap();
11754            let registry = Arc::new(Registry::default());
11755            let handle = SupervisorHandle::new();
11756            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
11757                .with_handle(handle.clone());
11758            let handler = ControlHandler::new(registry)
11759                .with_supervisor(handle)
11760                .with_privileged_operator_prompt(config.privileged_operator_prompt)
11761                .with_supervisor_rescan(supervisor, &path, config.port);
11762            handler.forwarding.operator_confirms().configure(
11763                Arc::new(ApproveOperator),
11764                crate::operator_confirm::OperatorLimits::default(),
11765            );
11766            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
11767
11768            for (disk_mode, needs_restart) in [
11769                (Some(initial), false),
11770                (Some(changed), true),
11771                (Some(changed), true),
11772                (None, running == PrivilegedOperatorPrompt::Enforce),
11773                (Some(initial), false),
11774            ] {
11775                let mut doc = json!({"version":1});
11776                if let Some(mode) = disk_mode {
11777                    doc["privileged_operator_prompt"] = json!(mode);
11778                }
11779                std::fs::write(&path, doc.to_string()).unwrap();
11780                for preview in [true, false] {
11781                    let request = ClientControlRequest::SupervisorRescan { preview };
11782                    let frame = Frame::build(
11783                        FrameType::Request,
11784                        control_flags(),
11785                        0,
11786                        0,
11787                        1,
11788                        serde_json::to_vec(&request).unwrap(),
11789                    )
11790                    .unwrap();
11791                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11792                    assert_eq!(frames.len(), 1);
11793                    let ClientControlResponse::SupervisorRescan { result } =
11794                        serde_json::from_slice(&frames[0].body).unwrap()
11795                    else {
11796                        panic!("expected a rescan response");
11797                    };
11798                    let expected = if needs_restart {
11799                        vec!["privileged_operator_prompt"]
11800                    } else {
11801                        Vec::new()
11802                    };
11803                    assert_eq!(result.restart_required, expected);
11804                    assert_eq!(handler.privileged_operator_prompt, running);
11805                    assert_eq!(
11806                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
11807                        running
11808                    );
11809                }
11810            }
11811        }
11812    }
11813
11814    #[tokio::test]
11815    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
11816        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
11817        let path = dir.join("subc.jsonc");
11818        let registry = Arc::new(Registry::default());
11819        let handle = SupervisorHandle::new();
11820        let supervisor =
11821            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
11822        let handler = ControlHandler::new(registry)
11823            .with_supervisor(handle)
11824            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
11825            .with_supervisor_rescan(supervisor, &path, None);
11826        handler.forwarding.operator_confirms().configure(
11827            Arc::new(ApproveOperator),
11828            crate::operator_confirm::OperatorLimits::default(),
11829        );
11830        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
11831        for preview in [true, false] {
11832            std::fs::write(
11833                &path,
11834                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
11835            )
11836            .unwrap();
11837            let request = ClientControlRequest::SupervisorRescan { preview };
11838            let frame = Frame::build(
11839                FrameType::Request,
11840                control_flags(),
11841                0,
11842                0,
11843                1,
11844                serde_json::to_vec(&request).unwrap(),
11845            )
11846            .unwrap();
11847            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11848            assert_eq!(frames.len(), 1);
11849            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
11850            assert_eq!(
11851                handler.privileged_operator_prompt,
11852                PrivilegedOperatorPrompt::Enforce
11853            );
11854        }
11855    }
11856
11857    /// One wire code has several senders, so the refusal line names the check
11858    /// that refused. This drives the shared refusal path for ordinary refusals
11859    /// with an unregistered
11860    /// target and requires the branch label on the event.
11861    #[tokio::test]
11862    async fn route_open_refusal_names_the_check_that_refused() {
11863        let handler = ControlHandler::new(Arc::new(Registry::default()));
11864        let capture = EventCapture::default();
11865        let _subscriber =
11866            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11867        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11868        let response = handler
11869            .handle_control_frame(
11870                &ctx,
11871                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
11872            )
11873            .await
11874            .unwrap();
11875
11876        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11877        let event = capture
11878            .events()
11879            .into_iter()
11880            .find(|event| {
11881                event.target == "control"
11882                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11883            })
11884            .expect("route.open refusal event");
11885        assert_eq!(
11886            event.fields.get("reason"),
11887            Some(&"\"not_registered\"".to_string())
11888        );
11889    }
11890
11891    #[tokio::test]
11892    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
11893        let registry = Arc::new(Registry::default());
11894        let supervisor_handle = SupervisorHandle::new();
11895        let supervisor =
11896            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
11897                .with_handle(supervisor_handle.clone())
11898                .with_connection_file_path(std::env::temp_dir().join(format!(
11899                    "subc-route-open-refusal-info-{}",
11900                    std::process::id()
11901                )));
11902        let module = supervisor
11903            .supervise_configured(
11904                ModuleSpec {
11905                    module_id: "warming".to_string(),
11906                    program: fake_aft_stub_path(),
11907                    args: Vec::new(),
11908                    env: Vec::new(),
11909                    reserved: false,
11910                    reserved_prefixes: Vec::new(),
11911                    protocol: ModuleProtocol::Subc,
11912                    overlap: Default::default(),
11913                },
11914                true,
11915            )
11916            .unwrap();
11917        assert_eq!(module.state().unwrap(), ModuleState::Running);
11918
11919        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11920        assert!(handler
11921            .counters()
11922            .snapshot()
11923            .get("route_open_refused_by_code")
11924            .is_none());
11925        let capture = EventCapture::default();
11926        let _subscriber =
11927            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11928        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
11929        let response = handler
11930            .handle_control_frame(
11931                &ctx,
11932                route_open_frame(394, "warming", unique_project_root("refusal-info")),
11933            )
11934            .await
11935            .unwrap();
11936        module.stop().await.unwrap();
11937
11938        assert_eq!(parse_error(&response[0])["code"], "module_warming");
11939        let event = capture
11940            .events()
11941            .into_iter()
11942            .find(|event| {
11943                event.target == "control"
11944                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
11945            })
11946            .expect("route.open refusal event");
11947        assert_eq!(
11948            event.fields.get("module_id"),
11949            Some(&"\"warming\"".to_string())
11950        );
11951        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
11952        assert_eq!(
11953            event.fields.get("reason"),
11954            Some(&"\"supervised_not_registered\"".to_string())
11955        );
11956        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
11957        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
11958        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
11959        assert_eq!(
11960            handler.counters().snapshot()["route_open_refused_by_code"],
11961            json!({ "module_warming": 1 })
11962        );
11963    }
11964
11965    const OUTAGE_START: &str = "route.open refusing module: not serving";
11966    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
11967
11968    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
11969        capture
11970            .events()
11971            .into_iter()
11972            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
11973            .collect()
11974    }
11975
11976    fn supervise_stub(
11977        registry: &Arc<Registry>,
11978        module_id: &str,
11979        enabled: bool,
11980    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
11981        let supervisor_handle = SupervisorHandle::new();
11982        let supervisor =
11983            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
11984                .with_handle(supervisor_handle.clone())
11985                .with_connection_file_path(std::env::temp_dir().join(format!(
11986                    "subc-route-outage-{module_id}-{}",
11987                    std::process::id()
11988                )));
11989        let module = supervisor
11990            .supervise_configured(
11991                ModuleSpec {
11992                    module_id: module_id.to_string(),
11993                    program: fake_aft_stub_path(),
11994                    args: Vec::new(),
11995                    env: Vec::new(),
11996                    reserved: false,
11997                    reserved_prefixes: Vec::new(),
11998                    protocol: ModuleProtocol::Subc,
11999                    overlap: Default::default(),
12000                },
12001                enabled,
12002            )
12003            .unwrap();
12004        (supervisor_handle, module)
12005    }
12006
12007    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
12008        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
12009            module_id: module_id.to_string(),
12010            drain_timeout_ms: Some(50),
12011        })
12012        .unwrap();
12013        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
12014    }
12015
12016    /// Two handlers built over one forwarding table must share one outage
12017    /// tracker; separate trackers would each log their own opening line for
12018    /// the same outage.
12019    #[test]
12020    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
12021        let registry = Arc::new(Registry::default());
12022        let forwarding = Arc::new(ForwardingTable::default());
12023        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12024        let second = ControlHandler::with_forwarding(registry, forwarding);
12025        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
12026    }
12027
12028    /// A client can name any module id it likes. Refusing an unknown one,
12029    /// however often, must not create outage state or outage lines, or the
12030    /// tracker would be a memory sink any client could fill.
12031    #[tokio::test(flavor = "current_thread")]
12032    async fn route_open_unknown_module_refusals_add_no_outage_state() {
12033        let handler = ControlHandler::new(Arc::new(Registry::default()));
12034        let capture = EventCapture::default();
12035        let _subscriber =
12036            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12037        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12038        for corr in 0..8 {
12039            let response = handler
12040                .handle_control_frame(
12041                    &ctx,
12042                    route_open_frame(
12043                        380 + corr,
12044                        &format!("nobody-{corr}"),
12045                        unique_project_root("outage-unknown"),
12046                    ),
12047                )
12048                .await
12049                .unwrap();
12050            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12051        }
12052
12053        assert_eq!(handler.route_outages.tracked_module_count(), 0);
12054        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12055        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12056    }
12057
12058    /// Drives the refusal path end to end: a supervised module that served
12059    /// before and stopped being registered with no instruction to stop is a
12060    /// WARN, and the same module refused after an operator `supervisor.restart`
12061    /// is an INFO.
12062    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12063    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12064        let registry = Arc::new(Registry::default());
12065        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
12066        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12067        let capture = EventCapture::default();
12068        let _subscriber =
12069            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12070        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12071        // The stub never registers, so pretend it served once: otherwise every
12072        // refusal would fall in its startup window.
12073        handler.route_outages.record_accepted("outage-restart");
12074
12075        let response = handler
12076            .handle_control_frame(
12077                &ctx,
12078                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12079            )
12080            .await
12081            .unwrap();
12082        assert_eq!(response[0].header.ty, FrameType::Error);
12083        let starts = outage_lines(&capture, OUTAGE_START);
12084        assert_eq!(starts.len(), 1, "{starts:?}");
12085        assert_eq!(starts[0].level, tracing::Level::WARN);
12086        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12087        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12088        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12089        handler.route_outages.record_accepted("outage-restart");
12090        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12091
12092        let restart = handler
12093            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12094            .await
12095            .unwrap();
12096        assert_eq!(
12097            restart[0].header.ty,
12098            FrameType::Response,
12099            "{:?}",
12100            parse_error(&restart[0])
12101        );
12102        handler
12103            .handle_control_frame(
12104                &ctx,
12105                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
12106            )
12107            .await
12108            .unwrap();
12109        module.stop().await.unwrap();
12110
12111        let starts = outage_lines(&capture, OUTAGE_START);
12112        assert_eq!(starts.len(), 2, "{starts:?}");
12113        assert_eq!(starts[1].level, tracing::Level::INFO);
12114        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
12115    }
12116
12117    /// A restart refused before it touched the module (here: the module is
12118    /// disabled) must clear its operator mark, so the next real outage is
12119    /// still reported as a warning.
12120    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12121    async fn failed_operator_restart_leaves_no_operator_mark() {
12122        let registry = Arc::new(Registry::default());
12123        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
12124        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12125        let capture = EventCapture::default();
12126        let _subscriber =
12127            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12128        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
12129        handler.route_outages.record_accepted("outage-disabled");
12130
12131        let restart = handler
12132            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
12133            .await
12134            .unwrap();
12135        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
12136        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
12137
12138        handler
12139            .handle_control_frame(
12140                &ctx,
12141                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
12142            )
12143            .await
12144            .unwrap();
12145        let starts = outage_lines(&capture, OUTAGE_START);
12146        assert_eq!(starts.len(), 1, "{starts:?}");
12147        assert_eq!(starts[0].level, tracing::Level::WARN);
12148    }
12149
12150    #[tokio::test(flavor = "current_thread")]
12151    async fn route_open_unknown_module_escapes_target_module_id() {
12152        let handler = ControlHandler::new(Arc::new(Registry::default()));
12153        let capture = EventCapture::default();
12154        let _subscriber =
12155            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12156        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
12157        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12158        let response = handler
12159            .handle_control_frame(
12160                &ctx,
12161                route_open_frame(
12162                    395,
12163                    hostile_module_id,
12164                    unique_project_root("hostile-target-module-id"),
12165                ),
12166            )
12167            .await
12168            .unwrap();
12169
12170        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12171        let event = capture
12172            .events()
12173            .into_iter()
12174            .find(|event| {
12175                event.target == "control"
12176                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12177            })
12178            .expect("route.open unknown-module refusal event");
12179        let logged = event.fields.get("module_id").expect("module_id field");
12180        assert!(!logged.bytes().any(|byte| byte < 0x20));
12181        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12182    }
12183
12184    #[tokio::test(flavor = "current_thread")]
12185    async fn route_open_module_rejection_uses_daemon_counter_key() {
12186        let registry = Arc::new(Registry::default());
12187        let forwarding = Arc::new(ForwardingTable::default());
12188        let handler =
12189            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12190        let module_connection = ConnectionId::new(95);
12191        let (module_ctx, mut module_rx) = route_ctx(module_connection);
12192        hello_via_sink(
12193            &handler,
12194            &module_ctx,
12195            &mut module_rx,
12196            hello_frame("aft", PROTOCOL_VERSION, 395),
12197        )
12198        .await;
12199
12200        let client_connection = ConnectionId::new(96);
12201        let (client_ctx, _client_rx) = route_ctx(client_connection);
12202        let capture = EventCapture::default();
12203        let _subscriber =
12204            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12205        let (route_task, bind) = relay_route_open(
12206            &handler,
12207            client_connection,
12208            &client_ctx.egress,
12209            &mut module_rx,
12210            396,
12211            "aft",
12212            "hostile-module-code",
12213        )
12214        .await;
12215        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
12216        let rejection = Frame::build(
12217            FrameType::Error,
12218            control_flags(),
12219            0,
12220            0,
12221            bind.header.corr,
12222            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
12223        )
12224        .unwrap();
12225        handler
12226            .handle_control_frame(&module_ctx, rejection)
12227            .await
12228            .unwrap();
12229
12230        let response = route_task.await.unwrap();
12231        assert_eq!(parse_error(&response[0])["code"], hostile_code);
12232        let counters = handler.counters().snapshot();
12233        assert_eq!(
12234            counters["route_open_refused_by_code"],
12235            json!({ "module_rejected": 1 })
12236        );
12237        assert!(counters["route_open_refused_by_code"]
12238            .get(hostile_code)
12239            .is_none());
12240
12241        let event = capture
12242            .events()
12243            .into_iter()
12244            .find(|event| {
12245                event.target == "control"
12246                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
12247            })
12248            .expect("route.open module-rejection refusal event");
12249        let logged = event.fields.get("module_code").expect("module_code field");
12250        assert!(!logged.bytes().any(|byte| byte < 0x20));
12251        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12252    }
12253
12254    #[tokio::test]
12255    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12256        let registry = Arc::new(Registry::default());
12257        let supervisor_handle = SupervisorHandle::new();
12258        let missing_program = std::env::temp_dir().join(format!(
12259            "subc-route-open-missing-program-{}",
12260            std::process::id()
12261        ));
12262        let supervisor =
12263            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12264                .with_handle(supervisor_handle.clone());
12265        let module = supervisor
12266            .supervise_configured(
12267                ModuleSpec {
12268                    module_id: "failed".to_string(),
12269                    program: missing_program,
12270                    args: Vec::new(),
12271                    env: Vec::new(),
12272                    reserved: false,
12273                    reserved_prefixes: Vec::new(),
12274                    protocol: ModuleProtocol::Subc,
12275                    overlap: Default::default(),
12276                },
12277                true,
12278            )
12279            .unwrap();
12280        assert_eq!(module.state().unwrap(), ModuleState::Failed);
12281
12282        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12283        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12284        let response = handler
12285            .handle_control_frame(
12286                &ctx,
12287                route_open_frame(305, "failed", unique_project_root("failed")),
12288            )
12289            .await
12290            .unwrap();
12291
12292        assert_eq!(response[0].header.ty, FrameType::Error);
12293        let error = parse_error(&response[0]);
12294        assert_eq!(error["code"], "target_unavailable");
12295        assert!(error["message"]
12296            .as_str()
12297            .unwrap()
12298            .contains("state=failed, enabled=true, live=false"));
12299    }
12300
12301    #[tokio::test]
12302    async fn route_open_role_mismatch_remains_target_unavailable() {
12303        let registry = Arc::new(Registry::default());
12304        let handler = ControlHandler::new(Arc::clone(&registry));
12305        handler
12306            .handle_control(
12307                ConnectionId::new(41),
12308                non_routable_hello_frame_with_control_ops("health-only", 306, None),
12309            )
12310            .unwrap();
12311
12312        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12313        let response = handler
12314            .handle_control_frame(
12315                &ctx,
12316                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12317            )
12318            .await
12319            .unwrap();
12320
12321        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12322        assert!(parse_error(&response[0])["message"]
12323            .as_str()
12324            .unwrap()
12325            .contains("does not provide the requested target"));
12326    }
12327
12328    #[tokio::test]
12329    async fn route_open_inactive_registration_remains_target_unavailable() {
12330        let registry = Arc::new(Registry::default());
12331        let handler = ControlHandler::new(Arc::clone(&registry));
12332        handler
12333            .handle_control(
12334                ConnectionId::new(43),
12335                hello_frame("inactive", PROTOCOL_VERSION, 308),
12336            )
12337            .unwrap();
12338        assert!(registry
12339            .set_module_state_for_test("inactive", ChannelState::Closed)
12340            .unwrap());
12341
12342        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12343        let response = handler
12344            .handle_control_frame(
12345                &ctx,
12346                route_open_frame(309, "inactive", unique_project_root("inactive")),
12347            )
12348            .await
12349            .unwrap();
12350
12351        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12352        assert!(parse_error(&response[0])["message"]
12353            .as_str()
12354            .unwrap()
12355            .contains("is not active"));
12356    }
12357
12358    #[tokio::test]
12359    async fn late_health_reply_is_recorded_through_the_module_response_path() {
12360        let registry = Arc::new(Registry::default());
12361        let forwarding = Arc::new(ForwardingTable::default());
12362        let supervisor_handle = SupervisorHandle::new();
12363        let supervisor =
12364            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
12365                .with_forwarding(Arc::clone(&forwarding))
12366                .with_handle(supervisor_handle.clone());
12367        let module = supervisor
12368            .supervise_configured(
12369                crate::ModuleSpec {
12370                    module_id: "late-health-response".to_string(),
12371                    program: PathBuf::from("disabled-module"),
12372                    args: Vec::new(),
12373                    env: Vec::new(),
12374                    reserved: false,
12375                    reserved_prefixes: Vec::new(),
12376                    protocol: ModuleProtocol::Subc,
12377                    overlap: Default::default(),
12378                },
12379                false,
12380            )
12381            .unwrap();
12382        let handler =
12383            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12384                .with_supervisor(supervisor_handle);
12385        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12386        handler
12387            .handle_control_frame(
12388                &module_ctx,
12389                hello_frame_with_control_ops(
12390                    "late-health-response",
12391                    PROTOCOL_VERSION,
12392                    7,
12393                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12394                ),
12395            )
12396            .await
12397            .unwrap();
12398        let probe_started_at = Instant::now() - Duration::from_millis(80);
12399        let pending = forwarding
12400            .begin_health_probe_rpc_for(
12401                "late-health-response",
12402                MODULE_CONTROL_OP_HEALTH_CHECK,
12403                probe_started_at,
12404                Instant::now() - Duration::from_millis(1),
12405            )
12406            .unwrap();
12407        assert!(forwarding
12408            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12409            .unwrap());
12410
12411        let responses = handler
12412            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12413            .await
12414            .unwrap();
12415
12416        assert!(responses.is_empty());
12417        let health = module.status().unwrap().health;
12418        assert_eq!(health.late_answer_count, 1);
12419        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12420    }
12421
12422    #[tokio::test]
12423    async fn health_probe_timeout_and_module_death_are_typed() {
12424        let registry = Arc::new(Registry::default());
12425        let forwarding = Arc::new(ForwardingTable::default());
12426        let handler =
12427            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12428                .with_health_probe_timeout(Duration::from_millis(50));
12429        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12430        hello_via_sink(
12431            &handler,
12432            &module_ctx,
12433            &mut module_rx,
12434            hello_frame_with_control_ops(
12435                "aft",
12436                PROTOCOL_VERSION,
12437                7,
12438                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12439            ),
12440        )
12441        .await;
12442
12443        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12444        let responses = handler
12445            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12446            .await
12447            .unwrap();
12448        assert_eq!(responses[0].header.ty, FrameType::Error);
12449        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12450        let _ = module_rx.try_recv();
12451
12452        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12453        let health_handler = handler.clone();
12454        let death_task = tokio::spawn(async move {
12455            health_handler
12456                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12457                .await
12458                .unwrap()
12459        });
12460        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12461            .await
12462            .unwrap()
12463            .unwrap();
12464        handler
12465            .cleanup_connection(module_ctx.connection_id)
12466            .unwrap();
12467        let responses = death_task.await.unwrap();
12468        assert_eq!(responses[0].header.ty, FrameType::Error);
12469        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12470    }
12471
12472    #[test]
12473    fn hello_requires_exact_protocol_version() {
12474        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12475            let registry = Arc::new(Registry::default());
12476            let handler = ControlHandler::new(Arc::clone(&registry));
12477            let responses = handler
12478                .handle_control(
12479                    ConnectionId::new(connection),
12480                    hello_frame("aft", offered, 9),
12481                )
12482                .unwrap();
12483
12484            assert_eq!(responses.len(), 1);
12485            assert_eq!(responses[0].header.ty, FrameType::Error);
12486            let error = parse_error(&responses[0]);
12487            assert_eq!(error["code"], "version_unsupported");
12488            assert!(registry.get_module("aft").unwrap().is_none());
12489            assert_eq!(registry.active_registration_count().unwrap(), 0);
12490        }
12491    }
12492
12493    #[test]
12494    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12495        let registry = Arc::new(Registry::default());
12496        let forwarding = Arc::new(ForwardingTable::default());
12497        let handler =
12498            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12499        let module_connection = ConnectionId::new(301);
12500        let registration = registry
12501            .register_with_control_ops(
12502                manifest("aft-push", PROTOCOL_VERSION),
12503                PROTOCOL_VERSION,
12504                module_connection,
12505                module_baseline_control_ops(),
12506            )
12507            .unwrap();
12508        let (module_tx, _module_rx) = mpsc::channel(8);
12509        let endpoint = forwarding
12510            .register_module_connection(
12511                module_connection,
12512                "aft-push".to_string(),
12513                PROTOCOL_VERSION,
12514                manifest_concurrency(&registration.manifest),
12515                FrameSink::new(module_tx),
12516            )
12517            .unwrap();
12518
12519        // A push op this version does not know is ignored (forward-compat), not errored.
12520        let unknown = Frame::build(
12521            FrameType::Push,
12522            control_flags(),
12523            0,
12524            0,
12525            5,
12526            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12527        )
12528        .unwrap();
12529        let out = handler.handle_status_update(endpoint, unknown).unwrap();
12530        assert!(
12531            out.is_empty(),
12532            "unknown push op must be ignored, got {out:?}"
12533        );
12534
12535        // A malformed body for a KNOWN op is a real error worth surfacing.
12536        let malformed = Frame::build(
12537            FrameType::Push,
12538            control_flags(),
12539            0,
12540            0,
12541            6,
12542            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
12543        )
12544        .unwrap();
12545        let out = handler.handle_status_update(endpoint, malformed).unwrap();
12546        assert_eq!(out.len(), 1);
12547        assert_eq!(out[0].header.ty, FrameType::Error);
12548        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
12549    }
12550
12551    #[test]
12552    fn hello_rejected_when_connection_already_owns_client_routes() {
12553        let registry = Arc::new(Registry::default());
12554        let forwarding = Arc::new(ForwardingTable::default());
12555        let handler =
12556            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12557        // Commits a client route on connection 202 (bound to a module on conn 101).
12558        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
12559        let client_connection = ConnectionId::new(202);
12560
12561        // That same connection now tries to register as a module: rejected, so one
12562        // connection never holds both client-route and module-endpoint state.
12563        let responses = handler
12564            .handle_control(
12565                client_connection,
12566                hello_frame("aft-second", PROTOCOL_VERSION, 9),
12567            )
12568            .unwrap();
12569        assert_eq!(responses[0].header.ty, FrameType::Error);
12570        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
12571        assert!(registry.get_module("aft-second").unwrap().is_none());
12572    }
12573
12574    #[tokio::test]
12575    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
12576        let registry = Arc::new(Registry::default());
12577        let forwarding = Arc::new(ForwardingTable::default());
12578        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
12579        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
12580        hello_via_sink(
12581            &handler,
12582            &module_ctx,
12583            &mut module_rx,
12584            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
12585        )
12586        .await;
12587        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
12588        let pending = forwarding
12589            .begin_route_bind_relay_for_test(
12590                client_ctx.connection_id,
12591                client_ctx.egress.clone(),
12592                2,
12593                "alpha",
12594            )
12595            .unwrap();
12596        forwarding
12597            .complete_pending_relay(
12598                module_ctx.connection_id,
12599                pending.corr,
12600                RouteBindRelayOutcome::Accepted,
12601            )
12602            .unwrap();
12603        client_rx.try_recv().unwrap();
12604        for module_id in ["beta", "alpha"] {
12605            let replies = handler
12606                .handle_control_frame(
12607                    &module_ctx,
12608                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
12609                )
12610                .await
12611                .unwrap();
12612            assert_eq!(replies.len(), 1, "second HELLO must be refused");
12613            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
12614        }
12615        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
12616        assert!(registry.get_module("beta").unwrap().is_none());
12617        assert!(matches!(
12618            forwarding
12619                .lookup_data_route(
12620                    client_ctx.connection_id,
12621                    pending.client_channel,
12622                    pending.client_epoch,
12623                )
12624                .unwrap(),
12625            DataRoute::Client(DataRouteState::Bound(_))
12626        ));
12627        assert!(handler
12628            .hello_launch_nonces
12629            .lock()
12630            .unwrap()
12631            .presented(module_ctx.connection_id, Some("alpha-nonce")));
12632        assert!(module_rx.try_recv().is_err());
12633    }
12634
12635    #[test]
12636    fn reserved_module_hello_requires_matching_launch_nonce() {
12637        let registry = Arc::new(Registry::default());
12638        let supervisor = SupervisorHandle::new();
12639        // The supervisor recorded the nonce it injected when it spawned the reserved
12640        // module; the HELLO verifier checks against the same shared handle.
12641        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12642        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
12643
12644        // A HELLO with NO nonce is rejected.
12645        let no_nonce = handler
12646            .handle_control(
12647                ConnectionId::new(1),
12648                hello_frame("vault", PROTOCOL_VERSION, 1),
12649            )
12650            .unwrap();
12651        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
12652        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
12653        assert!(registry.get_module("vault").unwrap().is_none());
12654
12655        // A HELLO with the WRONG nonce is rejected.
12656        let wrong = handler
12657            .handle_control(
12658                ConnectionId::new(2),
12659                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
12660            )
12661            .unwrap();
12662        assert_eq!(wrong[0].header.ty, FrameType::Error);
12663        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
12664        assert!(registry.get_module("vault").unwrap().is_none());
12665
12666        // A HELLO with the CORRECT nonce registers.
12667        let ok = handler
12668            .handle_control(
12669                ConnectionId::new(3),
12670                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
12671            )
12672            .unwrap();
12673        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
12674        assert!(registry.get_module("vault").unwrap().is_some());
12675    }
12676
12677    #[test]
12678    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
12679        let registry = Arc::new(Registry::default());
12680        let supervisor = SupervisorHandle::new();
12681        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12682        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12683        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
12684
12685        let squat = handler
12686            .handle_control(
12687                ConnectionId::new(1),
12688                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
12689            )
12690            .unwrap();
12691        assert_eq!(squat[0].header.ty, FrameType::Error);
12692        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
12693        assert!(parse_error(&squat[0])["message"]
12694            .as_str()
12695            .unwrap()
12696            .contains("fed:"));
12697
12698        let accepted_peer = handler
12699            .handle_control(
12700                ConnectionId::new(2),
12701                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
12702            )
12703            .unwrap();
12704        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
12705
12706        let accepted_short = handler
12707            .handle_control(
12708                ConnectionId::new(3),
12709                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
12710            )
12711            .unwrap();
12712        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
12713
12714        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
12715            let response = handler
12716                .handle_control(
12717                    ConnectionId::new(conn),
12718                    hello_frame(module_id, PROTOCOL_VERSION, conn),
12719                )
12720                .unwrap();
12721            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
12722        }
12723    }
12724
12725    #[test]
12726    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
12727        let registry = Arc::new(Registry::default());
12728        let supervisor = SupervisorHandle::new();
12729        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12730        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12731        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
12732        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
12733
12734        let owner_nonce = handler
12735            .handle_control(
12736                ConnectionId::new(1),
12737                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
12738            )
12739            .unwrap();
12740        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
12741        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
12742        assert!(registry.get_module("fed:special").unwrap().is_none());
12743
12744        let exact_nonce = handler
12745            .handle_control(
12746                ConnectionId::new(2),
12747                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
12748            )
12749            .unwrap();
12750        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
12751        assert!(registry.get_module("fed:special").unwrap().is_some());
12752    }
12753
12754    #[test]
12755    fn non_reserved_module_ignores_launch_nonce() {
12756        let registry = Arc::new(Registry::default());
12757        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
12758        // registration succeeds whether a spawned process echoes a nonce or not.
12759        let handler = ControlHandler::new(Arc::clone(&registry));
12760        let no_nonce = handler
12761            .handle_control(
12762                ConnectionId::new(1),
12763                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
12764            )
12765            .unwrap();
12766        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
12767        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
12768
12769        let echoed_nonce = handler
12770            .handle_control(
12771                ConnectionId::new(2),
12772                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
12773            )
12774            .unwrap();
12775        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
12776        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
12777    }
12778
12779    #[test]
12780    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
12781        let handler = ControlHandler::default();
12782        let conn = ConnectionId::new(1);
12783        let malformed = Frame::build(
12784            FrameType::Hello,
12785            control_flags(),
12786            0,
12787            0,
12788            3,
12789            b"{not json".to_vec(),
12790        )
12791        .unwrap();
12792
12793        let error = handler.handle_control(conn, malformed).unwrap();
12794        assert_eq!(error[0].header.ty, FrameType::Error);
12795        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
12796
12797        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12798        let pong = handler.handle_control(conn, ping).unwrap();
12799        assert_eq!(pong[0].header.ty, FrameType::Pong);
12800        assert_eq!(pong[0].header.corr, 4);
12801    }
12802
12803    #[test]
12804    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
12805        let registry = Arc::new(Registry::default());
12806        let handler = ControlHandler::new(Arc::clone(&registry));
12807
12808        handler
12809            .handle_control(
12810                ConnectionId::new(1),
12811                hello_frame("aft", PROTOCOL_VERSION, 1),
12812            )
12813            .unwrap();
12814        let duplicate = handler
12815            .handle_control(
12816                ConnectionId::new(2),
12817                hello_frame("aft", PROTOCOL_VERSION, 2),
12818            )
12819            .unwrap();
12820
12821        assert_eq!(duplicate[0].header.ty, FrameType::Error);
12822        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12823        let registration = registry.get_module("aft").unwrap().unwrap();
12824        assert_eq!(registration.connection_id, ConnectionId::new(1));
12825    }
12826
12827    #[test]
12828    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
12829        let registry = Arc::new(Registry::default());
12830        let forwarding = Arc::new(ForwardingTable::default());
12831        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
12832        let handler =
12833            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12834                .with_process_liveness(process_liveness);
12835        let (ctx, route_channel, route_epoch) =
12836            bind_liveness_route(&registry, &forwarding, "aft-dead");
12837        let responses = handler
12838            .handle_route_poll(
12839                &ctx,
12840                route_poll_frame(41, PollKind::Liveness, route_channel),
12841                route_channel,
12842                route_epoch,
12843                PollKind::Liveness,
12844            )
12845            .unwrap();
12846
12847        assert_eq!(responses.len(), 1);
12848        assert_eq!(responses[0].header.ty, FrameType::Response);
12849        assert_route_poll_liveness(&responses[0], false);
12850    }
12851
12852    #[test]
12853    fn liveness_poll_without_process_source_uses_bound_route() {
12854        let registry = Arc::new(Registry::default());
12855        let forwarding = Arc::new(ForwardingTable::default());
12856        let handler =
12857            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12858        let (ctx, route_channel, route_epoch) =
12859            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
12860        let responses = handler
12861            .handle_route_poll(
12862                &ctx,
12863                route_poll_frame(42, PollKind::Liveness, route_channel),
12864                route_channel,
12865                route_epoch,
12866                PollKind::Liveness,
12867            )
12868            .unwrap();
12869
12870        assert_route_poll_liveness(&responses[0], true);
12871    }
12872
12873    #[test]
12874    fn liveness_poll_untracked_process_source_uses_bound_route() {
12875        let registry = Arc::new(Registry::default());
12876        let forwarding = Arc::new(ForwardingTable::default());
12877        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
12878        let handler =
12879            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12880                .with_process_liveness(process_liveness);
12881        let (ctx, route_channel, route_epoch) =
12882            bind_liveness_route(&registry, &forwarding, "aft-untracked");
12883        let responses = handler
12884            .handle_route_poll(
12885                &ctx,
12886                route_poll_frame(43, PollKind::Liveness, route_channel),
12887                route_channel,
12888                route_epoch,
12889                PollKind::Liveness,
12890            )
12891            .unwrap();
12892
12893        assert_route_poll_liveness(&responses[0], true);
12894    }
12895
12896    #[tokio::test]
12897    async fn unknown_op_returns_unknown_control_op() {
12898        let handler = ControlHandler::default();
12899        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
12900        let request = Frame::build(
12901            FrameType::Request,
12902            control_flags(),
12903            0,
12904            0,
12905            55,
12906            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
12907        )
12908        .unwrap();
12909
12910        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12911
12912        assert_eq!(response.len(), 1);
12913        assert_eq!(response[0].header.ty, FrameType::Error);
12914        assert_eq!(response[0].header.corr, 55);
12915        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
12916    }
12917
12918    #[tokio::test]
12919    async fn supervisor_provenance_rejects_unknown_exact_module() {
12920        let handler = ControlHandler::default();
12921        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
12922        let request = Frame::build(
12923            FrameType::Request,
12924            control_flags(),
12925            0,
12926            0,
12927            57,
12928            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
12929        )
12930        .unwrap();
12931
12932        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12933
12934        assert_eq!(response.len(), 1);
12935        assert_eq!(response[0].header.ty, FrameType::Error);
12936        assert_eq!(response[0].header.corr, 57);
12937        let error = parse_error(&response[0]);
12938        assert_eq!(error["code"], "unknown_module");
12939        assert_eq!(error["message"], "module_id 'missing' is not supervised");
12940    }
12941
12942    #[test]
12943    fn provenance_probe_override_keeps_handler_tests_deterministic() {
12944        let expected = subc_control::RunningImageAgreement::Unavailable {
12945            reason: subc_control::RunningImageUnavailableReason::HashFailed,
12946        };
12947        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
12948        assert_eq!(handler.provenance_probe_override, Some(expected));
12949    }
12950
12951    #[test]
12952    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
12953        let verdict = reload_verdict(
12954            std::path::Path::new("/bin/new"),
12955            Some(std::path::Path::new("/bin/old")),
12956            subc_control::RunningImageAgreement::Unavailable {
12957                reason: subc_control::RunningImageUnavailableReason::HashFailed,
12958            },
12959        );
12960        assert!(matches!(
12961            verdict.path,
12962            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
12963                if configured == std::path::Path::new("/bin/new")
12964                    && spawned_from == std::path::Path::new("/bin/old")
12965        ));
12966    }
12967
12968    #[test]
12969    fn launch_spec_without_running_digest_is_unavailable_not_match() {
12970        use subc_control::{
12971            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
12972        };
12973        let configured = crate::supervise::ModuleSpec {
12974            module_id: "legacy".into(),
12975            program: "/module".into(),
12976            args: Vec::new(),
12977            env: Vec::new(),
12978            reserved: false,
12979            reserved_prefixes: Vec::new(),
12980            protocol: subc_control::ModuleProtocol::Subc,
12981            overlap: Default::default(),
12982        };
12983        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
12984        assert_eq!(
12985            launch_spec_agreement(&configured, false, Some(&digest)),
12986            Agreement::Unavailable {
12987                reason: Reason::NotRunning
12988            }
12989        );
12990        assert_eq!(
12991            launch_spec_agreement(&configured, true, None),
12992            Agreement::Unavailable {
12993                reason: Reason::SpawnedSpecUnavailable
12994            }
12995        );
12996        assert_eq!(
12997            launch_spec_agreement(&configured, true, Some(&digest)),
12998            Agreement::Match
12999        );
13000    }
13001
13002    #[cfg(windows)]
13003    #[test]
13004    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
13005        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
13006        let configured = dir.join("module.exe");
13007        std::fs::write(&configured, b"executable path fixture").unwrap();
13008        let spawned = std::fs::canonicalize(&configured).unwrap();
13009        assert_ne!(
13010            configured, spawned,
13011            "fixture must exercise different path spellings"
13012        );
13013        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
13014        let lower_case = configured.to_string_lossy().to_lowercase();
13015        for spelling in [
13016            &configured,
13017            Path::new(&forward_slashes),
13018            Path::new(&lower_case),
13019        ] {
13020            let verdict = reload_verdict(
13021                spelling,
13022                Some(&spawned),
13023                subc_control::RunningImageAgreement::Unavailable {
13024                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13025                },
13026            );
13027            assert_eq!(
13028                verdict.path,
13029                ReloadPathAgreement::Match,
13030                "configured {spelling:?}, spawned {spawned:?}"
13031            );
13032        }
13033    }
13034
13035    #[cfg(windows)]
13036    #[test]
13037    fn windows_reload_path_keeps_different_existing_program_pending() {
13038        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
13039        let old = dir.join("old.exe");
13040        let new = dir.join("new.exe");
13041        std::fs::write(&old, b"same executable content").unwrap();
13042        std::fs::copy(&old, &new).unwrap();
13043        let spawned = std::fs::canonicalize(&old).unwrap();
13044        let image = subc_control::RunningImageAgreement::Unavailable {
13045            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13046        };
13047        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
13048        assert!(
13049            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
13050            if configured == new && spawned_from == spawned)
13051        );
13052        let missing = dir.join("missing.exe");
13053        assert!(matches!(
13054            reload_verdict(&missing, Some(&spawned), image).path,
13055            ReloadPathAgreement::Mismatch { .. }
13056        ));
13057    }
13058
13059    #[test]
13060    fn reload_verdict_detects_replaced_image_at_same_path() {
13061        let image = subc_control::RunningImageAgreement::Mismatch {
13062            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13063                digest: "old".into(),
13064            },
13065            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13066                digest: "new".into(),
13067            },
13068        };
13069        let verdict = reload_verdict(
13070            std::path::Path::new("/bin/same"),
13071            Some(std::path::Path::new("/bin/same")),
13072            image.clone(),
13073        );
13074        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13075        assert_eq!(verdict.image, image);
13076    }
13077
13078    #[test]
13079    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13080        let image = subc_control::RunningImageAgreement::Unavailable {
13081            reason: subc_control::RunningImageUnavailableReason::NotRunning,
13082        };
13083        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13084        assert_eq!(
13085            verdict.path,
13086            subc_control::ReloadPathAgreement::Unavailable {
13087                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13088            }
13089        );
13090        assert_eq!(verdict.image, image);
13091
13092        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13093            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13094        };
13095        let verdict = reload_verdict(
13096            std::path::Path::new("/bin/same"),
13097            Some(std::path::Path::new("/bin/same")),
13098            unconfirmed.clone(),
13099        );
13100        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13101        assert_eq!(verdict.image, unconfirmed);
13102    }
13103
13104    #[test]
13105    fn reload_verdict_preserves_each_image_unavailability_reason() {
13106        use subc_control::RunningImageUnavailableReason as Reason;
13107
13108        for reason in [
13109            Reason::NotRunning,
13110            Reason::UnsupportedPlatform,
13111            Reason::RunningExecutableUnreadable,
13112            Reason::SpawnedPathUnreadable,
13113            Reason::HashFailed,
13114            Reason::ProcessIdentityUnconfirmed,
13115            Reason::Unknown("future_probe_reason".to_string()),
13116        ] {
13117            let image = subc_control::RunningImageAgreement::Unavailable {
13118                reason: reason.clone(),
13119            };
13120            let verdict = reload_verdict(
13121                std::path::Path::new("/bin/same"),
13122                Some(std::path::Path::new("/bin/same")),
13123                image.clone(),
13124            );
13125            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13126            assert_eq!(verdict.image, image, "{reason:?}");
13127        }
13128    }
13129
13130    #[tokio::test]
13131    async fn malformed_control_bodies_return_invalid_control_body() {
13132        let handler = ControlHandler::default();
13133        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
13134
13135        for (corr, body) in [
13136            (56, br#"{"route_channel":1}"#.as_slice()),
13137            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
13138            (
13139                58,
13140                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
13141            ),
13142        ] {
13143            let request = Frame::build(
13144                FrameType::Request,
13145                control_flags(),
13146                0,
13147                0,
13148                corr,
13149                body.to_vec(),
13150            )
13151            .unwrap();
13152            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13153
13154            assert_eq!(response.len(), 1);
13155            assert_eq!(response[0].header.ty, FrameType::Error);
13156            assert_eq!(response[0].header.corr, corr);
13157            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
13158        }
13159    }
13160
13161    #[tokio::test]
13162    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
13163        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13164        let registry = Arc::new(Registry::default());
13165        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
13166        let router = Router::with_control_handler(Arc::clone(&control));
13167        let connection = router.begin_connection();
13168        let (ctx, mut rx) = route_ctx(connection.id());
13169
13170        router
13171            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
13172            .await
13173            .unwrap();
13174        let response = rx.recv().await.unwrap();
13175        let ack = parse_ack(&response);
13176        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13177        let channel = 1;
13178
13179        let goodbye =
13180            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
13181        router.route_for_connection(&ctx, goodbye).await.unwrap();
13182        assert!(rx.try_recv().is_err());
13183        assert!(registry.get_module("aft").unwrap().is_none());
13184
13185        router
13186            .route_for_connection(&ctx, channel_request(channel, 13))
13187            .await
13188            .unwrap();
13189        let error_frame = rx.recv().await.unwrap();
13190        assert_eq!(error_frame.header.ty, FrameType::Error);
13191        assert_eq!(error_frame.header.channel, channel);
13192        let captured = crate::router::test_log::captured_logs(&logs);
13193        assert_eq!(
13194            captured
13195                .lines()
13196                .filter(|line| {
13197                    line.contains("module_id=aft")
13198                        && line.contains("reason=explicit_goodbye")
13199                        && line.contains("module registration ended")
13200                })
13201                .count(),
13202            1,
13203            "unexpected GOODBYE registry log: {captured}"
13204        );
13205    }
13206
13207    #[tokio::test]
13208    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
13209        let registry = Arc::new(Registry::default());
13210        let handler = ControlHandler::new(registry).with_capability_config(
13211            [("prov".to_string(), true), ("cons".to_string(), true)],
13212            BTreeMap::new(),
13213        );
13214        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
13215        register_capability_manifest(
13216            &handler,
13217            &provider_ctx,
13218            &mut provider_rx,
13219            capability_manifest("prov", &["thing/v1"], &[]),
13220            1,
13221        )
13222        .await;
13223        let mut consumer = capability_manifest("cons", &[], &[]);
13224        consumer.capabilities.as_mut().unwrap().requires.push(
13225            subc_protocol::manifest::CapabilityRequirement {
13226                capability: "thing/v1".to_string(),
13227                need: subc_protocol::manifest::CapabilityNeed::Required,
13228            },
13229        );
13230        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
13231        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
13232        assert_eq!(
13233            handler.capability_evaluator.verdict("cons", "thing/v1"),
13234            Some(CapabilityVerdict::Provided)
13235        );
13236        let (mut client_rx, _) = open_route_for_capability_test(
13237            &handler,
13238            &provider_ctx,
13239            &mut provider_rx,
13240            703,
13241            3,
13242            "prov",
13243            None,
13244        )
13245        .await;
13246        let goodbye =
13247            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13248        handler
13249            .handle_control_frame(&provider_ctx, goodbye)
13250            .await
13251            .unwrap();
13252        assert_eq!(
13253            handler.capability_evaluator.verdict("cons", "thing/v1"),
13254            Some(CapabilityVerdict::NeverProvided)
13255        );
13256        let closed = client_rx
13257            .try_recv()
13258            .expect("GOODBYE pushes route.closed before route GOODBYE");
13259        assert!(
13260            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
13261            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
13262        );
13263        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
13264        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
13265    }
13266
13267    struct RouteClosedFixture {
13268        handler: ControlHandler,
13269        #[cfg(unix)]
13270        supervisor: Supervisor,
13271        module_ctx: RouteCtx,
13272        #[cfg(unix)]
13273        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13274        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13275        route: Arc<crate::forwarding::RouteBinding>,
13276    }
13277
13278    async fn route_closed_fixture() -> RouteClosedFixture {
13279        let registry = Arc::new(Registry::default());
13280        let handler = ControlHandler::new(Arc::clone(&registry));
13281        #[cfg(unix)]
13282        let supervisor = Supervisor::new(registry, RestartPolicy::default())
13283            .with_forwarding(Arc::clone(&handler.forwarding));
13284        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13285        register_capability_manifest(
13286            &handler,
13287            &module_ctx,
13288            &mut module_rx,
13289            capability_manifest("prov", &[], &[]),
13290            1,
13291        )
13292        .await;
13293        let (client_rx, bind) = open_route_for_capability_test(
13294            &handler,
13295            &module_ctx,
13296            &mut module_rx,
13297            702,
13298            2,
13299            "prov",
13300            None,
13301        )
13302        .await;
13303        let ModuleControlRequest::RouteBind {
13304            route_channel: channel,
13305            epoch: route_epoch,
13306            ..
13307        } = bind
13308        else {
13309            panic!("expected route.bind");
13310        };
13311        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13312            handler
13313                .forwarding
13314                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13315                .unwrap()
13316        else {
13317            panic!("the acknowledged route must be bound");
13318        };
13319        RouteClosedFixture {
13320            handler,
13321            #[cfg(unix)]
13322            supervisor,
13323            module_ctx,
13324            #[cfg(unix)]
13325            module_rx,
13326            client_rx,
13327            route,
13328        }
13329    }
13330
13331    impl RouteClosedFixture {
13332        #[cfg(unix)]
13333        async fn cut(&mut self) {
13334            let notices = async {
13335                let mut draining = self.module_rx.recv().await.unwrap();
13336                assert!(matches!(
13337                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13338                        &draining.body
13339                    )
13340                    .unwrap(),
13341                    subc_protocol::session::ModuleControlCommand::Draining {
13342                        reason: RouteCloseReason::Restart,
13343                        ..
13344                    }
13345                ));
13346                draining.flushed.take().unwrap().send(()).unwrap();
13347                let mut closing = self.client_rx.recv().await.unwrap();
13348                assert!(matches!(
13349                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13350                    ClientControlPush::RouteClosing {
13351                        reason: RouteCloseReason::Restart,
13352                        ..
13353                    }
13354                ));
13355                closing.flushed.take().unwrap().send(()).unwrap();
13356            };
13357            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13358            result.unwrap();
13359        }
13360
13361        fn disconnect(&mut self) -> ClientControlPush {
13362            self.handler
13363                .cleanup_connection(self.module_ctx.connection_id)
13364                .unwrap();
13365            let closed = self.client_rx.try_recv().unwrap();
13366            assert_eq!(closed.header.ty, FrameType::Push);
13367            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13368            assert!(matches!(
13369                &push,
13370                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13371                    if channels == &[self.route.client_channel]
13372            ));
13373            assert_eq!(
13374                self.client_rx.try_recv().unwrap().header.ty,
13375                FrameType::Goodbye
13376            );
13377            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13378            push
13379        }
13380    }
13381
13382    #[cfg(unix)]
13383    #[tokio::test(start_paused = true)]
13384    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13385        let mut fixture = route_closed_fixture().await;
13386        fixture.cut().await;
13387        assert!(matches!(
13388            fixture.disconnect(),
13389            ClientControlPush::RouteClosed {
13390                reason: RouteCloseReason::Restart,
13391                drained: true,
13392                ..
13393            }
13394        ));
13395    }
13396
13397    #[cfg(unix)]
13398    #[tokio::test(start_paused = true)]
13399    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13400        let mut fixture = route_closed_fixture().await;
13401        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13402        let started = tokio::time::Instant::now();
13403        fixture.cut().await;
13404        assert!(started.elapsed() >= Duration::from_secs(2));
13405        assert_eq!(fixture.route.flow.in_flight(), 1);
13406        assert!(matches!(
13407            fixture.disconnect(),
13408            ClientControlPush::RouteClosed {
13409                reason: RouteCloseReason::Restart,
13410                drained: false,
13411                ..
13412            }
13413        ));
13414    }
13415
13416    #[tokio::test]
13417    async fn module_connection_crash_reports_crash_and_not_drained() {
13418        let mut fixture = route_closed_fixture().await;
13419        // The server calls cleanup_connection on module EOF. Without a planned
13420        // daemon shutdown, that disconnect must remain a crash with no prior
13421        // route.closing notice, even when the route has no in-flight calls.
13422        assert!(matches!(
13423            fixture.disconnect(),
13424            ClientControlPush::RouteClosed {
13425                reason: RouteCloseReason::Crash,
13426                drained: false,
13427                ..
13428            }
13429        ));
13430    }
13431
13432    #[tokio::test]
13433    async fn dropping_router_connection_releases_registration() {
13434        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13435        let registry = Arc::new(Registry::default());
13436        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
13437        let router = Router::with_control_handler(Arc::clone(&control));
13438        let connection = router.begin_connection();
13439        let connection_id = connection.id();
13440        let (ctx, mut rx) = route_ctx(connection_id);
13441
13442        router
13443            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13444            .await
13445            .unwrap();
13446        let response = rx.recv().await.unwrap();
13447        let ack = parse_ack(&response);
13448        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13449        assert!(registry.get_module("aft").unwrap().is_some());
13450
13451        drop(connection);
13452
13453        assert!(registry.get_module("aft").unwrap().is_none());
13454        assert_eq!(registry.active_registration_count().unwrap(), 0);
13455
13456        control
13457            .cleanup_connection(ConnectionId::new(u64::MAX))
13458            .unwrap();
13459        let captured = crate::router::test_log::captured_logs(&logs);
13460        println!("captured registry lifecycle logs:\n{captured}");
13461        let events: Vec<_> = captured
13462            .lines()
13463            .filter(|line| {
13464                line.contains("module registered module_id=aft ")
13465                    || line.contains("module registration ended")
13466            })
13467            .collect();
13468        assert_eq!(
13469            events.len(),
13470            2,
13471            "unexpected registry lifecycle logs: {captured}"
13472        );
13473        assert!(events[0].contains("module registered module_id=aft "));
13474        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13475        assert!(events[1].contains(&format!(
13476            "module_id=aft connection_id={}",
13477            connection_id.get()
13478        )));
13479        assert!(events[1].contains("reason=connection_closed"));
13480        assert!(events[1].contains("module registration ended"));
13481        assert_eq!(
13482            captured
13483                .lines()
13484                .filter(|line| line.contains("module registered module_id=aft "))
13485                .count(),
13486            1,
13487            "legacy registration admission line must appear once: {captured}"
13488        );
13489    }
13490
13491    #[tokio::test]
13492    async fn hello_registration_keeps_legacy_module_registered_line_once() {
13493        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13494        let registry = Arc::new(Registry::default());
13495        let control = ControlHandler::new(Arc::clone(&registry));
13496        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13497        hello_via_sink(
13498            &control,
13499            &ctx,
13500            &mut rx,
13501            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13502        )
13503        .await;
13504
13505        let captured = crate::router::test_log::captured_logs(&logs);
13506        let admissions: Vec<_> = captured
13507            .lines()
13508            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13509            .collect();
13510        assert_eq!(
13511            admissions.len(),
13512            1,
13513            "legacy admission line must remain exactly once: {captured}"
13514        );
13515        assert!(admissions[0].contains("routable_provider=true"));
13516        assert!(admissions[0].contains("connection_id=777"));
13517    }
13518
13519    fn capability_manifest(
13520        module_id: &str,
13521        provides: &[&str],
13522        must_never_reach: &[&str],
13523    ) -> ModuleManifest {
13524        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13525        manifest.capabilities = Some(CapabilityDeclarations {
13526            provides: provides
13527                .iter()
13528                .map(|capability| (*capability).to_string())
13529                .collect(),
13530            requires: Vec::new(),
13531            must_never_reach: must_never_reach
13532                .iter()
13533                .map(|capability| (*capability).to_string())
13534                .collect(),
13535        });
13536        manifest
13537    }
13538
13539    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
13540        Frame::build(
13541            FrameType::Hello,
13542            control_flags(),
13543            0,
13544            0,
13545            corr,
13546            serde_json::to_vec(&ModuleHelloBody {
13547                protocol_ver: manifest.protocol_ver,
13548                manifest,
13549                control_ops: None,
13550                launch_nonce: None,
13551            })
13552            .expect("capability test HELLO serializes"),
13553        )
13554        .expect("capability test HELLO frame builds")
13555    }
13556
13557    fn catalog_update_with_capabilities_frame(
13558        corr: u64,
13559        capabilities: CapabilityDeclarations,
13560    ) -> Frame {
13561        Frame::build(
13562            FrameType::Request,
13563            control_flags(),
13564            0,
13565            0,
13566            corr,
13567            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13568                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
13569                capabilities: Some(capabilities),
13570                ready: None,
13571            })
13572            .expect("capability catalog.update serializes"),
13573        )
13574        .expect("capability catalog.update frame builds")
13575    }
13576
13577    async fn register_capability_manifest(
13578        handler: &ControlHandler,
13579        ctx: &RouteCtx,
13580        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13581        manifest: ModuleManifest,
13582        corr: u64,
13583    ) {
13584        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
13585    }
13586
13587    async fn open_route_for_capability_test(
13588        handler: &ControlHandler,
13589        target_ctx: &RouteCtx,
13590        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13591        client_connection_id: u64,
13592        corr: u64,
13593        target_module_id: &str,
13594        consumer_identity: Option<ConsumerIdentity>,
13595    ) -> (
13596        mpsc::Receiver<crate::router::OutboundFrame>,
13597        ModuleControlRequest,
13598    ) {
13599        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
13600        let route_handler = handler.clone();
13601        let target_module_id = target_module_id.to_string();
13602        let route_task = tokio::spawn(async move {
13603            route_handler
13604                .handle_control_frame(
13605                    &client_ctx,
13606                    route_open_frame_with_admission_facts(
13607                        corr,
13608                        &target_module_id,
13609                        unique_project_root("admission-facts"),
13610                        consumer_identity,
13611                        None,
13612                    ),
13613                )
13614                .await
13615                .expect("capability test route.open succeeds")
13616        });
13617        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
13618            .await
13619            .expect("capability test route.open must reach route.bind")
13620            .expect("target control receiver stays open");
13621        let bind_request: ModuleControlRequest =
13622            serde_json::from_slice(&bind.body).expect("route.bind decodes");
13623        handler
13624            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
13625            .await
13626            .expect("capability test route.bind ACK succeeds");
13627        assert!(route_task.await.expect("route.open task joins").is_empty());
13628        let opened = client_rx
13629            .recv()
13630            .await
13631            .expect("successful route.open publishes a response");
13632        assert!(matches!(
13633            serde_json::from_slice::<ClientControlResponse>(&opened.body),
13634            Ok(ClientControlResponse::RouteOpen { .. })
13635        ));
13636        (client_rx, bind_request)
13637    }
13638
13639    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
13640        assert_eq!(frame.header.ty, FrameType::Push);
13641        assert_eq!(frame.header.channel, 0);
13642        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
13643            .expect("route.closed control push decodes");
13644        let ClientControlPush::RouteClosed { channels, .. } = &push else {
13645            panic!("expected route.closed");
13646        };
13647        assert_eq!(channels.len(), 1, "exactly one violating route closed");
13648        let channels = channels.clone();
13649        assert_eq!(
13650            push,
13651            ClientControlPush::RouteClosed {
13652                module_id: target_module_id.to_string(),
13653                channels,
13654                reason: RouteCloseReason::CapabilityDenied,
13655                drained: false,
13656                abandoned: 0,
13657                excluded_subscriptions: 0,
13658                terminal: Some(false),
13659            }
13660        );
13661    }
13662
13663    #[tokio::test]
13664    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
13665        let registry = Arc::new(Registry::default());
13666        let forwarding = Arc::new(ForwardingTable::default());
13667        let supervisor = SupervisorHandle::new();
13668        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13669        let handler =
13670            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13671                .with_supervisor(supervisor);
13672        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
13673        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
13674        register_capability_manifest(
13675            &handler,
13676            &target_ctx,
13677            &mut target_rx,
13678            capability_manifest("target", &["credentials-provider/v1"], &[]),
13679            1,
13680        )
13681        .await;
13682        register_capability_manifest(
13683            &handler,
13684            &opener_ctx,
13685            &mut opener_rx,
13686            capability_manifest("opener", &[], &["credentials-provider/v1"]),
13687            2,
13688        )
13689        .await;
13690
13691        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
13692        let replies = handler
13693            .handle_control_frame(
13694                &client_ctx,
13695                route_open_frame_with_admission_facts(
13696                    3,
13697                    "target",
13698                    unique_project_root("admission-facts"),
13699                    Some(ConsumerIdentity {
13700                        module_id: "opener".to_string(),
13701                        launch_nonce: "opener-nonce".to_string(),
13702                    }),
13703                    None,
13704                ),
13705            )
13706            .await
13707            .expect("denied route.open returns a typed frame");
13708        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13709        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13710        assert!(
13711            target_rx.try_recv().is_err(),
13712            "forbidden route.open must not relay route.bind"
13713        );
13714    }
13715
13716    #[tokio::test]
13717    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
13718        let registry = Arc::new(Registry::default());
13719        let forwarding = Arc::new(ForwardingTable::default());
13720        let supervisor = SupervisorHandle::new();
13721        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13722        let handler =
13723            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13724                .with_supervisor(supervisor);
13725        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
13726        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
13727        register_capability_manifest(
13728            &handler,
13729            &target_ctx,
13730            &mut target_rx,
13731            capability_manifest("target", &["credentials-provider/v1"], &[]),
13732            1,
13733        )
13734        .await;
13735        register_capability_manifest(
13736            &handler,
13737            &old_opener_ctx,
13738            &mut old_opener_rx,
13739            capability_manifest("opener", &[], &[]),
13740            2,
13741        )
13742        .await;
13743        let (mut client_rx, _) = open_route_for_capability_test(
13744            &handler,
13745            &target_ctx,
13746            &mut target_rx,
13747            712,
13748            3,
13749            "target",
13750            Some(ConsumerIdentity {
13751                module_id: "opener".to_string(),
13752                launch_nonce: "opener-nonce".to_string(),
13753            }),
13754        )
13755        .await;
13756        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13757
13758        handler
13759            .cleanup_connection(old_opener_ctx.connection_id)
13760            .expect("old opener registration cleans up");
13761        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
13762        register_capability_manifest(
13763            &handler,
13764            &new_opener_ctx,
13765            &mut new_opener_rx,
13766            capability_manifest("opener", &[], &["credentials-provider/v1"]),
13767            4,
13768        )
13769        .await;
13770
13771        assert_capability_denied_push(
13772            client_rx
13773                .try_recv()
13774                .expect("HELLO deny addition must emit route.closed")
13775                .frame,
13776            "target",
13777        );
13778        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13779        assert!(matches!(
13780            target_rx.try_recv(),
13781            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13782        ));
13783    }
13784
13785    #[tokio::test]
13786    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
13787        let registry = Arc::new(Registry::default());
13788        let forwarding = Arc::new(ForwardingTable::default());
13789        let supervisor = SupervisorHandle::new();
13790        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13791        let handler =
13792            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13793                .with_supervisor(supervisor);
13794        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
13795        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
13796        register_capability_manifest(
13797            &handler,
13798            &target_ctx,
13799            &mut target_rx,
13800            capability_manifest("target", &[], &[]),
13801            1,
13802        )
13803        .await;
13804        register_capability_manifest(
13805            &handler,
13806            &opener_ctx,
13807            &mut opener_rx,
13808            capability_manifest("opener", &[], &["credentials-provider/v1"]),
13809            2,
13810        )
13811        .await;
13812        let (mut client_rx, _) = open_route_for_capability_test(
13813            &handler,
13814            &target_ctx,
13815            &mut target_rx,
13816            722,
13817            3,
13818            "target",
13819            Some(ConsumerIdentity {
13820                module_id: "opener".to_string(),
13821                launch_nonce: "opener-nonce".to_string(),
13822            }),
13823        )
13824        .await;
13825        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13826
13827        let replies = handler
13828            .handle_control_frame(
13829                &target_ctx,
13830                catalog_update_with_capabilities_frame(
13831                    4,
13832                    CapabilityDeclarations {
13833                        provides: vec!["credentials-provider/v1".to_string()],
13834                        requires: Vec::new(),
13835                        must_never_reach: Vec::new(),
13836                    },
13837                ),
13838            )
13839            .await
13840            .expect("claim catalog.update succeeds");
13841        assert!(matches!(
13842            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
13843            Ok(ModuleControlResponseToModule::CatalogUpdate {})
13844        ));
13845        assert_capability_denied_push(
13846            client_rx
13847                .try_recv()
13848                .expect("claim addition must emit route.closed")
13849                .frame,
13850            "target",
13851        );
13852        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13853        assert!(matches!(
13854            target_rx.try_recv(),
13855            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13856        ));
13857    }
13858
13859    #[tokio::test]
13860    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
13861        let registry = Arc::new(Registry::default());
13862        let forwarding = Arc::new(ForwardingTable::default());
13863        let supervisor = SupervisorHandle::new();
13864        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13865        let handler =
13866            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13867                .with_supervisor(supervisor);
13868        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
13869        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
13870        register_capability_manifest(
13871            &handler,
13872            &target_ctx,
13873            &mut target_rx,
13874            capability_manifest("target", &["credentials-provider/v1"], &[]),
13875            1,
13876        )
13877        .await;
13878        register_capability_manifest(
13879            &handler,
13880            &opener_ctx,
13881            &mut opener_rx,
13882            capability_manifest("opener", &[], &[]),
13883            2,
13884        )
13885        .await;
13886        let (mut client_rx, _) = open_route_for_capability_test(
13887            &handler,
13888            &target_ctx,
13889            &mut target_rx,
13890            732,
13891            3,
13892            "target",
13893            Some(ConsumerIdentity {
13894                module_id: "opener".to_string(),
13895                launch_nonce: "opener-nonce".to_string(),
13896            }),
13897        )
13898        .await;
13899        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13900
13901        handler
13902            .handle_control_frame(
13903                &target_ctx,
13904                catalog_update_with_capabilities_frame(
13905                    4,
13906                    CapabilityDeclarations {
13907                        provides: Vec::new(),
13908                        requires: Vec::new(),
13909                        must_never_reach: Vec::new(),
13910                    },
13911                ),
13912            )
13913            .await
13914            .expect("claim removal catalog.update succeeds");
13915        assert_eq!(
13916            forwarding.active_binding_count().unwrap(),
13917            1,
13918            "removing an attested target claim must leave the route census unchanged"
13919        );
13920        assert!(
13921            client_rx.try_recv().is_err(),
13922            "claim removal must not emit route.closed capability_denied"
13923        );
13924        assert!(
13925            target_rx.try_recv().is_err(),
13926            "claim removal must not send the target a route GOODBYE"
13927        );
13928    }
13929
13930    /// A direct client may open a route to a denied capability provider; this
13931    /// policy applies only to attested supervised module origins, not to direct clients.
13932    #[tokio::test]
13933    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
13934        let registry = Arc::new(Registry::default());
13935        let forwarding = Arc::new(ForwardingTable::default());
13936        let supervisor = SupervisorHandle::new();
13937        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13938        let handler =
13939            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13940                .with_supervisor(supervisor);
13941        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
13942        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
13943        register_capability_manifest(
13944            &handler,
13945            &target_ctx,
13946            &mut target_rx,
13947            capability_manifest("target", &["credentials-provider/v1"], &[]),
13948            1,
13949        )
13950        .await;
13951        register_capability_manifest(
13952            &handler,
13953            &opener_ctx,
13954            &mut opener_rx,
13955            capability_manifest("opener", &[], &["credentials-provider/v1"]),
13956            2,
13957        )
13958        .await;
13959
13960        let (_client_rx, bind) = open_route_for_capability_test(
13961            &handler,
13962            &target_ctx,
13963            &mut target_rx,
13964            742,
13965            3,
13966            "target",
13967            None,
13968        )
13969        .await;
13970        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
13971            panic!("direct scope-honesty route must bind");
13972        };
13973        assert_eq!(principal, Some(Principal::Direct));
13974        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13975    }
13976
13977    /// A module that denies a capability receives no self-route exemption when it
13978    /// also attestedly provides that capability.
13979    #[tokio::test]
13980    async fn must_never_reach_self_route_is_capability_forbidden() {
13981        let registry = Arc::new(Registry::default());
13982        let forwarding = Arc::new(ForwardingTable::default());
13983        let supervisor = SupervisorHandle::new();
13984        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
13985        let handler =
13986            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13987                .with_supervisor(supervisor);
13988        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
13989        register_capability_manifest(
13990            &handler,
13991            &self_ctx,
13992            &mut self_rx,
13993            capability_manifest(
13994                "self-provider",
13995                &["credentials-provider/v1"],
13996                &["credentials-provider/v1"],
13997            ),
13998            1,
13999        )
14000        .await;
14001
14002        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
14003        let replies = handler
14004            .handle_control_frame(
14005                &client_ctx,
14006                route_open_frame_with_admission_facts(
14007                    2,
14008                    "self-provider",
14009                    unique_project_root("admission-facts"),
14010                    Some(ConsumerIdentity {
14011                        module_id: "self-provider".to_string(),
14012                        launch_nonce: "self-nonce".to_string(),
14013                    }),
14014                    None,
14015                ),
14016            )
14017            .await
14018            .expect("self-route refusal returns a typed frame");
14019        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14020        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14021        assert!(
14022            self_rx.try_recv().is_err(),
14023            "self denial must not relay route.bind"
14024        );
14025    }
14026
14027    #[test]
14028    fn unsupported_channel_zero_frame_returns_error() {
14029        let handler = ControlHandler::default();
14030        let request = Frame::build(
14031            FrameType::Request,
14032            control_flags(),
14033            0,
14034            0,
14035            21,
14036            b"opaque".to_vec(),
14037        )
14038        .unwrap();
14039
14040        let response = handler
14041            .handle_control(ConnectionId::new(1), request)
14042            .unwrap();
14043
14044        assert_eq!(response[0].header.ty, FrameType::Error);
14045        assert_eq!(
14046            parse_error(&response[0])["code"],
14047            "unsupported_control_frame"
14048        );
14049    }
14050
14051    /// Blue/green swap at the control-plane boundary. The supervisor that opens
14052    /// a swap is not wired yet, so the candidate is registered here directly
14053    /// into the registry and forwarding candidate slots, the way the swap's
14054    /// HELLO admission will.
14055    mod swap {
14056        use super::*;
14057
14058        const INCUMBENT: ConnectionId = ConnectionId::new(30);
14059        const CANDIDATE: ConnectionId = ConnectionId::new(40);
14060
14061        struct Swap {
14062            registry: Arc<Registry>,
14063            forwarding: Arc<ForwardingTable>,
14064            handler: ControlHandler,
14065            incumbent_ctx: RouteCtx,
14066            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14067            candidate_ctx: RouteCtx,
14068            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14069        }
14070
14071        async fn swap_with_incumbent() -> Swap {
14072            let registry = Arc::new(Registry::default());
14073            let forwarding = Arc::new(ForwardingTable::default());
14074            let handler =
14075                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14076            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14077            hello_via_sink(
14078                &handler,
14079                &incumbent_ctx,
14080                &mut incumbent_rx,
14081                hello_frame("aft", PROTOCOL_VERSION, 7),
14082            )
14083            .await;
14084            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14085            Swap {
14086                registry,
14087                forwarding,
14088                handler,
14089                incumbent_ctx,
14090                incumbent_rx,
14091                candidate_ctx,
14092                candidate_rx,
14093            }
14094        }
14095
14096        fn register_candidate(swap: &Swap, ready: Option<bool>) {
14097            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14098            candidate_manifest.ready = ready;
14099            let registration = swap
14100                .registry
14101                .register_candidate_with_control_ops(
14102                    candidate_manifest,
14103                    PROTOCOL_VERSION,
14104                    CANDIDATE,
14105                    module_baseline_control_ops(),
14106                )
14107                .unwrap();
14108            swap.forwarding
14109                .register_candidate_module_connection(
14110                    CANDIDATE,
14111                    "aft".to_string(),
14112                    PROTOCOL_VERSION,
14113                    manifest_concurrency(&registration.manifest),
14114                    swap.candidate_ctx.egress.clone(),
14115                )
14116                .unwrap();
14117        }
14118
14119        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
14120            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
14121            swap.registry.promote_candidate("aft").unwrap().unwrap();
14122            cutover.incumbent.unwrap()
14123        }
14124
14125        /// A swap candidate's HELLO reserves its candidate registry entry
14126        /// before installing its candidate forwarding endpoint. The swap
14127        /// supervisor promotes a ready candidate as soon as it reads it as
14128        /// registered, and promotion cuts forwarding over to the candidate
14129        /// endpoint, so the lookup the supervisor reads
14130        /// (`Registry::published_registration`) must not return the candidate
14131        /// until that endpoint exists. Otherwise the cutover finds no candidate
14132        /// endpoint and the swap fails as if the candidate had disconnected.
14133        #[tokio::test]
14134        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
14135            let mut swap = swap_with_incumbent().await;
14136            let hold = Arc::new(std::sync::Barrier::new(2));
14137            let handler = swap
14138                .handler
14139                .clone()
14140                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14141            handler
14142                .supervisor
14143                .open_swap("aft", "candidate-nonce".to_string());
14144            let hello = {
14145                let handler = handler.clone();
14146                let candidate_ctx = swap.candidate_ctx.clone();
14147                std::thread::spawn(move || {
14148                    handler
14149                        .handle_hello(
14150                            candidate_ctx.connection_id,
14151                            Some(candidate_ctx.egress.clone()),
14152                            hello_frame_with_nonce(
14153                                "aft",
14154                                PROTOCOL_VERSION,
14155                                8,
14156                                Some("candidate-nonce"),
14157                            ),
14158                        )
14159                        .unwrap()
14160                })
14161            };
14162
14163            hold.wait();
14164            // Parked: the candidate slot is claimed but not published, and
14165            // forwarding has no candidate endpoint.
14166            assert!(swap
14167                .registry
14168                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
14169                .unwrap()
14170                .is_some());
14171            assert!(swap
14172                .registry
14173                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14174                .unwrap()
14175                .is_none());
14176            assert!(swap
14177                .forwarding
14178                .module_endpoint_for_connection(CANDIDATE)
14179                .unwrap()
14180                .is_none());
14181
14182            hold.wait();
14183            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
14184            assert_eq!(
14185                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
14186                FrameType::HelloAck
14187            );
14188            let published = swap
14189                .registry
14190                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14191                .unwrap()
14192                .expect("candidate is published once its HELLO completes");
14193            assert_eq!(published.connection_id, CANDIDATE);
14194            assert!(swap
14195                .forwarding
14196                .module_endpoint_for_connection(CANDIDATE)
14197                .unwrap()
14198                .is_some());
14199        }
14200
14201        fn keyed_total(counters: &Value, key: &str) -> u64 {
14202            counters[key]
14203                .as_object()
14204                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
14205                .unwrap_or(0)
14206        }
14207
14208        #[tokio::test]
14209        async fn replacement_logs_old_end_and_new_admission_once() {
14210            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14211            let mut swap = swap_with_incumbent().await;
14212            swap.handler
14213                .supervisor
14214                .open_swap("aft", "candidate-nonce".to_string());
14215            hello_via_sink(
14216                &swap.handler,
14217                &swap.candidate_ctx,
14218                &mut swap.candidate_rx,
14219                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
14220            )
14221            .await;
14222            cutover(&swap);
14223            swap.handler.cleanup_connection(INCUMBENT).unwrap();
14224
14225            let captured = crate::router::test_log::captured_logs(&logs);
14226            println!("captured replacement registry logs:\n{captured}");
14227            let new_admissions: Vec<_> = captured
14228                .lines()
14229                .filter(|line| {
14230                    line.contains("connection_id=40")
14231                        && line.contains("swap candidate registered; not routable until cutover")
14232                })
14233                .collect();
14234            assert_eq!(
14235                new_admissions.len(),
14236                1,
14237                "unexpected admission logs: {captured}"
14238            );
14239            assert!(new_admissions[0].contains("module_id=aft"));
14240            assert!(new_admissions[0].contains("connection_id=40"));
14241            assert!(
14242                new_admissions[0].contains("swap candidate registered; not routable until cutover")
14243            );
14244            assert!(new_admissions[0].contains("ready=true"));
14245
14246            let old_admissions: Vec<_> = captured
14247                .lines()
14248                .filter(|line| {
14249                    line.contains("module registered module_id=aft ")
14250                        && line.contains("connection_id=30")
14251                })
14252                .collect();
14253            assert_eq!(
14254                old_admissions.len(),
14255                1,
14256                "unexpected incumbent admission: {captured}"
14257            );
14258
14259            let old_ends: Vec<_> = captured
14260                .lines()
14261                .filter(|line| {
14262                    line.contains("connection_id=30") && line.contains("module registration ended")
14263                })
14264                .collect();
14265            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
14266            assert!(old_ends[0].contains("module_id=aft"));
14267            assert!(old_ends[0].contains("connection_id=30"));
14268            assert!(old_ends[0].contains("reason=replaced"));
14269            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
14270            assert_eq!(
14271                captured
14272                    .lines()
14273                    .filter(|line| line.contains("module registration promoted"))
14274                    .count(),
14275                1,
14276                "unexpected promotion logs: {captured}"
14277            );
14278        }
14279
14280        /// An ack from the incumbent for a bind it was sent before cutover,
14281        /// arriving before the incumbent is drained. The incumbent is the live
14282        /// connection carrying every other client's routes, so the ack must
14283        /// not end it: the waiting client is told to retry, the reservation is
14284        /// given back, and the incumbent is told to drop just that binding.
14285        #[tokio::test]
14286        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14287            let mut swap = swap_with_incumbent().await;
14288            let handler = swap.handler.clone();
14289
14290            // A co-tenant route, bound on the incumbent before the swap.
14291            let cotenant = ConnectionId::new(31);
14292            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14293            let (cotenant_task, cotenant_bind) = relay_route_open(
14294                &handler,
14295                cotenant,
14296                &cotenant_ctx.egress,
14297                &mut swap.incumbent_rx,
14298                100,
14299                "aft",
14300                "swap-cotenant",
14301            )
14302            .await;
14303            handler
14304                .handle_control_frame(
14305                    &swap.incumbent_ctx,
14306                    route_bind_ack(cotenant_bind.header.corr),
14307                )
14308                .await
14309                .unwrap();
14310            assert!(cotenant_task.await.unwrap().is_empty());
14311            let (cotenant_channel, cotenant_epoch) =
14312                published_route(&cotenant_rx.recv().await.unwrap());
14313
14314            // A second route.open, relayed to the incumbent and not yet acked.
14315            let caller = ConnectionId::new(32);
14316            let (caller_ctx, mut caller_rx) = route_ctx(caller);
14317            let (caller_task, caller_bind) = relay_route_open(
14318                &handler,
14319                caller,
14320                &caller_ctx.egress,
14321                &mut swap.incumbent_rx,
14322                101,
14323                "aft",
14324                "swap-caller",
14325            )
14326            .await;
14327            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14328
14329            register_candidate(&swap, None);
14330            cutover(&swap);
14331
14332            // The incumbent acks after promotion and before any drain.
14333            let ack = handler
14334                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14335                .await;
14336            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14337            if module_loop_error.is_some() {
14338                // What the connection loop does with an untranslated router
14339                // error: end the connection, releasing every route on it.
14340                handler.cleanup_connection(INCUMBENT).unwrap();
14341            }
14342
14343            // 1. The incumbent's other routes survive.
14344            assert!(
14345                cotenant_rx.try_recv().is_err(),
14346                "the co-tenant route on the incumbent was torn down by one late ack: \
14347                 {module_loop_error:?}"
14348            );
14349            assert!(matches!(
14350                swap.forwarding
14351                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14352                    .unwrap(),
14353                DataRoute::Client(DataRouteState::Bound(_))
14354            ));
14355            assert_eq!(module_loop_error, None);
14356            assert!(swap
14357                .registry
14358                .get_module_by_connection(INCUMBENT)
14359                .unwrap()
14360                .is_some());
14361
14362            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
14363            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14364                .await
14365                .expect("the incumbent is told to drop the abandoned binding")
14366                .unwrap()
14367                .frame;
14368            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14369            assert_eq!(goodbye.header.channel, abandoned_channel);
14370            assert_eq!(goodbye.header.epoch, abandoned_epoch);
14371            assert!(swap.incumbent_rx.try_recv().is_err());
14372
14373            // 3. The waiting client gets a retryable refusal and no route.
14374            let response = caller_task.await.unwrap();
14375            assert_eq!(response.len(), 1);
14376            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14377            assert!(caller_rx.try_recv().is_err());
14378
14379            // 4. The reservation pair is given back, and the pending bind
14380            //    settled exactly once: one accepted open (the co-tenant) and one
14381            //    refused open (the caller), nothing counted twice.
14382            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14383            let counters = handler.counters().snapshot();
14384            assert_eq!(
14385                keyed_total(&counters, "route_open_accepted_by_principal"),
14386                1
14387            );
14388            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14389            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14390        }
14391
14392        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
14393        /// id would resolve to the promoted candidate and every new route.open
14394        /// would be refused as reloading, leaving neither process routable.
14395        #[tokio::test]
14396        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14397            let mut swap = swap_with_incumbent().await;
14398            register_candidate(&swap, None);
14399            let incumbent = cutover(&swap);
14400            swap.forwarding
14401                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14402                .unwrap()
14403                .expect("the incumbent is still registered");
14404
14405            let client = ConnectionId::new(33);
14406            let (client_ctx, mut client_rx) = route_ctx(client);
14407            let route_handler = swap.handler.clone();
14408            let open_ctx = RouteCtx {
14409                connection_id: client,
14410                egress: client_ctx.egress.clone(),
14411            };
14412            let mut route_task = tokio::spawn(async move {
14413                route_handler
14414                    .handle_control_frame(
14415                        &open_ctx,
14416                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14417                    )
14418                    .await
14419                    .unwrap()
14420            });
14421            let bind = tokio::select! {
14422                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14423                response = &mut route_task => {
14424                    let response = response.unwrap();
14425                    panic!(
14426                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
14427                        parse_error(&response[0])["code"]
14428                    );
14429                }
14430            };
14431            swap.handler
14432                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14433                .await
14434                .unwrap();
14435            assert!(route_task.await.unwrap().is_empty());
14436            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14437            match swap
14438                .forwarding
14439                .lookup_data_route(client, channel, epoch)
14440                .unwrap()
14441            {
14442                DataRoute::Client(DataRouteState::Bound(route)) => {
14443                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14444                }
14445                other => panic!("expected a bound route on the candidate, got {other:?}"),
14446            }
14447            assert!(swap.incumbent_rx.try_recv().is_err());
14448        }
14449
14450        /// A candidate declares itself ready with `catalog.update` on its own
14451        /// connection. If the connection-keyed registry lookups searched only the
14452        /// active slot, this would answer `not_registered` and the candidate
14453        /// would never become ready.
14454        #[tokio::test]
14455        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14456            let swap = swap_with_incumbent().await;
14457            register_candidate(&swap, Some(false));
14458            let update = Frame::build(
14459                FrameType::Request,
14460                control_flags(),
14461                0,
14462                0,
14463                55,
14464                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14465                    provides: manifest("aft", PROTOCOL_VERSION).provides,
14466                    capabilities: None,
14467                    ready: Some(true),
14468                })
14469                .unwrap(),
14470            )
14471            .unwrap();
14472
14473            let replies = swap
14474                .handler
14475                .handle_control_frame(&swap.candidate_ctx, update)
14476                .await
14477                .unwrap();
14478
14479            assert_eq!(replies.len(), 1);
14480            assert_eq!(
14481                replies[0].header.ty,
14482                FrameType::Response,
14483                "candidate catalog.update was refused: {:?}",
14484                serde_json::from_slice::<Value>(&replies[0].body).ok()
14485            );
14486            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14487            assert_eq!(
14488                swap.registry
14489                    .get_module("aft")
14490                    .unwrap()
14491                    .unwrap()
14492                    .connection_id,
14493                INCUMBENT
14494            );
14495        }
14496    }
14497
14498    /// The HELLO gate while the supervisor has a swap open: only the nonce it
14499    /// minted for the candidate admits a second process, into the candidate
14500    /// slot, and that check runs ahead of the reserved-module gate.
14501    mod swap_admission {
14502        use super::*;
14503
14504        const INCUMBENT_NONCE: &str = "incumbent-nonce";
14505        const CANDIDATE_NONCE: &str = "candidate-nonce";
14506
14507        fn handler_with_incumbent(
14508            module_id: &str,
14509            reserved: bool,
14510        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14511            let registry = Arc::new(Registry::default());
14512            let supervisor = SupervisorHandle::new();
14513            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14514            if reserved {
14515                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14516            }
14517            let handler =
14518                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
14519            let incumbent = handler
14520                .handle_control(
14521                    ConnectionId::new(1),
14522                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14523                )
14524                .unwrap();
14525            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14526            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14527            (registry, supervisor, handler)
14528        }
14529
14530        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
14531        /// admits every nonce, so while a swap is open the swap gate is the only
14532        /// thing between a key-holder and the candidate slot. A nonce the
14533        /// supervisor did not mint, or none at all, is refused, and neither the
14534        /// incumbent's registration nor the candidate slot moves.
14535        #[test]
14536        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
14537            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
14538
14539            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
14540                let replies = handler
14541                    .handle_control(
14542                        ConnectionId::new(connection),
14543                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
14544                    )
14545                    .unwrap();
14546                assert_eq!(replies[0].header.ty, FrameType::Error);
14547                assert_eq!(
14548                    parse_error(&replies[0])["code"],
14549                    "swap_token_invalid",
14550                    "nonce {nonce:?}"
14551                );
14552            }
14553            assert!(registry.get_candidate("aft").unwrap().is_none());
14554            assert_eq!(
14555                registry.get_module("aft").unwrap().unwrap().connection_id,
14556                ConnectionId::new(1)
14557            );
14558
14559            // Control: the minted token is admitted, into the candidate slot,
14560            // and only once.
14561            let admitted = handler
14562                .handle_control(
14563                    ConnectionId::new(4),
14564                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
14565                )
14566                .unwrap();
14567            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
14568            assert_eq!(
14569                registry
14570                    .get_candidate("aft")
14571                    .unwrap()
14572                    .unwrap()
14573                    .connection_id,
14574                ConnectionId::new(4)
14575            );
14576            assert_eq!(
14577                registry.get_module("aft").unwrap().unwrap().connection_id,
14578                ConnectionId::new(1),
14579                "the candidate must not take the active slot"
14580            );
14581            let replayed = handler
14582                .handle_control(
14583                    ConnectionId::new(5),
14584                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
14585                )
14586                .unwrap();
14587            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
14588
14589            // The case only this gate covers: the incumbent has died mid-swap,
14590            // so its duplicate refusal is gone too, and without the gate a
14591            // key-holder would take the id's ACTIVE slot.
14592            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
14593            let squatter = handler
14594                .handle_control(
14595                    ConnectionId::new(6),
14596                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
14597                )
14598                .unwrap();
14599            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
14600            assert!(
14601                registry.get_module("aft").unwrap().is_none(),
14602                "a squatter took the active slot of an id being swapped"
14603            );
14604        }
14605
14606        /// Design mutation arm (iii). A reserved module's candidate presents a
14607        /// nonce the reserved gate has never seen (that gate holds the
14608        /// incumbent's), so the swap gate must run first or the candidate is
14609        /// refused `reserved_module` and a reserved module can never be swapped.
14610        #[test]
14611        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
14612            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
14613
14614            let replies = handler
14615                .handle_control(
14616                    ConnectionId::new(2),
14617                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14618                )
14619                .unwrap();
14620
14621            assert_eq!(
14622                replies[0].header.ty,
14623                FrameType::HelloAck,
14624                "reserved candidate refused: {:?}",
14625                serde_json::from_slice::<Value>(&replies[0].body).ok()
14626            );
14627            assert_eq!(
14628                registry
14629                    .get_candidate("vault")
14630                    .unwrap()
14631                    .unwrap()
14632                    .connection_id,
14633                ConnectionId::new(2)
14634            );
14635        }
14636
14637        /// With no swap open the gate is inert: the incumbent's reserved gate
14638        /// and duplicate refusal behave exactly as before.
14639        #[test]
14640        fn without_an_open_swap_the_ordinary_gates_decide() {
14641            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
14642            supervisor.close_swap("vault");
14643
14644            let candidate = handler
14645                .handle_control(
14646                    ConnectionId::new(2),
14647                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14648                )
14649                .unwrap();
14650            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
14651            let duplicate = handler
14652                .handle_control(
14653                    ConnectionId::new(3),
14654                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
14655                )
14656                .unwrap();
14657            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14658            assert!(registry.get_candidate("vault").unwrap().is_none());
14659        }
14660    }
14661
14662    /// `scope.sync` and `scope.describe` through the real control handler: who
14663    /// may sync is decided by the registration and launch nonce of the module
14664    /// connection, never by the request body.
14665    mod scopes {
14666        mod idle_tests {
14667            use super::*;
14668            include!("scope_idle_tests.rs");
14669        }
14670        mod route_open_timings {
14671            use super::*;
14672            include!("route_open_timing_tests.rs");
14673        }
14674        mod activity {
14675            include!("activity_scope_tests.rs");
14676        }
14677        mod delta {
14678            include!("scope_handler_delta_tests.rs");
14679        }
14680        mod call_modes {
14681            include!("scope_call_mode_tests.rs");
14682        }
14683
14684        use subc_protocol::scope::{
14685            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
14686            ScopeStatus,
14687        };
14688
14689        use super::*;
14690
14691        const OWNER: &str = "prefrontal-core";
14692
14693        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
14694            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
14695        }
14696
14697        async fn call(
14698            handler: &ControlHandler,
14699            ctx: &RouteCtx,
14700            request: &ModuleControlRequestFromModule,
14701        ) -> Frame {
14702            let body = serde_json::to_vec(request).unwrap();
14703            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
14704            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
14705            assert_eq!(replies.len(), 1, "{replies:?}");
14706            replies.pop().unwrap()
14707        }
14708
14709        async fn sync(
14710            handler: &ControlHandler,
14711            ctx: &RouteCtx,
14712            generation: u64,
14713            scopes: Vec<ScopeRecord>,
14714        ) -> Result<ModuleControlResponseToModule, String> {
14715            let reply = call(
14716                handler,
14717                ctx,
14718                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
14719            )
14720            .await;
14721            match reply.header.ty {
14722                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
14723                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
14724            }
14725        }
14726
14727        async fn describe(
14728            handler: &ControlHandler,
14729            ctx: &RouteCtx,
14730            owner: &str,
14731            scope_ref: &str,
14732        ) -> ModuleControlResponseToModule {
14733            let reply = call(
14734                handler,
14735                ctx,
14736                &ModuleControlRequestFromModule::ScopeDescribe {
14737                    owner: Principal::Reserved {
14738                        module_id: owner.to_string(),
14739                    },
14740                    scope_ref: scope_ref.to_string(),
14741                },
14742            )
14743            .await;
14744            assert_eq!(
14745                reply.header.ty,
14746                FrameType::Response,
14747                "{:?}",
14748                parse_error(&reply)
14749            );
14750            serde_json::from_slice(&reply.body).unwrap()
14751        }
14752
14753        #[tokio::test]
14754        async fn scope_describe_never_echoes_bind_proof_or_hash() {
14755            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
14756            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
14757            let mut rig = rig().await;
14758            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
14759                .await;
14760            let _bound = rig
14761                .bound(
14762                    Some(AFT),
14763                    PLEXUS,
14764                    Some(
14765                        rig.selector("s", Some(1))
14766                            .with_bind_proof(Some(PROOF.into())),
14767                    ),
14768                )
14769                .await;
14770            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
14771            let output = serde_json::to_string(&reply).unwrap();
14772            assert!(output.contains("\"status\":\"live\""));
14773            assert!(!output.contains(PROOF));
14774            assert!(!output.contains(HASH));
14775        }
14776
14777        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
14778        async fn module(
14779            handler: &ControlHandler,
14780            connection: u64,
14781            module_id: &str,
14782            nonce: Option<&str>,
14783        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14784            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
14785            hello_via_sink(
14786                handler,
14787                &ctx,
14788                &mut rx,
14789                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
14790            )
14791            .await;
14792            (ctx, rx)
14793        }
14794
14795        /// `direct` and every other client connection has no registration, so
14796        /// it can neither sync nor own a scope.
14797        #[tokio::test]
14798        async fn a_client_connection_cannot_sync_or_describe() {
14799            let handler = ControlHandler::new(Arc::new(Registry::default()));
14800            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
14801            for request in [
14802                ModuleControlRequestFromModule::ScopeSync {
14803                    generation: 1,
14804                    scopes: vec![head("s", 1)],
14805                },
14806                ModuleControlRequestFromModule::ScopeDescribe {
14807                    owner: Principal::Direct,
14808                    scope_ref: "s".to_string(),
14809                },
14810            ] {
14811                let reply = call(&handler, &ctx, &request).await;
14812                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
14813            }
14814            assert!(
14815                !handler
14816                    .scopes
14817                    .read()
14818                    .unwrap()
14819                    .describe(
14820                        &Principal::Reserved {
14821                            module_id: OWNER.to_string()
14822                        },
14823                        "s"
14824                    )
14825                    .owner_synced
14826            );
14827        }
14828
14829        /// A module the supervisor did not spawn registers without a launch
14830        /// nonce, so it is never an owner's current launch.
14831        #[tokio::test]
14832        async fn a_module_without_a_supervised_launch_cannot_sync() {
14833            let handler = ControlHandler::new(Arc::new(Registry::default()));
14834            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
14835            assert_eq!(
14836                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
14837                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14838            );
14839        }
14840
14841        #[tokio::test]
14842        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
14843            let supervisor = SupervisorHandle::new();
14844            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14845            let handler = ControlHandler::new(Arc::new(Registry::default()))
14846                .with_supervisor(supervisor.clone());
14847            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14848            sync(&handler, &incumbent, 1, vec![head("s", 1)])
14849                .await
14850                .expect("the current launch syncs");
14851
14852            // A swap candidate registers with the swap token and is refused
14853            // while the incumbent keeps syncing.
14854            supervisor.open_swap(OWNER, "n2".to_string());
14855            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
14856            assert_eq!(
14857                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14858                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14859            );
14860            sync(&handler, &incumbent, 2, vec![head("s", 1)])
14861                .await
14862                .expect("the serving owner syncs during the swap");
14863
14864            // The swap fails and is rolled back. The candidate never held sync
14865            // authority, and still cannot sync.
14866            supervisor.close_swap(OWNER);
14867            assert_eq!(
14868                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14869                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14870            );
14871            sync(&handler, &incumbent, 3, vec![head("s", 1)])
14872                .await
14873                .expect("the serving owner syncs after the rollback");
14874            handler.cleanup_connection(candidate.connection_id).unwrap();
14875
14876            // A swap that cuts over. Promotion records the candidate's nonce as
14877            // the module's spawn nonce, which is what `set_spawn_nonce` does
14878            // here; the promoted connection then takes authority at any
14879            // generation and the superseded incumbent is refused.
14880            supervisor.open_swap(OWNER, "n3".to_string());
14881            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
14882            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
14883            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
14884                .await
14885                .expect("the promoted launch takes authority");
14886            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14887                panic!("unexpected reply {reply:?}");
14888            };
14889            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
14890            assert_eq!(
14891                sync(&handler, &incumbent, 4, Vec::new()).await,
14892                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14893            );
14894        }
14895
14896        /// Authority dies with its connection: the cleanup path releases it,
14897        /// so the owner's next connection takes it at any generation.
14898        #[tokio::test]
14899        async fn closing_the_authority_connection_frees_sync_authority() {
14900            let supervisor = SupervisorHandle::new();
14901            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14902            let handler = ControlHandler::new(Arc::new(Registry::default()))
14903                .with_supervisor(supervisor.clone());
14904            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14905            sync(&handler, &first, 10, vec![head("s", 1)])
14906                .await
14907                .unwrap();
14908            handler.cleanup_connection(first.connection_id).unwrap();
14909
14910            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
14911            sync(&handler, &second, 1, vec![head("s", 1)])
14912                .await
14913                .expect("the next connection takes the released authority");
14914        }
14915
14916        #[tokio::test]
14917        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
14918            let supervisor = SupervisorHandle::new();
14919            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14920            let handler =
14921                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
14922            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
14923            sync(&handler, &first, 10, vec![head("s", 1)])
14924                .await
14925                .unwrap();
14926            handler
14927                .handle_control_frame(
14928                    &first,
14929                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
14930                )
14931                .await
14932                .unwrap();
14933            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
14934            sync(&handler, &second, 1, vec![head("s", 1)])
14935                .await
14936                .expect("GOODBYE releases authority even if the old socket remains open");
14937        }
14938
14939        #[tokio::test]
14940        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
14941            let registry = Arc::new(Registry::default());
14942            let supervisor_handle = SupervisorHandle::new();
14943            let supervisor =
14944                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
14945                    .with_handle(supervisor_handle.clone())
14946                    .with_daemon_incarnation("incarnation-7".to_string());
14947            // Configured with enabled: false, so the supervisor lists the
14948            // module without spawning a process for it.
14949            supervisor
14950                .supervise_configured(
14951                    ModuleSpec {
14952                        module_id: OWNER.to_string(),
14953                        program: PathBuf::from("/nonexistent/prefrontal-core"),
14954                        args: Vec::new(),
14955                        env: Vec::new(),
14956                        reserved: false,
14957                        reserved_prefixes: Vec::new(),
14958                        protocol: ModuleProtocol::Subc,
14959                        overlap: Default::default(),
14960                    },
14961                    false,
14962                )
14963                .unwrap();
14964            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
14965            let handler =
14966                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14967            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
14968
14969            // Configured but not yet synced: a reader waits for the owner.
14970            let ModuleControlResponseToModule::ScopeDescribe {
14971                status,
14972                daemon_incarnation,
14973                owner_synced,
14974                owner_configured,
14975                scope,
14976                ..
14977            } = describe(&handler, &reader, OWNER, "s").await
14978            else {
14979                panic!("not a describe reply");
14980            };
14981            assert_eq!(status, ScopeStatus::NotLive);
14982            assert_eq!(daemon_incarnation, "incarnation-7");
14983            assert!(!owner_synced);
14984            assert!(owner_configured);
14985            assert!(scope.is_none());
14986
14987            // Not a supervised module: the owner will never sync, and a reader
14988            // refuses rather than waits.
14989            let ModuleControlResponseToModule::ScopeDescribe {
14990                status,
14991                owner_configured,
14992                ..
14993            } = describe(&handler, &reader, "ghost", "s").await
14994            else {
14995                panic!("not a describe reply");
14996            };
14997            assert_eq!(status, ScopeStatus::NotLive);
14998            assert!(!owner_configured);
14999
15000            // Live, with the stamp fields and the computed owner_authorized.
15001            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
15002            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
15003            let ModuleControlResponseToModule::ScopeDescribe {
15004                status,
15005                scope_epoch,
15006                owner_synced,
15007                scope,
15008                ..
15009            } = describe(&handler, &reader, OWNER, "s").await
15010            else {
15011                panic!("not a describe reply");
15012            };
15013            assert_eq!(status, ScopeStatus::Live);
15014            assert_eq!(scope_epoch, Some(4));
15015            assert!(owner_synced);
15016            let stamp = scope.expect("a live scope carries its stamp");
15017            assert!(
15018                stamp.owner_authorized,
15019                "prefrontal-core is the default authority"
15020            );
15021            assert_eq!(stamp.kind, ScopeKind::Head);
15022        }
15023
15024        #[tokio::test]
15025        async fn scope_authority_owners_decides_owner_authorized() {
15026            let supervisor = SupervisorHandle::new();
15027            supervisor.set_spawn_nonce("broca", "b1".to_string());
15028            let handler = ControlHandler::new(Arc::new(Registry::default()))
15029                .with_supervisor(supervisor)
15030                .with_scope_authority_owners(vec!["broca".to_string()]);
15031            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
15032            let mut gated = head("s", 1);
15033            gated.attributes.agent_id = Some("agent".to_string());
15034            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
15035            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
15036                describe(&handler, &broca, "broca", "s").await
15037            else {
15038                panic!("not a describe reply");
15039            };
15040            assert!(scope.unwrap().owner_authorized);
15041        }
15042
15043        /// With route admission, the stamp, the commit re-check and drains in
15044        /// place, the feature is advertised: the module ops in HELLO_ACK, and
15045        /// `scopes/v1` in HELLO_ACK and `server.describe`.
15046        #[tokio::test]
15047        async fn scope_ops_and_the_scopes_capability_are_advertised() {
15048            let handler = ControlHandler::new(Arc::new(Registry::default()));
15049            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
15050            let ack = hello_via_sink(
15051                &handler,
15052                &ctx,
15053                &mut rx,
15054                hello_frame("m", PROTOCOL_VERSION, 1),
15055            )
15056            .await;
15057            let ack = parse_ack(&ack);
15058            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
15059                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
15060            }
15061            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
15062
15063            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
15064            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15065            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15066            let reply = handler
15067                .handle_control_frame(&client, frame)
15068                .await
15069                .unwrap()
15070                .pop()
15071                .unwrap();
15072            let ClientControlResponse::ServerDescribe { capabilities, .. } =
15073                serde_json::from_slice(&reply.body).unwrap()
15074            else {
15075                panic!("not a server.describe reply");
15076            };
15077            assert!(
15078                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
15079                "{capabilities:?}"
15080            );
15081        }
15082
15083        // ---- route admission, stamps, commit re-check and drains ----------
15084
15085        const PLEXUS: &str = "plexus";
15086        const OTHER: &str = "other";
15087        const AFT: &str = "aft";
15088        const BROCA: &str = "broca";
15089        const MAGIC: &str = "magic-context";
15090
15091        fn nonce(module_id: &str) -> String {
15092            format!("nonce-{module_id}")
15093        }
15094
15095        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15096            let (tx, rx) = mpsc::channel(64);
15097            (
15098                RouteCtx {
15099                    connection_id: ConnectionId::new(connection),
15100                    egress: FrameSink::new(tx),
15101                },
15102                rx,
15103            )
15104        }
15105
15106        /// A daemon with a configured owner (prefrontal-core) registered on its
15107        /// own module connection, two routable targets (plexus, other), and
15108        /// launch nonces minted for the modules that open routes as carriers.
15109        struct Rig {
15110            handler: ControlHandler,
15111            forwarding: Arc<ForwardingTable>,
15112            owner: RouteCtx,
15113            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15114            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
15115            generation: u64,
15116            next_connection: u64,
15117            _supervisor: Supervisor,
15118        }
15119
15120        async fn rig() -> Rig {
15121            rig_with_flow_support(true).await
15122        }
15123
15124        async fn rig_with_flow_support(flow_support: bool) -> Rig {
15125            rig_with_scope_capabilities(flow_support, false).await
15126        }
15127
15128        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
15129            let mut provides = Vec::new();
15130            if flow_support {
15131                provides.push("flow-scopes/v1");
15132            }
15133            if run_support {
15134                provides.push("agent-run-scopes/v1");
15135            }
15136            rig_with_target_capabilities(&provides).await
15137        }
15138
15139        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
15140            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
15141            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15142            // Decoder versions alone cannot admit scoped calls: only a capability
15143            // declaration promises the required run or flow behaviour.
15144            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15145            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
15146            Frame::build(
15147                FrameType::Hello,
15148                control_flags(),
15149                0,
15150                0,
15151                connection,
15152                serde_json::to_vec(&body).unwrap(),
15153            )
15154            .unwrap()
15155        }
15156
15157        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
15158            let registry = Arc::new(Registry::default());
15159            let forwarding = Arc::new(ForwardingTable::default());
15160            let supervisor_handle = SupervisorHandle::new();
15161            let supervisor =
15162                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
15163                    .with_handle(supervisor_handle.clone());
15164            supervisor
15165                .supervise_configured(
15166                    ModuleSpec {
15167                        module_id: OWNER.to_string(),
15168                        program: PathBuf::from("/nonexistent/prefrontal-core"),
15169                        args: Vec::new(),
15170                        env: Vec::new(),
15171                        reserved: false,
15172                        reserved_prefixes: Vec::new(),
15173                        protocol: ModuleProtocol::Subc,
15174                        overlap: Default::default(),
15175                    },
15176                    false,
15177                )
15178                .unwrap();
15179            for module_id in [OWNER, AFT, BROCA, MAGIC] {
15180                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
15181            }
15182            let handler =
15183                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15184                    .with_supervisor(supervisor_handle);
15185            let (owner, mut owner_rx) = wide_ctx(1);
15186            hello_via_sink(
15187                &handler,
15188                &owner,
15189                &mut owner_rx,
15190                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
15191            )
15192            .await;
15193            let mut modules = BTreeMap::new();
15194            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
15195                let (ctx, mut rx) = wide_ctx(connection);
15196                let hello = hello_with_capabilities(module_id, connection, provides);
15197                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
15198                modules.insert(module_id.to_string(), (ctx, rx));
15199            }
15200            Rig {
15201                handler,
15202                forwarding,
15203                owner,
15204                _owner_rx: owner_rx,
15205                modules,
15206                generation: 0,
15207                next_connection: 100,
15208                _supervisor: supervisor,
15209            }
15210        }
15211
15212        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
15213            ScopeCarrier::new(Principal::Reserved {
15214                module_id: module_id.to_string(),
15215            })
15216            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
15217        }
15218
15219        /// The scope most tests open under: aft carries to any module, broca
15220        /// only to plexus and other, and the owner delegates as agent-1.
15221        fn session(scope_epoch: u64) -> ScopeRecord {
15222            let mut record = head("s", scope_epoch);
15223            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
15224            record.attributes.agent_id = Some("agent-1".to_string());
15225            record.attributes.delegates = true;
15226            record
15227        }
15228
15229        impl Rig {
15230            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
15231                self.generation += 1;
15232                sync(&self.handler, &self.owner, self.generation, scopes)
15233                    .await
15234                    .expect("the owner's sync is accepted");
15235            }
15236
15237            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15238                ScopeSelector::new(
15239                    Principal::Reserved {
15240                        module_id: OWNER.to_string(),
15241                    },
15242                    scope_ref.to_string(),
15243                    scope_epoch,
15244                )
15245            }
15246
15247            fn open_frame(
15248                &mut self,
15249                opener: Option<&str>,
15250                target: &str,
15251                scope: Option<ScopeSelector>,
15252            ) -> (
15253                RouteCtx,
15254                mpsc::Receiver<crate::router::OutboundFrame>,
15255                Frame,
15256            ) {
15257                self.next_connection += 1;
15258                let (ctx, rx) = wide_ctx(self.next_connection);
15259                let root = unique_project_root("scoped-open");
15260                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15261                    target: RouteTarget::ToolProvider {
15262                        module_id: target.to_string(),
15263                    },
15264                    identity: BindIdentity::new(
15265                        root.path().to_path_buf(),
15266                        "unit".to_string(),
15267                        "session".to_string(),
15268                    ),
15269                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
15270                        module_id: module_id.to_string(),
15271                        launch_nonce: nonce(module_id),
15272                    }),
15273                    consumer_capabilities: None,
15274                    role_versions: None,
15275                    admission_facts: None,
15276                    scope,
15277                })
15278                .unwrap();
15279                let frame = Frame::build(
15280                    FrameType::Request,
15281                    control_flags(),
15282                    0,
15283                    0,
15284                    self.next_connection,
15285                    body,
15286                )
15287                .unwrap();
15288                (ctx, rx, frame)
15289            }
15290
15291            /// Open and expect a refusal before anything is relayed.
15292            async fn refused(
15293                &mut self,
15294                opener: Option<&str>,
15295                target: &str,
15296                scope: Option<ScopeSelector>,
15297            ) -> String {
15298                self.refusal_body(opener, target, scope).await["code"]
15299                    .as_str()
15300                    .unwrap()
15301                    .to_string()
15302            }
15303
15304            async fn refusal_body(
15305                &mut self,
15306                opener: Option<&str>,
15307                target: &str,
15308                scope: Option<ScopeSelector>,
15309            ) -> Value {
15310                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15311                let replies = tokio::time::timeout(
15312                    Duration::from_secs(2),
15313                    self.handler.handle_control_frame(&ctx, frame),
15314                )
15315                .await
15316                .expect("the open must be refused before waiting for a bind ack")
15317                .unwrap();
15318                assert_eq!(replies.len(), 1, "{replies:?}");
15319                assert_eq!(replies[0].header.ty, FrameType::Error);
15320                let (_, module_rx) = self.modules.get_mut(target).unwrap();
15321                assert!(
15322                    module_rx.try_recv().is_err(),
15323                    "a refused open relays nothing"
15324                );
15325                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15326                parse_error(&replies[0])
15327            }
15328
15329            /// Start an open and return its task and the bind the target got.
15330            async fn relayed(
15331                &mut self,
15332                opener: Option<&str>,
15333                target: &str,
15334                scope: Option<ScopeSelector>,
15335            ) -> Relayed {
15336                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15337                let handler = self.handler.clone();
15338                let task_ctx = ctx.clone();
15339                let task = tokio::spawn(async move {
15340                    handler
15341                        .handle_control_frame(&task_ctx, frame)
15342                        .await
15343                        .unwrap()
15344                });
15345                let (_, module_rx) = self.modules.get_mut(target).unwrap();
15346                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15347                    .await
15348                    .expect("the target receives the relayed route.bind")
15349                    .unwrap()
15350                    .frame;
15351                Relayed {
15352                    target: target.to_string(),
15353                    client: ctx,
15354                    client_rx: rx,
15355                    task,
15356                    bind,
15357                }
15358            }
15359
15360            async fn ack(&self, relayed: &Relayed) {
15361                let (module, _) = &self.modules[&relayed.target];
15362                self.handler
15363                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15364                    .await
15365                    .unwrap();
15366            }
15367
15368            /// Open, ack and return the bound route.
15369            async fn bound(
15370                &mut self,
15371                opener: Option<&str>,
15372                target: &str,
15373                scope: Option<ScopeSelector>,
15374            ) -> Bound {
15375                let relayed = self.relayed(opener, target, scope).await;
15376                self.ack(&relayed).await;
15377                let Relayed {
15378                    target,
15379                    client,
15380                    mut client_rx,
15381                    task,
15382                    bind,
15383                } = relayed;
15384                assert!(
15385                    task.await.unwrap().is_empty(),
15386                    "the open is answered by commit"
15387                );
15388                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15389                Bound {
15390                    target,
15391                    client,
15392                    client_rx,
15393                    channel,
15394                    epoch,
15395                    bind,
15396                }
15397            }
15398
15399            fn live(&self, route: &Bound) -> bool {
15400                matches!(
15401                    self.forwarding
15402                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15403                        .unwrap(),
15404                    DataRoute::Client(DataRouteState::Bound(_))
15405                )
15406            }
15407        }
15408
15409        struct Relayed {
15410            target: String,
15411            client: RouteCtx,
15412            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15413            task: tokio::task::JoinHandle<Vec<Frame>>,
15414            bind: Frame,
15415        }
15416
15417        struct Bound {
15418            target: String,
15419            client: RouteCtx,
15420            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15421            channel: u16,
15422            epoch: u32,
15423            bind: Frame,
15424        }
15425
15426        impl Bound {
15427            /// The reason of the `route.closed` this client was sent, after
15428            /// checking it also got a GOODBYE on exactly this route.
15429            fn closed_reason(&mut self) -> RouteCloseReason {
15430                let mut reason = None;
15431                let mut goodbye = false;
15432                while let Ok(outbound) = self.client_rx.try_recv() {
15433                    let frame = outbound.frame;
15434                    match frame.header.ty {
15435                        FrameType::Goodbye => {
15436                            assert_eq!(
15437                                (frame.header.channel, frame.header.epoch),
15438                                (self.channel, self.epoch)
15439                            );
15440                            goodbye = true;
15441                        }
15442                        FrameType::Push => {
15443                            let ClientControlPush::RouteClosed {
15444                                reason: r,
15445                                module_id,
15446                                ..
15447                            } = serde_json::from_slice(&frame.body).unwrap()
15448                            else {
15449                                panic!("unexpected push");
15450                            };
15451                            assert_eq!(module_id, self.target);
15452                            reason = Some(r);
15453                        }
15454                        other => panic!("unexpected frame {other:?}"),
15455                    }
15456                }
15457                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15458                reason.expect("the client is told why the route closed")
15459            }
15460
15461            fn untouched(&mut self) -> bool {
15462                self.client_rx.try_recv().is_err()
15463            }
15464
15465            fn stamp(&self) -> Option<ScopeStamp> {
15466                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15467                    ModuleControlRequest::RouteBind { scope, .. } => scope,
15468                    other => panic!("expected a route.bind, got {other:?}"),
15469                }
15470            }
15471        }
15472
15473        #[tokio::test]
15474        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15475        ) {
15476            let mut rig = rig().await;
15477            let mut record = session(1);
15478            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15479            record.child_owners = vec![Principal::Reserved {
15480                module_id: MAGIC.to_string(),
15481            }];
15482            rig.sync(vec![record]).await;
15483            let scope = || Some(rig_selector("s", Some(1)));
15484
15485            // Admitted: the owner, a bare carrier to any module, a targeted
15486            // carrier to its listed module.
15487            rig.bound(Some(OWNER), PLEXUS, scope()).await;
15488            rig.bound(Some(AFT), OTHER, scope()).await;
15489            rig.bound(Some(BROCA), PLEXUS, scope()).await;
15490
15491            // Refused scope_not_carrier: a targeted carrier to an unlisted
15492            // module, a module that is not listed at all (a child owner is not
15493            // a carrier), and a direct key-holder.
15494            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
15495                assert_eq!(
15496                    rig.refused(opener, target, scope()).await,
15497                    error_codes::SCOPE_NOT_CARRIER,
15498                    "{opener:?} -> {target}"
15499                );
15500            }
15501        }
15502
15503        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15504            ScopeSelector::new(
15505                Principal::Reserved {
15506                    module_id: OWNER.to_string(),
15507                },
15508                scope_ref.to_string(),
15509                scope_epoch,
15510            )
15511        }
15512
15513        #[tokio::test]
15514        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
15515            let mut rig = rig().await;
15516            rig.sync(vec![session(1)]).await;
15517            for opener in [OWNER, AFT] {
15518                assert_eq!(
15519                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
15520                        .await,
15521                    error_codes::SCOPE_EPOCH_REQUIRED,
15522                    "{opener}"
15523                );
15524            }
15525        }
15526
15527        #[tokio::test]
15528        async fn admission_separates_not_synced_not_live_and_ended() {
15529            let mut rig = rig().await;
15530            // Before the configured owner's first sync: retryable.
15531            let code = rig
15532                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15533                .await;
15534            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
15535            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
15536
15537            // An owner that is not configured will never sync: terminal.
15538            let ghost = ScopeSelector::new(
15539                Principal::Reserved {
15540                    module_id: "ghost".to_string(),
15541                },
15542                "s".to_string(),
15543                Some(1),
15544            );
15545            assert_eq!(
15546                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
15547                error_codes::SCOPE_NOT_LIVE
15548            );
15549
15550            rig.sync(vec![session(2)]).await;
15551            assert_eq!(
15552                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
15553                    .await,
15554                error_codes::SCOPE_NOT_LIVE
15555            );
15556            for epoch in [1, 3] {
15557                assert_eq!(
15558                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
15559                        .await,
15560                    error_codes::SCOPE_ENDED,
15561                    "epoch {epoch}"
15562                );
15563            }
15564            // Control: the live epoch is admitted.
15565            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
15566                .await;
15567        }
15568
15569        #[tokio::test]
15570        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
15571            let mut rig = rig().await;
15572            rig.sync(vec![session(1)]).await;
15573            let route = rig
15574                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15575                .await;
15576            let stamp = route.stamp().expect("a scoped bind carries the stamp");
15577            assert_eq!(stamp.scope_ref, "s");
15578            assert_eq!(stamp.scope_epoch, 1);
15579            assert_eq!(stamp.kind, ScopeKind::Head);
15580            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
15581            assert!(stamp.attributes.delegates);
15582            assert!(stamp.owner_authorized);
15583            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15584            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
15585
15586            // broca owns a scope of its own on its own module connection; it is
15587            // not in scope_authority_owners, so its stamp is not authorized.
15588            let (broca, mut broca_rx) = wide_ctx(50);
15589            hello_via_sink(
15590                &rig.handler,
15591                &broca,
15592                &mut broca_rx,
15593                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
15594            )
15595            .await;
15596            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
15597                .await
15598                .unwrap();
15599            let own = ScopeSelector::new(
15600                Principal::Reserved {
15601                    module_id: BROCA.to_string(),
15602                },
15603                "b".to_string(),
15604                Some(1),
15605            );
15606            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
15607            assert!(!route.stamp().unwrap().owner_authorized);
15608        }
15609
15610        #[tokio::test]
15611        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
15612            let mut rig = rig().await;
15613            let mut record = head("s", 1);
15614            record.carriers = vec![carrier(AFT, None)];
15615            let flow_id = "Flow:run-7/step_2!~";
15616            record.attributes.flow_id = Some(flow_id.to_string());
15617            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
15618                .await
15619                .unwrap();
15620            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15621                panic!("not a sync reply");
15622            };
15623            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
15624            let route = rig
15625                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15626                .await;
15627            assert!(rig.live(&route), "the stamped bind committed");
15628            let stamp = route.stamp().expect("a flow scope carries a stamp");
15629            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
15630            assert_eq!(stamp.attributes.agent_id, None);
15631            assert!(!stamp.attributes.delegates);
15632            assert!(stamp.owner_authorized);
15633            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15634            assert_eq!(unscoped.stamp(), None);
15635        }
15636
15637        #[tokio::test]
15638        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
15639            let mut rig = rig_with_flow_support(false).await;
15640            let mut record = session(1);
15641            record.attributes.flow_id = Some("flow:7".to_string());
15642            rig.sync(vec![record]).await;
15643            for opener in [OWNER, AFT] {
15644                let body = rig
15645                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15646                    .await;
15647                assert_eq!(body["code"], "target_flow_unsupported");
15648                let message = body["message"].as_str().unwrap();
15649                for required in [PLEXUS, "flow-scopes/v1"] {
15650                    assert!(message.contains(required), "{message}");
15651                }
15652            }
15653        }
15654
15655        #[tokio::test]
15656        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
15657            let mut rig = rig_with_flow_support(true).await;
15658            let mut record = session(1);
15659            record.attributes.flow_id = Some("flow:7".to_string());
15660            rig.sync(vec![record]).await;
15661            for opener in [OWNER, AFT] {
15662                let route = rig
15663                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15664                    .await;
15665                assert!(rig.live(&route));
15666                assert_eq!(
15667                    route.stamp().unwrap().attributes.flow_id.as_deref(),
15668                    Some("flow:7")
15669                );
15670            }
15671        }
15672
15673        #[tokio::test]
15674        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
15675            use std::future::Future;
15676
15677            let mut rig = rig_with_flow_support(true).await;
15678            let mut record = session(1);
15679            record.attributes.flow_id = Some("flow:7".to_string());
15680            rig.sync(vec![record]).await;
15681            let (client, mut client_rx, frame) =
15682                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
15683            // Hold the route-open response permit so admission sees the first
15684            // target but relay reservation cannot capture an endpoint yet.
15685            for _ in 0..64 {
15686                client.egress.try_send(route_bind_ack(1)).unwrap();
15687            }
15688            let handler = rig.handler.clone();
15689            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15690            std::future::poll_fn(|cx| {
15691                assert!(open.as_mut().poll(cx).is_pending());
15692                std::task::Poll::Ready(())
15693            })
15694            .await;
15695            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15696
15697            let old_connection = rig.modules[PLEXUS].0.connection_id;
15698            rig.handler.cleanup_connection(old_connection).unwrap();
15699            let (replacement, mut replacement_rx) = wide_ctx(200);
15700            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
15701            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15702            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15703            let hello = Frame::build(
15704                FrameType::Hello,
15705                control_flags(),
15706                0,
15707                0,
15708                200,
15709                serde_json::to_vec(&body).unwrap(),
15710            )
15711            .unwrap();
15712            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
15713
15714            client_rx.try_recv().unwrap();
15715            let replies = tokio::time::timeout(Duration::from_secs(2), open)
15716                .await
15717                .expect("the replacement is refused without waiting for a bind ack")
15718                .unwrap();
15719            assert_eq!(replies.len(), 1);
15720            let body = parse_error(&replies[0]);
15721            assert_eq!(body["code"], "target_flow_unsupported");
15722            for required in [PLEXUS, "flow-scopes/v1"] {
15723                assert!(body["message"].as_str().unwrap().contains(required));
15724            }
15725            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
15726            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15727            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15728            assert!(rig
15729                .handler
15730                .registry
15731                .get_module_by_connection(replacement.connection_id)
15732                .unwrap()
15733                .is_some());
15734        }
15735
15736        #[tokio::test]
15737        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
15738            use std::future::Future;
15739            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15740            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15741            const NEXT_HASH: &str =
15742                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15743            let mut rig = rig().await;
15744            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
15745            record.carriers.push(ScopeCarrier::new(Principal::Direct));
15746            rig.sync(vec![record.clone()]).await;
15747            let (client, mut rx, frame) = rig.open_frame(
15748                None,
15749                PLEXUS,
15750                Some(
15751                    rig.selector("s", Some(1))
15752                        .with_bind_proof(Some(PROOF.into())),
15753                ),
15754            );
15755            // Hold egress reservation while the scope owner changes its bind-proof
15756            // hash, so a proof valid at admission must be checked again at relay.
15757            for _ in 0..64 {
15758                client.egress.try_send(route_bind_ack(1)).unwrap();
15759            }
15760            let handler = rig.handler.clone();
15761            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15762            std::future::poll_fn(|cx| {
15763                assert!(open.as_mut().poll(cx).is_pending());
15764                std::task::Poll::Ready(())
15765            })
15766            .await;
15767            record.bind_proof_sha256 = Some(NEXT_HASH.into());
15768            rig.sync(vec![record]).await;
15769            rx.try_recv().unwrap();
15770            let replies = tokio::time::timeout(Duration::from_secs(2), open)
15771                .await
15772                .unwrap()
15773                .unwrap();
15774            assert_eq!(
15775                parse_error(&replies[0])["code"],
15776                error_codes::SCOPE_BIND_PROOF_MISMATCH
15777            );
15778            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
15779            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15780            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15781        }
15782
15783        #[tokio::test]
15784        async fn scope_bind_proof_reserved_openers_ignore_proof() {
15785            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15786            let mut rig = rig().await;
15787            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15788                .await;
15789            for opener in [OWNER, AFT] {
15790                for proof in [None, Some("malformed+proof=")] {
15791                    let route = rig
15792                        .bound(
15793                            Some(opener),
15794                            PLEXUS,
15795                            Some(
15796                                rig.selector("s", Some(1))
15797                                    .with_bind_proof(proof.map(str::to_string)),
15798                            ),
15799                        )
15800                        .await;
15801                    assert!(rig.live(&route));
15802                }
15803            }
15804        }
15805
15806        #[tokio::test]
15807        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
15808            use std::future::Future;
15809            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15810            const NEXT_HASH: &str =
15811                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15812            let mut rig = rig().await;
15813            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15814                .await;
15815            let (client, mut rx, frame) =
15816                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
15817            for _ in 0..64 {
15818                client.egress.try_send(route_bind_ack(1)).unwrap();
15819            }
15820            let handler = rig.handler.clone();
15821            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15822            std::future::poll_fn(|cx| {
15823                assert!(open.as_mut().poll(cx).is_pending());
15824                std::task::Poll::Ready(())
15825            })
15826            .await;
15827            rig.sync(vec![
15828                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
15829            ])
15830            .await;
15831            rx.try_recv().unwrap();
15832            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15833            let bind = std::future::poll_fn(|cx| {
15834                assert!(open.as_mut().poll(cx).is_pending());
15835                module_rx.poll_recv(cx)
15836            })
15837            .await
15838            .unwrap()
15839            .frame;
15840            handler
15841                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
15842                .await
15843                .unwrap();
15844            let replies = tokio::time::timeout(Duration::from_secs(2), open)
15845                .await
15846                .unwrap()
15847                .unwrap();
15848            assert_eq!(replies.len(), 1);
15849            // Proof rechecking allowed the relay. Independent tag settlement
15850            // still asks the opener to retry after any same-epoch content change.
15851            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
15852            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15853            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
15854            let route = rig
15855                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
15856                .await;
15857            assert!(rig.live(&route));
15858        }
15859
15860        #[tokio::test]
15861        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
15862        {
15863            let mut rig = rig_with_flow_support(false).await;
15864            rig.sync(vec![session(1)]).await;
15865            let route = rig
15866                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15867                .await;
15868            assert!(rig.live(&route));
15869            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
15870            let mut record = session(1);
15871            record.attributes.flow_id = Some("flow:7".to_string());
15872            rig.sync(vec![record]).await;
15873            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
15874            assert!(rig.live(&unscoped));
15875            assert_eq!(unscoped.stamp(), None);
15876        }
15877
15878        #[tokio::test]
15879        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
15880            let mut rig = rig().await;
15881            let mut record = session(1);
15882            record.attributes.flow_id = Some("flow:7".to_string());
15883            rig.sync(vec![record.clone()]).await;
15884            let before = rig.forwarding.published_scope_tag(OWNER, "s");
15885            let mut owner_route = rig
15886                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15887                .await;
15888            let mut carrier_route = rig
15889                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
15890                .await;
15891            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15892            rig.sync(vec![record.clone()]).await;
15893            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15894            assert!(rig.live(&owner_route) && owner_route.untouched());
15895            assert!(rig.live(&carrier_route) && carrier_route.untouched());
15896
15897            record.attributes.flow_id = Some("flow:8".to_string());
15898            rig.sync(vec![record]).await;
15899            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
15900            let before = before.unwrap();
15901            assert_eq!(after.scope_epoch, before.scope_epoch);
15902            assert!(after.version > before.version);
15903            for route in [&mut owner_route, &mut carrier_route] {
15904                assert!(!rig.live(route));
15905                assert_eq!(
15906                    route.closed_reason(),
15907                    RouteCloseReason::ScopeDelegationChanged
15908                );
15909            }
15910            assert!(rig.live(&unscoped) && unscoped.untouched());
15911            // Each provider also receives a GOODBYE for its drained route;
15912            // consume it before expecting the next route.bind on that sink.
15913            for target in [PLEXUS, OTHER] {
15914                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
15915                let goodbye = module_rx
15916                    .try_recv()
15917                    .expect("the provider sees the drain")
15918                    .frame;
15919                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15920                assert!(module_rx.try_recv().is_err());
15921            }
15922            let rebound = rig
15923                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15924                .await;
15925            assert_eq!(
15926                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
15927                Some("flow:8")
15928            );
15929        }
15930
15931        /// The owner's sync lands between admission and the module's ack. The
15932        /// open is refused by name, the module's other routes stay up, and the
15933        /// reserved pair is released. Changed content is retryable; an ended
15934        /// scope is not.
15935        #[tokio::test]
15936        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
15937            let mut rig = rig().await;
15938            rig.sync(vec![session(1)]).await;
15939            let mut cotenant = rig
15940                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15941                .await;
15942
15943            let mut changed = session(1);
15944            changed.child_owners.push(Principal::Reserved {
15945                module_id: MAGIC.to_string(),
15946            });
15947            let mut ended = None;
15948            for (code, next) in [
15949                (error_codes::SCOPE_CHANGED, vec![changed]),
15950                (error_codes::SCOPE_ENDED, Vec::new()),
15951            ] {
15952                let relayed = rig
15953                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15954                    .await;
15955                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
15956                ended = Some(next.is_empty());
15957                rig.sync(next).await;
15958                rig.ack(&relayed).await;
15959                let replies = relayed.task.await.unwrap();
15960                assert_eq!(replies.len(), 1, "{replies:?}");
15961                assert_eq!(parse_error(&replies[0])["code"], code);
15962                assert_eq!(
15963                    subc_protocol::error_codes::is_retryable_route_open(code),
15964                    code == error_codes::SCOPE_CHANGED
15965                );
15966                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15967                // The module is told to drop just the binding it created.
15968                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15969                // Collected, because ending the scope also closes the co-tenant
15970                // route, whose GOODBYE comes first.
15971                let mut goodbyes = Vec::new();
15972                while let Ok(outbound) = plexus_rx.try_recv() {
15973                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
15974                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
15975                }
15976                assert!(
15977                    goodbyes.contains(&(bind_channel, bind_epoch)),
15978                    "{goodbyes:?}"
15979                );
15980                assert!(rig
15981                    .handler
15982                    .registry
15983                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
15984                    .unwrap()
15985                    .is_some());
15986            }
15987            assert_eq!(ended, Some(true));
15988            // The co-tenant stayed up through the change, and closed only when
15989            // the scope ended, by the drain rule rather than by the commit.
15990            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
15991        }
15992
15993        /// Each row of the drain table on one set of routes: the owner's, a
15994        /// bare carrier's, and a targeted carrier's to each of its targets.
15995        #[tokio::test]
15996        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
15997            struct Case {
15998                name: &'static str,
15999                change: fn(&mut ScopeRecord),
16000                /// Closed routes by index: owner->plexus, aft->plexus,
16001                /// broca->plexus, broca->other.
16002                closed: [Option<RouteCloseReason>; 4],
16003            }
16004            use RouteCloseReason::*;
16005            let cases = [
16006                Case {
16007                    name: "a carrier entry removed",
16008                    change: |r| {
16009                        r.carriers.retain(|c| {
16010                            c.principal
16011                                != Principal::Reserved {
16012                                    module_id: AFT.to_string(),
16013                                }
16014                        })
16015                    },
16016                    closed: [None, Some(ScopeCarrierRemoved), None, None],
16017                },
16018                Case {
16019                    name: "a target removed from a carrier",
16020                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
16021                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
16022                },
16023                Case {
16024                    name: "a bare carrier narrowed to targets",
16025                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
16026                    closed: [None, Some(ScopeCarrierRemoved), None, None],
16027                },
16028                Case {
16029                    name: "delegates turned off",
16030                    change: |r| r.attributes.delegates = false,
16031                    closed: [Some(ScopeDelegationChanged); 4],
16032                },
16033                Case {
16034                    name: "agent_id changed",
16035                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
16036                    closed: [Some(ScopeDelegationChanged); 4],
16037                },
16038                Case {
16039                    name: "a carrier added, child owners changed, the record re-sent",
16040                    change: |r| {
16041                        r.carriers.push(carrier(MAGIC, None));
16042                        r.child_owners.push(Principal::Reserved {
16043                            module_id: MAGIC.to_string(),
16044                        });
16045                    },
16046                    closed: [None; 4],
16047                },
16048                Case {
16049                    name: "a target added",
16050                    change: |r| {
16051                        r.carriers[1]
16052                            .targets
16053                            .as_mut()
16054                            .unwrap()
16055                            .push("third".to_string())
16056                    },
16057                    closed: [None; 4],
16058                },
16059                Case {
16060                    name: "delegates turned on",
16061                    change: |r| r.attributes.delegates = true,
16062                    closed: [None; 4],
16063                },
16064            ];
16065            for case in cases {
16066                let mut rig = rig().await;
16067                rig.sync(vec![session(1)]).await;
16068                let scope = || Some(rig_selector("s", Some(1)));
16069                let mut routes = [
16070                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
16071                    rig.bound(Some(AFT), PLEXUS, scope()).await,
16072                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
16073                    rig.bound(Some(BROCA), OTHER, scope()).await,
16074                ];
16075                let mut record = session(1);
16076                (case.change)(&mut record);
16077                rig.sync(vec![record]).await;
16078                for (index, expected) in case.closed.iter().enumerate() {
16079                    let route = &mut routes[index];
16080                    match expected {
16081                        Some(reason) => {
16082                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
16083                            assert_eq!(
16084                                route.closed_reason(),
16085                                *reason,
16086                                "{}: route {index}",
16087                                case.name
16088                            );
16089                        }
16090                        None => {
16091                            assert!(rig.live(route), "{}: route {index} closed", case.name);
16092                            assert!(
16093                                route.untouched(),
16094                                "{}: route {index} was told something",
16095                                case.name
16096                            );
16097                        }
16098                    }
16099                }
16100            }
16101        }
16102
16103        #[tokio::test]
16104        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
16105            // Removed, and replaced by a higher epoch.
16106            for next in [Vec::new(), vec![session(2)]] {
16107                let mut rig = rig().await;
16108                rig.sync(vec![session(1)]).await;
16109                let mut route = rig
16110                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16111                    .await;
16112                rig.sync(next).await;
16113                assert!(!rig.live(&route));
16114                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
16115            }
16116
16117            // A child whose parent ends: its routes close as parent-ended, the
16118            // child stays live, and routes under the parent close as ended.
16119            let mut rig = rig().await;
16120            let mut child = session(1);
16121            child.scope_ref = "child".to_string();
16122            child.kind = ScopeKind::Worker;
16123            child.parent = Some(ScopeParent::new(
16124                Principal::Reserved {
16125                    module_id: OWNER.to_string(),
16126                },
16127                "s".to_string(),
16128                1,
16129            ));
16130            rig.sync(vec![session(1), child.clone()]).await;
16131            let mut child_route = rig
16132                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
16133                .await;
16134            assert_eq!(
16135                child_route.stamp().unwrap().parent_state,
16136                Some(ParentState::Linked)
16137            );
16138            rig.sync(vec![child]).await;
16139            assert!(!rig.live(&child_route));
16140            assert_eq!(
16141                child_route.closed_reason(),
16142                RouteCloseReason::ScopeParentEnded
16143            );
16144        }
16145
16146        #[tokio::test]
16147        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
16148        ) {
16149            let mut rig = rig().await;
16150            rig.sync(vec![session(1)]).await;
16151            let mut route = rig
16152                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16153                .await;
16154            let before = rig.forwarding.published_scope_tag(OWNER, "s");
16155            rig.sync(vec![session(1)]).await;
16156            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16157            assert!(rig.live(&route) && route.untouched());
16158
16159            // A call in flight on the route when another carrier is added. A
16160            // forwarded REQUEST holds one credit on the route's flow until the
16161            // module answers; the router takes it exactly like this.
16162            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
16163                .forwarding
16164                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16165                .unwrap()
16166            else {
16167                panic!("the route is bound");
16168            };
16169            binding.flow.acquire_tagged(9, false).await.unwrap();
16170            let mut widened = session(1);
16171            widened.carriers.push(carrier(MAGIC, None));
16172            rig.sync(vec![widened]).await;
16173            assert!(rig.live(&route) && route.untouched());
16174            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16175            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
16176            // The call's credit is still held on an open flow, so its answer
16177            // will be delivered: closing the route would have closed the flow.
16178            assert_eq!(binding.flow.in_flight(), 1);
16179            binding
16180                .flow
16181                .acquire_tagged(10, false)
16182                .await
16183                .expect("the flow is still open");
16184        }
16185
16186        /// A swap's superseded endpoint keeps its routes until drained; ending
16187        /// the scope closes them there too.
16188        #[tokio::test]
16189        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
16190            let mut rig = rig().await;
16191            rig.sync(vec![session(1)]).await;
16192            let mut on_incumbent = rig
16193                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16194                .await;
16195
16196            // Swap plexus: register a candidate and cut over, leaving the
16197            // incumbent superseded with the route still on it.
16198            let (candidate, _candidate_rx) = wide_ctx(9);
16199            let registration = rig
16200                .handler
16201                .registry
16202                .register_candidate_with_control_ops(
16203                    manifest(PLEXUS, PROTOCOL_VERSION),
16204                    PROTOCOL_VERSION,
16205                    candidate.connection_id,
16206                    module_baseline_control_ops(),
16207                )
16208                .unwrap();
16209            rig.forwarding
16210                .register_candidate_module_connection(
16211                    candidate.connection_id,
16212                    PLEXUS.to_string(),
16213                    PROTOCOL_VERSION,
16214                    manifest_concurrency(&registration.manifest),
16215                    candidate.egress.clone(),
16216                )
16217                .unwrap();
16218            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
16219            rig.handler
16220                .registry
16221                .promote_candidate(PLEXUS)
16222                .unwrap()
16223                .unwrap();
16224            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
16225
16226            rig.sync(Vec::new()).await;
16227            assert!(!rig.live(&on_incumbent));
16228            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
16229            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16230            let goodbye = incumbent_rx
16231                .try_recv()
16232                .expect("the superseded endpoint is told")
16233                .frame;
16234            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16235        }
16236    }
16237}
16238
16239#[cfg(test)]
16240mod concurrency_default_exposure_tests {
16241    use super::*;
16242
16243    fn hello_body(role_json: &str) -> Vec<u8> {
16244        format!(
16245            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
16246        )
16247        .into_bytes()
16248    }
16249
16250    fn manifest_from(body: &[u8]) -> ModuleManifest {
16251        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
16252        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
16253            .expect("manifest parses")
16254    }
16255
16256    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
16257
16258    #[test]
16259    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
16260        let body = hello_body(&format!(
16261            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
16262        ));
16263        let manifest = manifest_from(&body);
16264        // Precondition: serde really resolved it to the default, so the typed
16265        // manifest alone cannot answer the question this probe exists for.
16266        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16267        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
16268    }
16269
16270    #[test]
16271    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
16272        let body = hello_body(&format!(
16273            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
16274        ));
16275        let manifest = manifest_from(&body);
16276        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16277        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16278    }
16279
16280    #[test]
16281    fn non_management_roles_are_never_reported() {
16282        let body = hello_body(
16283            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
16284        );
16285        let manifest = manifest_from(&body);
16286        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16287    }
16288}