Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    net::SocketAddr,
5    path::{Path, PathBuf},
6    sync::{Arc, Mutex, RwLock},
7    time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22    error_codes,
23    manifest::{
24        validate_hello_capability_grammar, validate_hello_event_declarations,
25        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27    },
28    scope::{
29        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31        SCOPE_SYNC_OP,
32    },
33    session::{
34        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37        ROLE_VERSIONS_FIELD,
38    },
39    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40    Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46    caller_identity::CallerIdentity,
47    capability_requirements::{
48        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50        RuntimeModule,
51    },
52    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
53    forwarding::{
54        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
55        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
56        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
57    },
58    observability::{
59        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
60    },
61    privileged_prompt::{prompt_text, PrivilegedOperation},
62    provenance::{
63        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
64    },
65    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
66    router::{RouteCtx, RouterError},
67    scopes::{HelloLaunchNonces, ScopeTable},
68    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
69    stderr_tail::{CaptureState, TailEntry},
70    supervise::{
71        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
72        SupervisorHandle, SwapHelloAdmission,
73    },
74    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
75};
76
77/// Lowest envelope version this subc build will negotiate.
78///
79/// Module HELLO negotiation is exact: peers must use the daemon's locked
80/// protocol version. Older and newer peers receive `version_unsupported` and
81/// are not registered.
82pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
83
84const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
85const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
86const CAP_PING_PONG: &str = "ping_pong_v1";
87const CAP_SESSION_ATTACH: &str = "session_attach_v1";
88const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
89
90const SUBC_CONTROL_OPS: &[&str] = &[
91    ops::ACTIVITY_RECORD,
92    ops::ACTIVITY_QUERY,
93    ops::ACTIVITY_SUBSCRIBE,
94    ops::SERVER_DESCRIBE,
95    ops::CATALOG_LIST,
96    ops::ROUTE_OPEN,
97    ops::ROUTE_POLL,
98    ops::ROUTE_CLOSING,
99    ops::ROUTE_CLOSED,
100    ops::SUPERVISOR_LIST,
101    ops::SUPERVISOR_RESTART,
102    ops::SUPERVISOR_SWAP,
103    ops::SUPERVISOR_RELOAD,
104    ops::SUPERVISOR_RESCAN,
105    ops::SUPERVISOR_RELEASE_RESERVED,
106    ops::SUPERVISOR_SET_ENABLED,
107    ops::SUPERVISOR_HEALTH_PROBE,
108    ops::SUPERVISOR_HEALTH,
109    ops::SUPERVISOR_STDERR_TAIL,
110    ops::SUPERVISOR_TERMINALS,
111    ops::SUPERVISOR_ROUTES,
112    ops::SUPERVISOR_PROVENANCE,
113    ops::SUPERVISOR_SPAWN_SNAPSHOT,
114    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
115];
116
117const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
118    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
119    "supervisor.live_roots",
120    SCOPE_SYNC_OP,
121    SCOPE_APPLY_OP,
122    SCOPE_DESCRIBE_OP,
123    "operator.confirm",
124];
125
126/// Module-originated ops the daemon answers but does not advertise in
127/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
128/// belongs to is incomplete, so no module is told it works before it does.
129const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
130
131const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
132
133/// How long subc waits for a module to ack a relayed route.bind before returning
134/// `module_timeout`. The ack waits on the module's own configure, which for AFT
135/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
136/// and DB-open work — on a cold page cache or a large repo that legitimately
137/// exceeds a couple of seconds. The default is generous because rejecting a VALID
138/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
139/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
140pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
141
142/// How many CONSECUTIVE full-budget relay timeouts against one target module
143/// open that module's bind-relay breaker.
144///
145/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
146/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
147/// so three consecutive full-budget timeouts take ~36s to observe: every client
148/// whose open contributed to opening the breaker had already given up on its
149/// own. That is what makes opening the breaker unable to turn a call that would
150/// have succeeded into a refusal — it can only make an already-failing module
151/// fail faster.
152///
153/// Two would be reachable inside one default deadline. One would convict a
154/// module on a single cold-cache bind, which is exactly the valid-but-slow case
155/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
156pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
157
158/// How long a module's bind-relay breaker stays open before exactly one
159/// `route.open` is let through as a probe.
160///
161/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
162/// re-pays a full-budget stall almost continuously, and the breaker stops being
163/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
164/// deadline: a client that starts retrying after the module recovers has to get
165/// a probe opportunity inside its own deadline, or the breaker converts a
166/// recovered module into a failed call — the failure it exists to prevent,
167/// pointed the other way.
168///
169/// 20s sits between those with room on both sides, and it caps what a wedged
170/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
171/// rather than one per `route.open` per connection. The stall that motivated
172/// this, with its measurements, is written up in
173/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
174/// waited the whole budget out.
175pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
176
177const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
178const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
179
180fn reload_verdict(
181    configured: &Path,
182    spawned_from: Option<&Path>,
183    image: subc_control::RunningImageAgreement,
184) -> PendingReloadVerdict {
185    let path = match spawned_from {
186        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
187            ReloadPathAgreement::Match
188        }
189        Some(spawned_from) => ReloadPathAgreement::Mismatch {
190            configured: configured.to_path_buf(),
191            spawned_from: spawned_from.to_path_buf(),
192        },
193        None => ReloadPathAgreement::Unavailable {
194            reason: if matches!(
195                image,
196                subc_control::RunningImageAgreement::Unavailable {
197                    reason: subc_control::RunningImageUnavailableReason::NotRunning
198                }
199            ) {
200                ReloadPathUnavailableReason::NotRunning
201            } else {
202                ReloadPathUnavailableReason::SpawnedPathUnavailable
203            },
204        },
205    };
206    PendingReloadVerdict { path, image }
207}
208
209fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
210    if configured == spawned_from {
211        return true;
212    }
213    #[cfg(windows)]
214    {
215        // Windows spawn records the resolved verbatim path (\\?\...), while
216        // configuration keeps its original spelling. Resolve both with the OS
217        // instead of stripping prefixes or folding case, which could conflate
218        // distinct names in case-sensitive directories. Different resolved
219        // paths still differ even if their files have identical content.
220        match (
221            std::fs::canonicalize(configured),
222            std::fs::canonicalize(spawned_from),
223        ) {
224            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
225            _ => false,
226        }
227    }
228    #[cfg(not(windows))]
229    {
230        false
231    }
232}
233
234#[derive(Clone)]
235struct DaemonProvenanceFacts {
236    build: DaemonBuildProvenance,
237    pid: Option<u32>,
238    started_at_ms: Option<u64>,
239    start_clock: Option<crate::clock::StartClock>,
240    executable_path: Option<PathBuf>,
241    executable_identity: Option<SpawnedFileIdentity>,
242    process_start_time: Option<u64>,
243    probe: ExecutableIdentityProbe,
244}
245
246impl Default for DaemonProvenanceFacts {
247    fn default() -> Self {
248        Self {
249            build: DaemonBuildProvenance {
250                build_git_sha: None,
251                build_lock_digest: None,
252            },
253            pid: None,
254            started_at_ms: None,
255            start_clock: None,
256            executable_path: None,
257            executable_identity: None,
258            process_start_time: None,
259            probe: ExecutableIdentityProbe::default(),
260        }
261    }
262}
263
264#[derive(Debug, Clone)]
265struct SupervisorRescanContext {
266    supervisor: Supervisor,
267    config_path: PathBuf,
268    configured_port: Option<u16>,
269    storage_config: Option<crate::daemon_config::StorageConfig>,
270    admission_facts_carrier_module_id: Option<String>,
271    admission_facts_targets: Option<Vec<String>>,
272    scope_authority_owners: Vec<String>,
273    privileged_operator_prompt: PrivilegedOperatorPrompt,
274}
275
276/// Refusal labels passed to `observe_route_open_refusal` that mean the target
277/// module is not serving right now, and so open or extend an outage in the
278/// route outage tracker. Every one of them is only reachable after the target
279/// was found in the registry, which is what keeps an arbitrary client-chosen
280/// id from ever creating tracker state.
281///
282/// Deliberately absent: `not_registered` and `removed` (the id may be
283/// anything a client sent, and a removed module is gone on purpose),
284/// `protocol_none` (such a module never serves routes, so nothing is out),
285/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
286/// capability and admission-facts refusals (they refuse the caller, not a
287/// module outage), and `relay_reservation_failed` (its code ranges over
288/// capacity limits as well as a vanished connection). Capacity, breaker,
289/// relay-timeout and module-rejection refusals do not pass through that
290/// function at all; the breaker logs its own transitions.
291///
292/// The two not-serving refusals that bypass that function record themselves
293/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
294/// `required_capability_unprovided` is not tracked: the module itself is up,
295/// and the outage belongs to the missing provider.
296const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
297    "reloading",
298    "supervisor_not_live",
299    "registration_not_active",
300    "no_forwarding_connection",
301    "relay_send_failed",
302];
303
304/// Real channel-0 control handler for subc itself.
305#[derive(Clone)]
306pub struct ControlHandler {
307    registry: Arc<Registry>,
308    activity: Option<crate::activity::ActivityLedger>,
309    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
310    forwarding: Arc<ForwardingTable>,
311    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
312    supervisor: SupervisorHandle,
313    subc_capabilities: Arc<[String]>,
314    /// Daemon-wide route.bind relay budget. Used as the fallback when the
315    /// target module has no per-module override in
316    /// `route_bind_relay_timeouts`.
317    route_bind_relay_timeout: Duration,
318    /// Per-module route.bind relay budget overrides, keyed by module id. When
319    /// `handle_route_open` resolves the deadline for a target module, a
320    /// per-module entry wins over the daemon-wide value above.
321    route_bind_relay_timeouts: BTreeMap<String, Duration>,
322    /// Per-target-module bind-relay breaker state. Shared with the forwarding
323    /// table, which is where a new module connection resets it.
324    route_bind_breakers: RouteBindBreakers,
325    /// Live relay admissions keyed by target module. Shared through the
326    /// forwarding table so cloned or separately built handlers enforce one cap.
327    route_bind_concurrency: RouteBindConcurrency,
328    /// Start and end of each module's not-serving period as seen by
329    /// `route.open`, so an outage gets one line at each edge instead of only
330    /// the per-refusal INFO lines. Taken from the forwarding table, so every
331    /// handler built over one table shares it.
332    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
333    /// Consecutive relay timeouts that open a module's breaker.
334    route_bind_breaker_threshold: u32,
335    /// How long a breaker stays open before one probe is admitted.
336    route_bind_breaker_cooldown: Duration,
337    health_probe_timeout: Duration,
338    /// Central storage policy. When set, each registering module receives its
339    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
340    storage_config: Option<crate::daemon_config::StorageConfig>,
341    /// The machine id established at boot, served on every HELLO_ACK and on
342    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
343    /// changes the file, never this value. `None` serves no id.
344    machine_id: Option<crate::machine_id::MachineId>,
345    admission_facts_carrier_module_id: Option<String>,
346    admission_facts_targets: Option<Vec<String>>,
347    /// Scope records with their sync authorities and tombstones; see
348    /// `crate::scopes`. Shared by clones of this handler, so every connection
349    /// reads and writes one table.
350    scopes: Arc<RwLock<ScopeTable>>,
351    /// Wakes the expiry loop when an accepted scope change may have added a
352    /// deadline, so the loop can wait without ticking while no live scope has
353    /// one. A permit is stored when nothing waits, so no wake is lost.
354    scope_deadline_added: Arc<tokio::sync::Notify>,
355    /// The configured `scope_authority_owners`, kept so a rescan can report a
356    /// changed value as needing a daemon restart; rescan never applies it.
357    scope_authority_owners: Vec<String>,
358    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
359    privileged_operator_prompt: PrivilegedOperatorPrompt,
360    /// The local and peer address of each accepted TCP client connection, read
361    /// from the daemon's own socket, so a privileged request can name its caller.
362    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
363    #[cfg(test)]
364    caller_lookup: Option<TestCallerLookup>,
365    /// The launch nonce each module connection presented at HELLO, which is how
366    /// a `scope.sync` is matched to the owner's current launch.
367    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
368    rescan: Option<SupervisorRescanContext>,
369    connected_clients: ConnectedClients,
370    counters: DaemonCounters,
371    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
372    daemon_provenance: DaemonProvenanceFacts,
373    #[cfg(test)]
374    control_dispatch_delay: Option<Duration>,
375    #[cfg(test)]
376    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
377    /// Holds `route.open` after the route.bind is queued to the module and
378    /// before it is marked enqueued, so a test can make the module's reply (or
379    /// a client teardown) land exactly inside that window.
380    #[cfg(test)]
381    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
382    /// Holds a HELLO after its registry entry is reserved and before its
383    /// forwarding endpoint is installed, so a test can read the catalog and
384    /// open routes while the module is registered but not yet routable. A
385    /// blocking barrier because the HELLO handler is synchronous; the test
386    /// runs the HELLO on its own thread.
387    #[cfg(test)]
388    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
389}
390
391impl fmt::Debug for ControlHandler {
392    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
393        f.debug_struct("ControlHandler")
394            .field("registry", &self.registry)
395            .field("forwarding", &self.forwarding)
396            .field("process_liveness", &self.process_liveness.is_some())
397            .field("supervisor", &self.supervisor)
398            .field("subc_capabilities", &self.subc_capabilities)
399            .finish()
400    }
401}
402
403struct RouteOpenRequest {
404    target: RouteTarget,
405    identity: BindIdentity,
406    consumer_identity: Option<ConsumerIdentity>,
407    consumer_capabilities: Option<Vec<String>>,
408    role_versions: Option<BTreeMap<String, String>>,
409    admission_facts: Option<serde_json::Value>,
410    scope: Option<ScopeSelector>,
411}
412
413#[derive(Clone, Copy)]
414enum RouteOpenPhase {
415    Admission,
416    ProjectRoot,
417    EgressReservation,
418    BindSend,
419    BindAckWait,
420}
421
422/// Store elapsed time for each route-open phase, including attempts that end
423/// in an early refusal or backend error. Admission also includes rechecks and
424/// commit bookkeeping performed between the other timed phases.
425struct RouteOpenTimings {
426    module_id: String,
427    principal: String,
428    admission: Duration,
429    project_root: Duration,
430    egress_reservation: Duration,
431    bind_send: Duration,
432    bind_ack_wait: Duration,
433    phase: Option<RouteOpenPhase>,
434    phase_started_at: Instant,
435}
436
437impl RouteOpenTimings {
438    fn new() -> Self {
439        Self {
440            module_id: String::new(),
441            // Refusals before attestation must not claim a verified principal.
442            principal: "unresolved".to_string(),
443            admission: Duration::ZERO,
444            project_root: Duration::ZERO,
445            egress_reservation: Duration::ZERO,
446            bind_send: Duration::ZERO,
447            bind_ack_wait: Duration::ZERO,
448            phase: Some(RouteOpenPhase::Admission),
449            phase_started_at: Instant::now(),
450        }
451    }
452
453    fn finish(&mut self) {
454        let elapsed = self.phase_started_at.elapsed();
455        match self.phase.take() {
456            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
457            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
458            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
459            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
460            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
461            None => {}
462        }
463    }
464
465    fn enter(&mut self, phase: RouteOpenPhase) {
466        self.finish();
467        self.phase = Some(phase);
468        self.phase_started_at = Instant::now();
469    }
470}
471
472struct RouteBindReservationGuard {
473    forwarding: Arc<ForwardingTable>,
474    endpoint: ModuleEndpointId,
475    relay_corr: u64,
476    armed: bool,
477}
478
479struct ModuleControlRpcGuard {
480    forwarding: Arc<ForwardingTable>,
481    endpoint: ModuleEndpointId,
482    corr: u64,
483    armed: bool,
484}
485
486impl ModuleControlRpcGuard {
487    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
488        Self {
489            forwarding,
490            endpoint,
491            corr,
492            armed: true,
493        }
494    }
495
496    fn disarm(&mut self) {
497        self.armed = false;
498    }
499}
500
501impl Drop for ModuleControlRpcGuard {
502    fn drop(&mut self) {
503        if self.armed {
504            let _ = self
505                .forwarding
506                .cancel_module_control_rpc(self.endpoint, self.corr);
507        }
508    }
509}
510
511impl RouteBindReservationGuard {
512    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
513        Self {
514            forwarding,
515            endpoint,
516            relay_corr,
517            armed: true,
518        }
519    }
520
521    fn release_and_disarm(&mut self) {
522        if !self.armed {
523            return;
524        }
525        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
526            self.endpoint,
527            self.relay_corr,
528            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
529        ) {
530            send_goodbye_target_best_effort(
531                &self.forwarding.counters(),
532                &target,
533                "canceled route.bind",
534            );
535        }
536        self.armed = false;
537    }
538
539    fn disarm(&mut self) {
540        self.armed = false;
541    }
542}
543
544impl Drop for RouteBindReservationGuard {
545    fn drop(&mut self) {
546        self.release_and_disarm();
547    }
548}
549
550/// Per-target-module circuit breaker around the `route.bind` relay.
551///
552/// The connection reader is serial per connection, so a module whose `on_bind`
553/// sits on the ack blocks every LATER frame on the connections that call it,
554/// including calls to unrelated modules. This does not make any module's bind
555/// fast; it stops the daemon paying the full budget again and again for a
556/// condition it has already observed.
557///
558/// State is keyed by TARGET MODULE and shared by every connection: a wedged
559/// module wedges everyone, so what one connection learned should protect the
560/// rest.
561///
562/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
563/// relay to that module has actually timed out, and is removed again when a
564/// relay is accepted or the module reconnects, so it cannot grow with traffic
565/// or with modules that behave.
566///
567/// # Why a `std` mutex here is not the head-of-line defect again
568///
569/// Acquisition never awaits. The critical section is a hash lookup plus a few
570/// integer updates, with no I/O and no `.await` inside it, so a reader task
571/// cannot be descheduled behind it the way it can behind
572/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
573/// primitive, held for the same kind of work, as the refusal counter this very
574/// path already increments.
575///
576/// It is also NOT on the data-plane splice path: only `route.open` and module
577/// registration touch it, so bound-route frames gain no state check and no
578/// contention.
579#[derive(Debug, Clone, Default)]
580pub(crate) struct RouteBindBreakers {
581    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
582}
583
584#[derive(Debug, Clone, Default)]
585pub(crate) struct RouteBindConcurrency {
586    modules: Arc<Mutex<HashMap<String, usize>>>,
587}
588
589struct RouteBindConcurrencyGuard {
590    concurrency: RouteBindConcurrency,
591    module_id: String,
592}
593
594impl RouteBindConcurrency {
595    /// Admit without waiting. Waiting here would move the bind stall from the
596    /// module reply to a semaphore and restore reader head-of-line blocking.
597    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
598        let mut modules = self
599            .modules
600            .lock()
601            .expect("route.bind concurrency mutex poisoned");
602        let in_flight = modules.entry(module_id.to_string()).or_default();
603        if *in_flight >= limit {
604            return Err(*in_flight);
605        }
606        *in_flight += 1;
607        Ok(RouteBindConcurrencyGuard {
608            concurrency: self.clone(),
609            module_id: module_id.to_string(),
610        })
611    }
612}
613
614impl Drop for RouteBindConcurrencyGuard {
615    fn drop(&mut self) {
616        let mut modules = self
617            .concurrency
618            .modules
619            .lock()
620            .expect("route.bind concurrency mutex poisoned");
621        let remove = {
622            let in_flight = modules
623                .get_mut(&self.module_id)
624                .expect("admitted route.bind has a concurrency entry");
625            *in_flight -= 1;
626            *in_flight == 0
627        };
628        if remove {
629            modules.remove(&self.module_id);
630        }
631    }
632}
633
634#[derive(Debug, Default)]
635struct ModuleBreakerState {
636    /// Relay timeouts observed with no accepted relay in between.
637    consecutive_timeouts: u32,
638    /// `Some` while the breaker is open: the instant the cooldown expires and
639    /// the next arrival may probe. `None` means closed.
640    cooldown_until: Option<Instant>,
641    /// A half-open probe has been admitted and has not settled yet. This is
642    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
643    /// that read the cooldown, so concurrent opens arriving at the moment the
644    /// cooldown expires cannot all decide that they are the probe.
645    probe_in_flight: Option<Arc<()>>,
646}
647
648/// What the breaker decided for one `route.open`, before any relay work.
649enum RouteBindAdmission<'a> {
650    Admitted {
651        guard: RouteBindBreakerGuard<'a>,
652        /// This open is the single half-open probe, so the transition is worth
653        /// one log line.
654        probe: bool,
655    },
656    Refused {
657        consecutive_timeouts: u32,
658        /// What is left of the cooldown. Zero when the refusal is because the
659        /// one probe is already in flight rather than because the cooldown has
660        /// not elapsed.
661        retry_in: Duration,
662        probe_in_flight: bool,
663    },
664}
665
666/// An outstanding admission, which must be told how its relay settled.
667///
668/// `Drop` settles it as inconclusive, so an early return between admission and
669/// the relay -- or the whole handler being cancelled when the client
670/// disconnects -- releases a half-open probe slot instead of leaving the
671/// breaker wedged half-open with no further probes.
672struct RouteBindBreakerGuard<'a> {
673    breakers: RouteBindBreakers,
674    module_id: &'a str,
675    probe_token: Option<Arc<()>>,
676    settled: bool,
677}
678
679impl RouteBindBreakerGuard<'_> {
680    /// The module answered within the budget and took the bind. THE ONLY
681    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
682    /// breaker, which is a transition worth logging.
683    fn record_accepted(&mut self) -> bool {
684        self.settled = true;
685        self.breakers.record_accepted(self.module_id)
686    }
687
688    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
689    /// COUNTS TOWARD OPENING.
690    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
691        self.settled = true;
692        self.breakers.record_timeout(
693            self.module_id,
694            self.probe_token.as_ref(),
695            threshold,
696            cooldown,
697        )
698    }
699
700    /// Everything else: the module REJECTED the bind, its connection went away
701    /// mid-relay, or the waiter was cancelled.
702    ///
703    /// None of these is evidence that a module is slow, and each already has
704    /// its own refusal with its own code. A module that rejects a bind in
705    /// microseconds is healthy and must never be convicted for it; a module
706    /// that died has said nothing about the module that replaces it. So these
707    /// neither increment nor reset the count -- they only release a probe slot.
708    fn record_inconclusive(&mut self) {
709        self.settled = true;
710        self.breakers
711            .record_inconclusive(self.module_id, self.probe_token.as_ref());
712    }
713}
714
715impl Drop for RouteBindBreakerGuard<'_> {
716    fn drop(&mut self) {
717        if !self.settled {
718            self.breakers
719                .record_inconclusive(self.module_id, self.probe_token.as_ref());
720        }
721    }
722}
723
724/// The breaker moved to open, reported so the caller can log it outside the
725/// lock. Opening is rare and load-bearing; the refusals that follow are
726/// frequent and are counted rather than logged.
727struct BreakerOpened {
728    consecutive_timeouts: u32,
729    /// True when a failed probe re-opened an already-open breaker, which reads
730    /// very differently in a log from a first opening.
731    reopened_after_probe: bool,
732}
733
734impl RouteBindBreakers {
735    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
736        self.modules
737            .lock()
738            .expect("route.bind breaker mutex poisoned")
739    }
740
741    /// Decide whether this `route.open` may attempt its relay. Takes the map
742    /// lock and nothing else, and never awaits.
743    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
744        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
745            probe: probe_token.is_some(),
746            guard: RouteBindBreakerGuard {
747                breakers: self.clone(),
748                module_id,
749                probe_token,
750                settled: false,
751            },
752        };
753
754        let mut modules = self.lock();
755        let Some(state) = modules.get_mut(module_id) else {
756            return admitted(None);
757        };
758        let Some(cooldown_until) = state.cooldown_until else {
759            return admitted(None);
760        };
761        if state.probe_in_flight.is_some() {
762            return RouteBindAdmission::Refused {
763                consecutive_timeouts: state.consecutive_timeouts,
764                retry_in: Duration::ZERO,
765                probe_in_flight: true,
766            };
767        }
768        let now = Instant::now();
769        if now < cooldown_until {
770            return RouteBindAdmission::Refused {
771                consecutive_timeouts: state.consecutive_timeouts,
772                retry_in: cooldown_until - now,
773                probe_in_flight: false,
774            };
775        }
776        let token = Arc::new(());
777        state.probe_in_flight = Some(Arc::clone(&token));
778        admitted(Some(token))
779    }
780
781    fn record_accepted(&self, module_id: &str) -> bool {
782        self.lock()
783            .remove(module_id)
784            .is_some_and(|state| state.cooldown_until.is_some())
785    }
786
787    fn record_timeout(
788        &self,
789        module_id: &str,
790        probe_token: Option<&Arc<()>>,
791        threshold: u32,
792        cooldown: Duration,
793    ) -> Option<BreakerOpened> {
794        let mut modules = self.lock();
795        let state = modules.entry(module_id.to_string()).or_default();
796        let was_open = state.cooldown_until.is_some();
797        let was_probe = Self::owns_probe(state, probe_token);
798        if was_probe {
799            state.probe_in_flight = None;
800        }
801        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
802        if state.consecutive_timeouts < threshold {
803            return None;
804        }
805        state.cooldown_until = Some(Instant::now() + cooldown);
806        Some(BreakerOpened {
807            consecutive_timeouts: state.consecutive_timeouts,
808            reopened_after_probe: was_open && was_probe,
809        })
810    }
811
812    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
813        // A relay can finish after the breaker was reset or after it opened
814        // again and started a new probe. Only the guard whose token matches the
815        // active probe may release it, so a late relay never frees a newer probe.
816        state
817            .probe_in_flight
818            .as_ref()
819            .zip(token)
820            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
821    }
822
823    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
824        if let Some(state) = self.lock().get_mut(module_id) {
825            if Self::owns_probe(state, probe_token) {
826                state.probe_in_flight = None;
827            }
828        }
829    }
830
831    /// Discard what was learned about a module, because the process it was
832    /// learned about is gone. Returns the discarded count when it was non-zero.
833    ///
834    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
835    /// `module_id` is a configuration identity that outlives any particular
836    /// child; what the breaker observed was the process behind the module
837    /// connection of the moment. When a new connection registers under that id
838    /// the verdict's subject no longer exists, so the verdict is stale by
839    /// construction rather than merely likely to be wrong. Keeping it would
840    /// apply a dead process's record to a live one, which is the same defect
841    /// class this breaker exists to stop the daemon committing.
842    ///
843    /// A half-open probe in flight is discarded with the rest: it was a
844    /// question about the old process.
845    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
846        self.lock()
847            .remove(module_id)
848            .map(|state| state.consecutive_timeouts)
849            .filter(|discarded| *discarded > 0)
850    }
851
852    /// Open breakers, for the `server.describe` counters object. `None` when
853    /// none is open, so the key stays absent rather than present-and-empty.
854    ///
855    /// This is the operator's answer to "is this module refusing instantly or
856    /// is it fine?", which look identical from a client that retries and then
857    /// succeeds.
858    fn open_snapshot(&self) -> Option<serde_json::Value> {
859        let now = Instant::now();
860        let modules = self.lock();
861        let open = modules
862            .iter()
863            .filter_map(|(module_id, state)| {
864                let cooldown_until = state.cooldown_until?;
865                Some((
866                    module_id.clone(),
867                    serde_json::json!({
868                        "consecutive_timeouts": state.consecutive_timeouts,
869                        "cooldown_remaining_ms":
870                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
871                        "probe_in_flight": state.probe_in_flight.is_some(),
872                    }),
873                ))
874            })
875            .collect::<serde_json::Map<String, serde_json::Value>>();
876        (!open.is_empty()).then_some(serde_json::Value::Object(open))
877    }
878}
879
880impl ControlHandler {
881    pub fn new(registry: Arc<Registry>) -> Self {
882        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
883    }
884
885    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
886        let counters = forwarding.counters();
887        // Taken from the forwarding table rather than created here, so that the
888        // breaker a `route.open` consults is the same one a module's
889        // registration resets, however many handlers are built over one table.
890        let route_bind_breakers = forwarding.route_bind_breakers();
891        let route_bind_concurrency = forwarding.route_bind_concurrency();
892        let route_outages = forwarding.route_outages();
893        Self {
894            registry,
895            activity: None,
896            activity_streams: Arc::new(Mutex::new(HashMap::new())),
897            forwarding,
898            process_liveness: None,
899            supervisor: SupervisorHandle::new(),
900            subc_capabilities: Arc::from([
901                CAP_MANIFEST_REGISTRATION.to_string(),
902                CAP_CHANNEL_LIFECYCLE.to_string(),
903                CAP_PING_PONG.to_string(),
904                CAP_SESSION_ATTACH.to_string(),
905                CAP_ADMISSION_FACTS_RELAY.to_string(),
906                CAP_SCOPES_V1.to_string(),
907                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
908                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
909                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
910                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
911                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
912                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
913            ]),
914            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
915            route_bind_relay_timeouts: BTreeMap::new(),
916            route_bind_breakers,
917            route_bind_concurrency,
918            route_outages,
919            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
920            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
921            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
922            storage_config: None,
923            machine_id: None,
924            admission_facts_carrier_module_id: None,
925            admission_facts_targets: None,
926            scopes: Arc::new(RwLock::new(ScopeTable::new(
927                crate::daemon_config::default_scope_authority_owners(),
928            ))),
929            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
930            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
931            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
932            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
933            #[cfg(test)]
934            caller_lookup: None,
935            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
936            rescan: None,
937            connected_clients: ConnectedClients::new(),
938            counters,
939            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
940            daemon_provenance: DaemonProvenanceFacts::default(),
941            #[cfg(test)]
942            control_dispatch_delay: None,
943            #[cfg(test)]
944            provenance_probe_override: None,
945            #[cfg(test)]
946            route_bind_relay_sent_hold: None,
947            #[cfg(test)]
948            hello_registry_to_forwarding_hold: None,
949        }
950    }
951
952    /// Set the central storage policy: registering modules then receive their
953    /// resolved storage descriptor in HELLO_ACK.
954    pub fn with_storage_config(
955        mut self,
956        storage_config: Option<crate::daemon_config::StorageConfig>,
957    ) -> Self {
958        self.storage_config = storage_config;
959        self
960    }
961
962    /// Open the ledger on its own writer thread; an unavailable sink does not
963    /// prevent the daemon from serving routing or supervisor requests.
964    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
965        self.activity = Some(crate::activity::ActivityLedger::open(
966            run_dir.join("activity.jsonl"),
967            incarnation,
968        ));
969        self.supervisor.set_activity(self.activity.clone());
970        self
971    }
972
973    /// Set the machine id served to every registering module (HELLO_ACK) and on
974    /// `server.describe`.
975    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
976        self.machine_id = machine_id;
977        self
978    }
979
980    /// Configure the exact reserved module and target ids permitted to relay
981    /// opaque admission facts. Config-file loading validates this authority;
982    /// this builder keeps the same policy available to embedded test daemons.
983    pub fn with_admission_facts_config(
984        mut self,
985        carrier_module_id: Option<String>,
986        targets: Option<Vec<String>>,
987    ) -> Self {
988        self.admission_facts_carrier_module_id = carrier_module_id;
989        self.admission_facts_targets = targets;
990        self
991    }
992
993    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
994    /// Replaces the scope table with an empty one under the new list, so call it
995    /// while building the handler, before any module can sync.
996    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
997        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
998        self.scope_authority_owners = owners;
999        self
1000    }
1001
1002    /// Set the startup policy before installing the rescan context or serving requests.
1003    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1004        self.privileged_operator_prompt = mode;
1005        self
1006    }
1007
1008    /// Override the route.bind relay timeout. Used by tests that assert the
1009    /// timeout path so they don't block on the production-safe default.
1010    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1011        self.route_bind_relay_timeout = timeout;
1012        self
1013    }
1014
1015    /// Install per-module route.bind relay budget overrides. A module id
1016    /// listed here wins over the daemon-wide default set via
1017    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1018    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1019    /// the same `Duration` the bind path will use.
1020    pub fn with_route_bind_relay_timeouts(
1021        mut self,
1022        timeouts: impl IntoIterator<Item = (String, Duration)>,
1023    ) -> Self {
1024        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1025        self
1026    }
1027
1028    /// Resolve the route.bind relay budget for a specific target module id.
1029    /// Per-module overrides win; the daemon-wide value (set via
1030    /// `with_route_bind_relay_timeout` or the built-in default) is the
1031    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1032    /// the same resolution `handle_route_open` will use.
1033    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1034        self.route_bind_relay_timeouts
1035            .get(module_id)
1036            .copied()
1037            .unwrap_or(self.route_bind_relay_timeout)
1038    }
1039
1040    /// Override the per-module bind-relay breaker policy.
1041    ///
1042    /// Used by tests, which cannot spend three production budgets opening a
1043    /// breaker or twenty seconds waiting for its cooldown. The production
1044    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1045    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1046    /// reasoning for the numbers.
1047    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1048        self.route_bind_breaker_threshold = threshold.max(1);
1049        self.route_bind_breaker_cooldown = cooldown;
1050        self
1051    }
1052
1053    #[cfg(test)]
1054    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1055        self.health_probe_timeout = timeout;
1056        self
1057    }
1058
1059    #[cfg(test)]
1060    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1061        self.control_dispatch_delay = Some(delay);
1062        self
1063    }
1064
1065    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1066    /// announce it is parked, once to be released. Build the barrier for two
1067    /// parties, the test being the other one.
1068    #[cfg(test)]
1069    pub(crate) fn with_route_bind_relay_sent_hold(
1070        mut self,
1071        hold: Arc<tokio::sync::Barrier>,
1072    ) -> Self {
1073        self.route_bind_relay_sent_hold = Some(hold);
1074        self
1075    }
1076
1077    /// A HELLO waits on `hold` twice between reserving its registry entry and
1078    /// installing its forwarding endpoint: once to announce it is parked, once
1079    /// to be released. Build the barrier for two parties, the test being the
1080    /// other one, and run the HELLO on a thread of its own.
1081    #[cfg(test)]
1082    pub(crate) fn with_hello_registry_to_forwarding_hold(
1083        mut self,
1084        hold: Arc<std::sync::Barrier>,
1085    ) -> Self {
1086        self.hello_registry_to_forwarding_hold = Some(hold);
1087        self
1088    }
1089
1090    #[cfg(test)]
1091    fn hold_between_registry_and_forwarding(&self) {
1092        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1093            hold.wait();
1094            hold.wait();
1095        }
1096    }
1097
1098    pub fn with_process_liveness(
1099        mut self,
1100        process_liveness: Arc<dyn ModuleProcessLiveness>,
1101    ) -> Self {
1102        self.process_liveness = Some(process_liveness);
1103        self
1104    }
1105
1106    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1107        self.supervisor = supervisor;
1108        self.supervisor.set_activity(self.activity.clone());
1109        self
1110    }
1111
1112    pub fn with_daemon_provenance(
1113        mut self,
1114        pid: u32,
1115        started_at_ms: u64,
1116        executable_path: Option<PathBuf>,
1117        build_git_sha: Option<String>,
1118        build_lock_digest: Option<String>,
1119    ) -> Self {
1120        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1121        let process_start_time = process_start_time(pid);
1122        self.daemon_provenance = DaemonProvenanceFacts {
1123            build: DaemonBuildProvenance {
1124                build_git_sha,
1125                build_lock_digest,
1126            },
1127            pid: Some(pid),
1128            started_at_ms: Some(started_at_ms),
1129            start_clock: None,
1130            executable_path,
1131            executable_identity,
1132            process_start_time,
1133            probe: ExecutableIdentityProbe::default(),
1134        };
1135        self
1136    }
1137
1138    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1139        self.daemon_provenance.start_clock = Some(clock);
1140        self
1141    }
1142
1143    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1144    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1145        self.scopes
1146            .write()
1147            .unwrap_or_else(|poisoned| poisoned.into_inner())
1148            .set_wall_clock(Arc::new(clock));
1149        self
1150    }
1151
1152    #[cfg(test)]
1153    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1154        self.provenance_probe_override = Some(result);
1155        self
1156    }
1157
1158    /// Install the configured module set and its reserved capability bindings.
1159    /// Bindings are configuration-scoped and may point at a provider that has not
1160    /// been installed yet, so this does not require the bound module to exist.
1161    pub fn with_capability_config(
1162        self,
1163        modules: impl IntoIterator<Item = (String, bool)>,
1164        reserved_capabilities: BTreeMap<String, String>,
1165    ) -> Self {
1166        self.capability_evaluator
1167            .configure(modules, reserved_capabilities);
1168        self
1169    }
1170
1171    pub fn with_supervisor_rescan(
1172        mut self,
1173        supervisor: Supervisor,
1174        config_path: impl Into<PathBuf>,
1175        configured_port: Option<u16>,
1176    ) -> Self {
1177        self.rescan = Some(SupervisorRescanContext {
1178            supervisor,
1179            config_path: config_path.into(),
1180            configured_port,
1181            storage_config: self.storage_config.clone(),
1182            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1183            admission_facts_targets: self.admission_facts_targets.clone(),
1184            scope_authority_owners: self.scope_authority_owners.clone(),
1185            privileged_operator_prompt: self.privileged_operator_prompt,
1186        });
1187        self
1188    }
1189
1190    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1191        self.connected_clients = connected_clients;
1192        self
1193    }
1194
1195    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1196        Arc::clone(&self.forwarding)
1197    }
1198
1199    pub(crate) fn counters(&self) -> DaemonCounters {
1200        self.counters.clone()
1201    }
1202
1203    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1204    /// requirement event without depending on an operator polling a status command.
1205    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1206        tokio::spawn(async move {
1207            loop {
1208                self.capability_evaluator
1209                    .wait_for_change_or_deadline()
1210                    .await;
1211                self.refresh_capability_requirements();
1212            }
1213        });
1214    }
1215
1216    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1217    /// deadline passed during sleep is enforced on the first tick after wake.
1218    /// While no live scope carries a deadline the loop waits for an accepted
1219    /// scope change instead of ticking, so a daemon without deadlines never
1220    /// wakes for expiry.
1221    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1222        tokio::spawn(async move {
1223            loop {
1224                let has_deadlines = self
1225                    .scopes
1226                    .read()
1227                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1228                    .has_deadlines();
1229                if !has_deadlines {
1230                    self.scope_deadline_added.notified().await;
1231                    continue;
1232                }
1233                tokio::time::sleep(Duration::from_secs(1)).await;
1234                if let Err(error) = self.sweep_expired_scopes() {
1235                    warn!(%error, "scope expiry sweep failed");
1236                }
1237            }
1238        });
1239    }
1240
1241    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1242        let mut table = self
1243            .scopes
1244            .write()
1245            .unwrap_or_else(|poisoned| poisoned.into_inner());
1246        let swept = table.sweep_expired();
1247        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1248        drop(table);
1249        self.close_scope_drained_routes(drained);
1250        Ok(())
1251    }
1252
1253    fn publish_scope_changes(
1254        &self,
1255        changes: &[crate::scopes::ScopeTagChange],
1256        expired: &[crate::scopes::ScopeExpired],
1257    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1258        if changes.is_empty() {
1259            return Ok(Vec::new());
1260        }
1261        let drained = self
1262            .forwarding
1263            .publish_scope_changes(changes)
1264            .map_err(RouterError::Forwarding)?;
1265        let mut counts = HashMap::new();
1266        for route in &drained {
1267            *counts
1268                .entry((
1269                    route.scope.owner.as_str(),
1270                    route.scope.scope_ref.as_str(),
1271                    route.scope.tag.scope_epoch,
1272                ))
1273                .or_insert(0usize) += 1;
1274        }
1275        for scope in expired {
1276            let routes_closed = counts
1277                .get(&(
1278                    scope.owner.as_str(),
1279                    scope.scope_ref.as_str(),
1280                    scope.scope_epoch,
1281                ))
1282                .copied()
1283                .unwrap_or(0);
1284            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1285                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1286                routes_closed, "scope expired");
1287        }
1288        Ok(drained)
1289    }
1290
1291    fn runtime_capability_snapshot(
1292        &self,
1293    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1294        let runtime = self
1295            .supervisor
1296            .list()
1297            .into_iter()
1298            .map(|module| {
1299                let status = module.status().map_err(|err| {
1300                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1301                })?;
1302                Ok(RuntimeModule {
1303                    module_id: status.module_id,
1304                    state: status.state,
1305                    enabled: status.enabled,
1306                })
1307            })
1308            .collect::<Result<Vec<_>, RouterError>>()?;
1309        // Include every registration, published or not. The capability
1310        // evaluator counts a module as providing its capabilities as soon as it
1311        // has registered, even before `catalog.list` shows it, so a HELLO in
1312        // progress does not change other modules' required-capability
1313        // verdicts. See `Registry::list_registered_modules`.
1314        let registrations = self.registry.list_registered_modules().map_err(|err| {
1315            RouterError::backend(
1316                0,
1317                0,
1318                format!("failed to list capability registrations: {err}"),
1319            )
1320        })?;
1321        let registrations = registrations
1322            .into_iter()
1323            .map(|registration| RegisteredModule {
1324                module_id: registration.manifest.module_id,
1325                module_version: registration.manifest.module_version,
1326                capabilities: registration.manifest.capabilities,
1327            })
1328            .collect();
1329        Ok((runtime, registrations))
1330    }
1331
1332    /// The capability side effects of a module becoming the active registration
1333    /// for its id: cache its manifest (warning if its claims drifted), run the
1334    /// deny census when its declarations call for one, and recompute the
1335    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1336    /// candidate's does not, and the supervisor does it at promotion instead,
1337    /// through [`crate::supervise::SwapPromotionObserver`].
1338    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1339        let cached_registration = RegisteredModule {
1340            module_id: registration.manifest.module_id.clone(),
1341            module_version: registration.manifest.module_version.clone(),
1342            capabilities: registration.manifest.capabilities.clone(),
1343        };
1344        if self.capability_evaluator.record_hello(&cached_registration) {
1345            warn!(
1346                module_id = %cached_registration.module_id,
1347                "capability claims drifted from the cached manifest"
1348            );
1349        }
1350        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1351            self.enforce_capability_denies();
1352        }
1353        self.refresh_capability_requirements();
1354    }
1355
1356    /// Point the shared supervisor handle at this handler for swap promotions.
1357    /// Called wherever a handler is put behind the `Arc` the router serves, so
1358    /// it can be held weakly.
1359    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1360        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1361            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1362        self.supervisor.set_swap_promotion_observer(observer);
1363    }
1364
1365    pub fn refresh_capability_requirements(&self) {
1366        match self.runtime_capability_snapshot() {
1367            Ok((runtime, registrations)) => {
1368                log_requirement_events(
1369                    self.capability_evaluator
1370                        .evaluate_now(&runtime, &registrations),
1371                );
1372            }
1373            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1374        }
1375    }
1376
1377    /// Reconcile only live, attested route bindings after a capability deny edge
1378    /// or target claim was added. This is deliberately a control-plane census:
1379    /// the opaque forwarding hot path must not grow a per-frame capability check.
1380    fn enforce_capability_denies(&self) {
1381        let registrations = match self.registry.list_registered_modules() {
1382            Ok(snapshot) => snapshot,
1383            Err(err) => {
1384                warn!(error = %err, "failed to read registrations for capability deny census");
1385                return;
1386            }
1387        };
1388        let manifests = registrations
1389            .into_iter()
1390            .map(|registration| {
1391                (
1392                    registration.manifest.module_id.clone(),
1393                    registration.manifest,
1394                )
1395            })
1396            .collect::<BTreeMap<_, _>>();
1397        let census = match self.forwarding.route_census(None) {
1398            Ok(census) => census,
1399            Err(err) => {
1400                warn!(error = %err, "failed to read route census for capability deny enforcement");
1401                return;
1402            }
1403        };
1404
1405        for (target_module_id, routes) in census {
1406            let Some(target_manifest) = manifests.get(&target_module_id) else {
1407                continue;
1408            };
1409            let mut closed_routes = Vec::new();
1410            let mut module_goodbyes = Vec::new();
1411            for route in routes {
1412                let Principal::Reserved {
1413                    module_id: opening_module_id,
1414                } = &route.principal
1415                else {
1416                    continue;
1417                };
1418                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1419                    continue;
1420                };
1421                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1422                    continue;
1423                };
1424
1425                match self.forwarding.release_client_route(
1426                    route.goodbye_target.connection_id,
1427                    route.goodbye_target.channel,
1428                    route.goodbye_target.epoch,
1429                ) {
1430                    Ok(RouteRelease::Removed(module_goodbye)) => {
1431                        warn!(
1432                            opening_module_id,
1433                            target_module_id,
1434                            capability,
1435                            "force-closing route because an attested capability deny edge now matches"
1436                        );
1437                        closed_routes.push(route);
1438                        module_goodbyes.push(module_goodbye);
1439                    }
1440                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1441                    Err(err) => warn!(
1442                        opening_module_id,
1443                        target_module_id,
1444                        capability,
1445                        error = %err,
1446                        "failed to force-close capability-denied route"
1447                    ),
1448                }
1449            }
1450
1451            if closed_routes.is_empty() {
1452                continue;
1453            }
1454            send_route_control_pushes(
1455                &self.forwarding,
1456                closed_routes,
1457                ClientControlPush::RouteClosed {
1458                    module_id: target_module_id,
1459                    channels: Vec::new(),
1460                    reason: RouteCloseReason::CapabilityDenied,
1461                    drained: false,
1462                    abandoned: 0,
1463                    excluded_subscriptions: 0,
1464                    terminal: Some(false),
1465                },
1466            );
1467            self.emit_route_goodbyes(module_goodbyes);
1468        }
1469    }
1470
1471    /// Why a registered module is not accepting new route binds, or `None` when
1472    /// it is. This is the module's effective readiness: its declared readiness
1473    /// first, then every `need: required` capability it declares evaluating to
1474    /// `provided`. `route.open` and `catalog.list` both read it here so the
1475    /// catalog never reports a module routable that `route.open` would refuse.
1476    fn not_ready_reason(
1477        &self,
1478        registration: &crate::registry::ModuleRegistration,
1479    ) -> Option<NotReadyReason> {
1480        if !registration.ready {
1481            return Some(NotReadyReason {
1482                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1483                capability: None,
1484            });
1485        }
1486        self.first_unprovided_required_capability(registration)
1487            .map(|capability| NotReadyReason {
1488                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1489                capability: Some(capability),
1490            })
1491    }
1492
1493    /// The lexicographically first capability this registration declares
1494    /// `need: required` whose evaluator verdict is not `provided`.
1495    ///
1496    /// The verdicts are the capability evaluator's own; nothing here decides
1497    /// what "provided" means. The evaluator counts a capability provided as
1498    /// soon as a module claiming it has REGISTERED, not once that module is
1499    /// ready. That distinction is what keeps two modules that require each
1500    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1501    /// is ready", each would wait for the other to become ready first and
1502    /// neither ever would. Do not tighten it to readiness.
1503    ///
1504    /// A required capability with no verdict at all means this registration's
1505    /// HELLO or catalog.update landed after the last recompute; recompute once
1506    /// rather than let a missing verdict read as either answer. If it is still
1507    /// missing (the recompute itself failed) the capability counts as
1508    /// unprovided: the refusal is retryable, and routing a module whose
1509    /// required provider is unknown is the outcome this check exists to stop.
1510    fn first_unprovided_required_capability(
1511        &self,
1512        registration: &crate::registry::ModuleRegistration,
1513    ) -> Option<String> {
1514        let required = registration
1515            .manifest
1516            .capabilities
1517            .iter()
1518            .flat_map(|declarations| declarations.requires.iter())
1519            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1520            .map(|requirement| requirement.capability.as_str())
1521            .collect::<BTreeSet<_>>();
1522        if required.is_empty() {
1523            return None;
1524        }
1525        let module_id = registration.manifest.module_id.as_str();
1526        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1527        if required
1528            .iter()
1529            .any(|capability| verdict(capability).is_none())
1530        {
1531            self.refresh_capability_requirements();
1532        }
1533        required
1534            .into_iter()
1535            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1536            .map(str::to_string)
1537    }
1538
1539    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1540        self.capability_evaluator
1541            .statuses()
1542            .into_iter()
1543            .map(capability_requirement_status)
1544            .collect()
1545    }
1546
1547    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1548    /// registration-release watch. The signal is what the supervisor waits on
1549    /// before spawning a replacement, so it must only fire once forwarding
1550    /// teardown is also done (see [`Self::cleanup_connection`] /
1551    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1552    /// admitted but its forwarding endpoint could not be installed.
1553    fn deregister_connection(
1554        &self,
1555        connection_id: ConnectionId,
1556        reason: RegistrationEndReason,
1557    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1558        self.registry
1559            .deregister_connection_with_reason(connection_id, reason)
1560    }
1561
1562    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1563        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1564            return None;
1565        }
1566        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1567            parse_client_control_request(&frame.body)
1568        else {
1569            return None;
1570        };
1571        Some(target_module_id(&target).to_string())
1572    }
1573
1574    pub(crate) fn route_open_capacity_refusal(
1575        &self,
1576        ctx: &RouteCtx,
1577        frame: &Frame,
1578        target_module_id: &str,
1579        in_flight: usize,
1580        limit: usize,
1581    ) -> Result<Frame, RouterError> {
1582        self.route_open_admission_refusal_frame(
1583            ctx,
1584            frame,
1585            target_module_id,
1586            "open_admission_full",
1587            (in_flight, limit),
1588            format!(
1589                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1590            ),
1591        )
1592    }
1593
1594    fn route_open_target_capacity_refusal(
1595        &self,
1596        ctx: &RouteCtx,
1597        frame: &Frame,
1598        target_module_id: &str,
1599        in_flight: usize,
1600    ) -> Result<Frame, RouterError> {
1601        self.route_open_admission_refusal_frame(
1602            ctx,
1603            frame,
1604            target_module_id,
1605            "target_binds_full",
1606            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1607            format!(
1608                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1609            ),
1610        )
1611    }
1612
1613    /// Admission pressure clears as existing binds settle, so its refusal must
1614    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1615    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1616    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1617    /// cannot currently reach its target; `module_timeout` would falsely claim
1618    /// that a wait expired. A new, cleaner code would be terminal to deployed
1619    /// clients, so it requires a client-tolerance rollout before daemon emission.
1620    fn route_open_admission_refusal_frame(
1621        &self,
1622        ctx: &RouteCtx,
1623        frame: &Frame,
1624        target_module_id: &str,
1625        reason: &'static str,
1626        (in_flight, limit): (usize, usize),
1627        message: impl Into<String>,
1628    ) -> Result<Frame, RouterError> {
1629        let code = error_codes::TARGET_UNAVAILABLE;
1630        self.counters.increment_route_open_refused(code);
1631        info!(
1632            target: "control",
1633            code,
1634            reason,
1635            module_id = ?target_module_id,
1636            connection_id = ctx.connection_id.get(),
1637            in_flight,
1638            limit,
1639            "route.open refused"
1640        );
1641        control_error_frame(frame, code, message.into())
1642    }
1643
1644    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1645    ///
1646    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1647    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1648    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1649    #[cfg(test)]
1650    pub fn handle_control(
1651        &self,
1652        connection_id: ConnectionId,
1653        frame: Frame,
1654    ) -> Result<Vec<Frame>, RouterError> {
1655        match frame.header.ty {
1656            FrameType::Ping => Ok(vec![pong(&frame)?]),
1657            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1658            FrameType::Goodbye => self.handle_goodbye(connection_id),
1659            ty => Ok(vec![control_error_frame(
1660                &frame,
1661                "unsupported_control_frame",
1662                format!("unsupported channel-0 frame {ty:?}"),
1663            )?]),
1664        }
1665    }
1666
1667    pub async fn handle_control_frame(
1668        &self,
1669        ctx: &RouteCtx,
1670        frame: Frame,
1671    ) -> Result<Vec<Frame>, RouterError> {
1672        self.handle_control_frame_timed(ctx, frame, None).await
1673    }
1674
1675    pub(crate) fn record_caller_endpoints(
1676        &self,
1677        connection: ConnectionId,
1678        endpoints: Option<(SocketAddr, SocketAddr)>,
1679    ) {
1680        let mut callers = self
1681            .caller_endpoints
1682            .lock()
1683            .unwrap_or_else(|p| p.into_inner());
1684        callers.remove(&connection);
1685        if let Some(endpoints) = endpoints {
1686            callers.insert(connection, endpoints);
1687        }
1688    }
1689
1690    #[cfg(test)]
1691    pub(crate) fn with_caller_lookup_for_test(
1692        mut self,
1693        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1694    ) -> Self {
1695        self.caller_lookup = Some(Arc::new(lookup));
1696        self
1697    }
1698
1699    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1700        let endpoints = self
1701            .caller_endpoints
1702            .lock()
1703            .unwrap_or_else(|p| p.into_inner())
1704            .get(&connection)
1705            .copied();
1706        #[cfg(test)]
1707        let lookup = self.caller_lookup.clone();
1708        tokio::task::spawn_blocking(move || {
1709            #[cfg(test)]
1710            if let Some(lookup) = lookup {
1711                return lookup(endpoints);
1712            }
1713            endpoints
1714                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1715                .unwrap_or_default()
1716        })
1717        .await
1718        .unwrap_or_default()
1719    }
1720
1721    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1722        frame.header.channel == 0
1723            && frame.header.ty == FrameType::Request
1724            && self
1725                .forwarding
1726                .module_endpoint_for_connection(ctx.connection_id)
1727                .is_ok_and(|endpoint| endpoint.is_none())
1728            && parse_client_control_request(&frame.body)
1729                .ok()
1730                .is_some_and(|request| privileged_operation(&request).is_some())
1731    }
1732
1733    pub(crate) async fn handle_control_frame_timed(
1734        &self,
1735        ctx: &RouteCtx,
1736        frame: Frame,
1737        dispatch_started_at: Option<StdInstant>,
1738    ) -> Result<Vec<Frame>, RouterError> {
1739        match frame.header.ty {
1740            FrameType::Ping => Ok(vec![pong(&frame)?]),
1741            FrameType::Hello => {
1742                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1743            }
1744            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1745            FrameType::Cancel => {
1746                // A Cancel on channel 0 names a waiting operator.confirm or a
1747                // subscription. Successful cancellation has no ordinary reply.
1748                if self
1749                    .forwarding
1750                    .operator_confirms()
1751                    .cancel(ctx.connection_id, frame.header.corr)
1752                    || self
1753                        .supervisor
1754                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1755                    || self.cancel_activity_subscription(ctx, &frame)?
1756                {
1757                    Ok(Vec::new())
1758                } else {
1759                    Ok(vec![control_error_frame(
1760                        &frame,
1761                        "unknown_subscription",
1762                        "no subscription has this correlation id",
1763                    )?])
1764                }
1765            }
1766            FrameType::Request => {
1767                // This additive operation is not part of the existing exhaustive
1768                // module-control enum. Probe the op before decoding that enum.
1769                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1770                if op
1771                    .as_ref()
1772                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1773                {
1774                    return self.handle_activity_record(ctx, frame).await;
1775                }
1776                if op.as_ref().is_some_and(|probe| {
1777                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1778                }) {
1779                    return self.handle_activity_read(ctx, frame).await;
1780                }
1781                if op
1782                    .as_ref()
1783                    .is_some_and(|probe| probe.op == "operator.confirm")
1784                {
1785                    return self.handle_operator_confirm(ctx, frame);
1786                }
1787                if self
1788                    .forwarding
1789                    .module_endpoint_for_connection(ctx.connection_id)
1790                    .map_err(RouterError::Forwarding)?
1791                    .is_some()
1792                {
1793                    if !is_known_module_request_op(&frame.body) {
1794                        return Ok(vec![control_error_frame(
1795                            &frame,
1796                            "unsupported_control_frame",
1797                            "module-originated channel-0 REQUEST is not supported",
1798                        )?]);
1799                    }
1800                    let request = match parse_module_control_request_from_module(&frame.body) {
1801                        Ok(request) => request,
1802                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1803                            return Ok(vec![control_error_frame(
1804                                &frame,
1805                                "unsupported_control_frame",
1806                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1807                            )?])
1808                        }
1809                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1810                            return Ok(vec![control_error_frame(
1811                                &frame,
1812                                "invalid_control_body",
1813                                format!("malformed module control body: {err}"),
1814                            )?])
1815                        }
1816                    };
1817                    let op = module_control_request_op(&request);
1818                    let corr = frame.header.corr;
1819                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1820                    let result =
1821                        self.handle_module_control_request(ctx.connection_id, frame, request);
1822                    log_slow_control_dispatch(
1823                        dispatch_started_at,
1824                        op,
1825                        ctx.connection_id,
1826                        corr,
1827                        None,
1828                    );
1829                    return result;
1830                }
1831
1832                if is_known_module_request_op(&frame.body) {
1833                    return Ok(vec![control_error_frame(
1834                        &frame,
1835                        "not_registered",
1836                        "catalog.update requires an active module registration owned by this connection",
1837                    )?]);
1838                }
1839
1840                let request = match parse_client_control_request(&frame.body) {
1841                    Ok(request) => request,
1842                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1843                        return Ok(vec![control_error_frame(
1844                            &frame,
1845                            "unknown_control_op",
1846                            format!("unknown client control op: {err}"),
1847                        )?])
1848                    }
1849                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1850                        return Ok(vec![control_error_frame(
1851                            &frame,
1852                            "invalid_control_body",
1853                            format!("malformed client control body: {err}"),
1854                        )?])
1855                    }
1856                };
1857                let op = client_control_request_op(&request);
1858                let corr = frame.header.corr;
1859                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1860                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1861                #[cfg(test)]
1862                if let Some(delay) = self.control_dispatch_delay {
1863                    tokio::time::sleep(delay).await;
1864                }
1865                let result = self
1866                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1867                    .await;
1868                if let Some(timings) = route_open_timings.as_mut() {
1869                    timings.finish();
1870                }
1871                log_slow_control_dispatch(
1872                    dispatch_started_at,
1873                    op,
1874                    ctx.connection_id,
1875                    corr,
1876                    route_open_timings.as_ref(),
1877                );
1878                result
1879            }
1880            FrameType::Push => {
1881                let Some(endpoint) = self
1882                    .forwarding
1883                    .module_endpoint_for_connection(ctx.connection_id)
1884                    .map_err(RouterError::Forwarding)?
1885                else {
1886                    return Ok(vec![control_error_frame(
1887                        &frame,
1888                        "unsupported_control_frame",
1889                        "client-originated channel-0 PUSH is not supported",
1890                    )?]);
1891                };
1892                self.handle_status_update(endpoint, frame)
1893            }
1894            FrameType::Response | FrameType::Error
1895                if self
1896                    .forwarding
1897                    .module_endpoint_for_connection(ctx.connection_id)
1898                    .map_err(RouterError::Forwarding)?
1899                    .is_some() =>
1900            {
1901                self.handle_module_relay_response(ctx.connection_id, frame)
1902            }
1903            ty => Ok(vec![control_error_frame(
1904                &frame,
1905                "unsupported_control_frame",
1906                format!("unsupported channel-0 frame {ty:?}"),
1907            )?]),
1908        }
1909    }
1910
1911    pub fn cleanup_connection(
1912        &self,
1913        connection_id: ConnectionId,
1914    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1915        self.caller_endpoints
1916            .lock()
1917            .unwrap_or_else(|p| p.into_inner())
1918            .remove(&connection_id);
1919        self.forwarding
1920            .operator_confirms()
1921            .connection_closed(connection_id);
1922        self.cleanup_connection_with_end_reason(
1923            connection_id,
1924            RegistrationEndReason::ConnectionClosed,
1925        )
1926    }
1927
1928    fn cleanup_connection_with_end_reason(
1929        &self,
1930        connection_id: ConnectionId,
1931        requested_reason: RegistrationEndReason,
1932    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1933        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1934            self.registry
1935                .get_module_by_connection(connection_id)?
1936                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
1937                .and_then(|module| module.registration_end_reason().ok().flatten())
1938                .unwrap_or(requested_reason)
1939        } else {
1940            requested_reason
1941        };
1942        let crash_closed = self
1943            .registry
1944            .get_module_by_connection(connection_id)?
1945            .and_then(|registration| {
1946                self.forwarding
1947                    .module_endpoint_for_connection(connection_id)
1948                    .ok()
1949                    .flatten()
1950                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1951                    .map(|routes| (registration.manifest.module_id, routes))
1952            });
1953        let crash_closed = crash_closed.map(|(module_id, routes)| {
1954            let terminal = match self.supervisor.get(&module_id) {
1955                None => false,
1956                Some(module) => match module.will_recover_after_connection_loss() {
1957                    Ok(will_recover) => !will_recover,
1958                    Err(err) => {
1959                        warn!(
1960                            %module_id,
1961                            error = %err,
1962                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
1963                        );
1964                        false
1965                    }
1966                },
1967            };
1968            // The forwarding table gates all providers at the start of daemon
1969            // shutdown, before their connections are closed. An ordinary
1970            // module disconnect still reports crash if that gate is not set.
1971            let (reason, drained) = match self
1972                .forwarding
1973                .daemon_drain_result_for_connection(connection_id)
1974            {
1975                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
1976                Ok(None) => (RouteCloseReason::Crash, false),
1977                Err(err) => {
1978                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1979                    (RouteCloseReason::Crash, false)
1980                }
1981            };
1982            (module_id, routes, reason, drained, terminal)
1983        });
1984        let registrations = self.deregister_connection(connection_id, end_reason);
1985        let cleanup = if crash_closed.is_some() {
1986            self.forwarding.cleanup_connection_counted(connection_id)
1987        } else {
1988            // A module connection's teardown needs the count of abandoned
1989            // route.bind relays for its route.closed notice below. Any other
1990            // connection, such as a client's, sends no such notice and needs
1991            // only its routes released, so it uses the route-only wrapper and
1992            // reports zero.
1993            self.forwarding
1994                .cleanup_connection(connection_id)
1995                .map(|released| crate::forwarding::ConnectionCleanup {
1996                    released,
1997                    abandoned_relays: 0,
1998                })
1999        };
2000        // The route.closed push waits for forwarding teardown because only
2001        // teardown knows how many pending route.bind relays it aborted. It still
2002        // goes out before the GOODBYEs for the released routes, and its targets
2003        // were captured above, before teardown removed those routes.
2004        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2005            let abandoned = cleanup
2006                .as_ref()
2007                .map_or(0, |cleanup| cleanup.abandoned_relays);
2008            send_route_control_pushes(
2009                &self.forwarding,
2010                routes,
2011                ClientControlPush::RouteClosed {
2012                    module_id,
2013                    channels: Vec::new(),
2014                    reason,
2015                    drained,
2016                    abandoned,
2017                    excluded_subscriptions: 0,
2018                    terminal: Some(terminal),
2019                },
2020            );
2021        }
2022        if let Ok(cleanup) = cleanup {
2023            self.emit_route_goodbyes(cleanup.released);
2024        }
2025        // Signal the registration-release watch only now that BOTH registry and
2026        // forwarding teardown are done, so a supervisor waiting to spawn a
2027        // replacement never observes release while old routes still exist.
2028        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2029            crate::supervise::notify_registration_release();
2030            self.capability_evaluator.wake_deadline_loop();
2031            self.refresh_capability_requirements();
2032        }
2033        self.supervisor.remove_spawn_subscribers(connection_id);
2034        self.activity_streams
2035            .lock()
2036            .unwrap_or_else(|p| p.into_inner())
2037            .retain(|(id, _), task| {
2038                if *id == connection_id {
2039                    task.abort();
2040                    false
2041                } else {
2042                    true
2043                }
2044            });
2045        // Sync authority dies with its connection, so the owner's next
2046        // connection can take it; the owner's scopes stay as they are.
2047        self.hello_launch_nonces
2048            .lock()
2049            .unwrap_or_else(|poisoned| poisoned.into_inner())
2050            .forget(connection_id);
2051        self.scopes
2052            .write()
2053            .unwrap_or_else(|poisoned| poisoned.into_inner())
2054            .release_connection(connection_id);
2055        registrations
2056    }
2057
2058    pub(crate) fn handle_route_goodbye(
2059        &self,
2060        connection_id: ConnectionId,
2061        route_channel: u16,
2062        route_epoch: u32,
2063    ) -> Result<bool, RouterError> {
2064        debug!(
2065            connection_id = connection_id.get(),
2066            route_channel, route_epoch, "handling route GOODBYE"
2067        );
2068        let RouteRelease::Removed(released_route) = self
2069            .forwarding
2070            .release_client_route(connection_id, route_channel, route_epoch)
2071            .map_err(RouterError::Forwarding)?
2072        else {
2073            return Ok(false);
2074        };
2075        self.emit_route_goodbyes(vec![released_route]);
2076        Ok(true)
2077    }
2078
2079    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2080        for released in released_routes {
2081            let frame = match Frame::build_with_version(
2082                released.negotiated_ver,
2083                FrameType::Goodbye,
2084                control_flags(),
2085                released.channel,
2086                released.epoch,
2087                0,
2088                Vec::new(),
2089            ) {
2090                Ok(frame) => frame,
2091                Err(err) => {
2092                    warn!(
2093                        route_channel = released.channel,
2094                        error = %err,
2095                        "failed to build route GOODBYE frame"
2096                    );
2097                    continue;
2098                }
2099            };
2100            if !released.close_on_delivery_failure() {
2101                crate::forwarding::send_module_route_goodbye(
2102                    &self.counters,
2103                    &released.sink,
2104                    frame,
2105                    released.module_id.as_deref(),
2106                    "client route released",
2107                );
2108                continue;
2109            }
2110            if let Err(err) = released.sink.try_send(frame) {
2111                warn!(
2112                    target_connection_id = released.connection_id.get(),
2113                    route_channel = released.channel,
2114                    error = %err,
2115                    "route GOODBYE was not delivered to client; closing target connection"
2116                );
2117                if self
2118                    .forwarding
2119                    .escalate_client_delivery_failure(
2120                        released.connection_id,
2121                        released.channel,
2122                        released.epoch,
2123                        CloseReason::new(
2124                            "route_goodbye_delivery_failed",
2125                            format!(
2126                                "failed to enqueue route GOODBYE for channel {}: {err}",
2127                                released.channel
2128                            ),
2129                        ),
2130                        crate::forwarding::UndeliveredFrame {
2131                            module_id: released.module_id.as_deref(),
2132                            sink: &released.sink,
2133                        },
2134                    )
2135                    .unwrap_or(false)
2136                {
2137                    self.counters.increment_goodbye_relay_client_failed();
2138                }
2139            }
2140        }
2141    }
2142
2143    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
2144    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
2145    /// own commit failed after the module had already accepted). Without this, a
2146    /// module that accepts late keeps a binding subc has torn down, so a later
2147    /// frame on that module channel could misdeliver if the channel is reused.
2148    ///
2149    /// Never closes the shared module connection on failure: a dropped notification
2150    /// only wastes a bounded amount of warm module-side state, which the module's
2151    /// own idle reaper reclaims. Only call this once the route.bind relay was
2152    /// actually enqueued to the module — if the relay send itself failed, the
2153    /// module never created a binding and there is nothing to tear down.
2154    fn send_abandoned_route_bind_goodbye(
2155        &self,
2156        module_sink: &crate::FrameSink,
2157        negotiated_ver: u8,
2158        module_channel: u16,
2159        module_epoch: u32,
2160    ) {
2161        let frame = match Frame::build_with_version(
2162            negotiated_ver,
2163            FrameType::Goodbye,
2164            control_flags(),
2165            module_channel,
2166            module_epoch,
2167            0,
2168            Vec::new(),
2169        ) {
2170            Ok(frame) => frame,
2171            Err(err) => {
2172                warn!(
2173                    route_channel = module_channel,
2174                    error = %err,
2175                    "failed to build GOODBYE for abandoned route.bind"
2176                );
2177                return;
2178            }
2179        };
2180        crate::forwarding::send_module_route_goodbye(
2181            &self.counters,
2182            module_sink,
2183            frame,
2184            None,
2185            "abandoned route.bind",
2186        );
2187    }
2188
2189    fn handle_hello(
2190        &self,
2191        connection_id: ConnectionId,
2192        sink: Option<crate::FrameSink>,
2193        frame: Frame,
2194    ) -> Result<Vec<Frame>, RouterError> {
2195        debug!(
2196            connection_id = connection_id.get(),
2197            corr = frame.header.corr,
2198            "handling HELLO"
2199        );
2200        // A module connection has one identity for its entire lifetime. A second
2201        // registration would leave the old registry owner behind while replacing
2202        // its forwarding endpoint and launch nonce.
2203        if self
2204            .registry
2205            .get_module_by_connection(connection_id)
2206            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2207            .is_some()
2208        {
2209            return Ok(vec![control_error_frame(
2210                &frame,
2211                "invalid_hello",
2212                "connection is already registered as a module",
2213            )?]);
2214        }
2215        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2216            Ok(value) => value,
2217            Err(err) => {
2218                return Ok(vec![control_error_frame(
2219                    &frame,
2220                    "invalid_hello",
2221                    format!("malformed HELLO body: {err}"),
2222                )?])
2223            }
2224        };
2225        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2226            return Ok(vec![control_error_frame(
2227                &frame,
2228                "invalid_capability_grammar",
2229                err.to_string(),
2230            )?]);
2231        }
2232        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2233            return Ok(vec![control_error_frame(
2234                &frame,
2235                "invalid_manifest",
2236                err.to_string(),
2237            )?]);
2238        }
2239        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2240            return Ok(vec![control_error_frame(
2241                &frame,
2242                "invalid_event_declaration",
2243                err.to_string(),
2244            )?]);
2245        }
2246        if let Some(provenance) = hello_value
2247            .get("manifest")
2248            .and_then(|manifest| manifest.get("provenance"))
2249        {
2250            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2251                return Ok(vec![control_error_frame(
2252                    &frame,
2253                    "invalid_manifest",
2254                    format!("malformed manifest provenance: {err}"),
2255                )?]);
2256            }
2257        }
2258        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2259            Ok(hello) => hello,
2260            Err(err) => {
2261                return Ok(vec![control_error_frame(
2262                    &frame,
2263                    "invalid_hello",
2264                    format!("malformed HELLO body: {err}"),
2265                )?])
2266            }
2267        };
2268
2269        if hello.protocol_ver != hello.manifest.protocol_ver {
2270            return Ok(vec![control_error_frame(
2271                &frame,
2272                "invalid_manifest",
2273                format!(
2274                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2275                    hello.protocol_ver, hello.manifest.protocol_ver
2276                ),
2277            )?]);
2278        }
2279
2280        if hello.manifest.module_id.trim().is_empty() {
2281            return Ok(vec![control_error_frame(
2282                &frame,
2283                "invalid_manifest",
2284                "manifest module_id must not be empty",
2285            )?]);
2286        }
2287
2288        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2289            Ok(negotiated_ver) => negotiated_ver,
2290            Err(message) => {
2291                return Ok(vec![control_error_frame(
2292                    &frame,
2293                    "version_unsupported",
2294                    message,
2295                )?])
2296            }
2297        };
2298
2299        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2300        // swap is open for this id, the only HELLO admitted as a second process
2301        // is the one carrying the candidate's launch nonce (the swap token), and
2302        // it registers into the candidate slot rather than being refused as a
2303        // duplicate. Run after the reserved gate, a reserved module's candidate
2304        // would be refused `reserved_module` for presenting a nonce that gate
2305        // does not know. See `SupervisorHandle::swap_hello_admission`.
2306        let swap_admission = self
2307            .supervisor
2308            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2309        if swap_admission == SwapHelloAdmission::Refused {
2310            warn!(
2311                module_id = %hello.manifest.module_id,
2312                connection_id = connection_id.get(),
2313                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2314            );
2315            return Ok(vec![control_error_frame(
2316                &frame,
2317                "swap_token_invalid",
2318                format!(
2319                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2320                    hello.manifest.module_id
2321                ),
2322            )?]);
2323        }
2324        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2325
2326        // Reserved-module identity gate: a module_id configured `reserved` may be
2327        // registered ONLY by the process subc spawned for it, proven by echoing the
2328        // one-time launch nonce subc injected. A non-reserved id has no recorded
2329        // nonce and always passes. This blocks a key-holder from impersonating a
2330        // security-boundary module (e.g. the credential vault) while the real one is
2331        // down/restarting and its registration slot is momentarily free. A swap
2332        // candidate has already proven the same thing with its own nonce above.
2333        if let Some(rejection) = (!swap_candidate)
2334            .then(|| {
2335                self.supervisor.reserved_hello_rejection(
2336                    &hello.manifest.module_id,
2337                    hello.launch_nonce.as_deref(),
2338                )
2339            })
2340            .flatten()
2341        {
2342            let message = match rejection {
2343                ReservedHelloRejection::Exact { module_id } => format!(
2344                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2345                ),
2346                ReservedHelloRejection::Prefix {
2347                    prefix,
2348                    owner_module_id,
2349                } => format!(
2350                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2351                    hello.manifest.module_id
2352                ),
2353            };
2354            return Ok(vec![control_error_frame(
2355                &frame,
2356                "reserved_module",
2357                message,
2358            )?]);
2359        }
2360
2361        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2362            &hello.manifest.module_id,
2363            hello.manifest.capabilities.as_ref(),
2364        );
2365        if let Some(refusal) = reserved_capability_refusals.first() {
2366            let capability = refusal.capability.clone();
2367            let bound_module = refusal.claimants[0].clone();
2368            log_duplicate_claim_events(reserved_capability_refusals);
2369            return Ok(vec![control_error_frame(
2370                &frame,
2371                "reserved_capability",
2372                format!(
2373                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2374                    capability, bound_module, hello.manifest.module_id
2375                ),
2376            )?]);
2377        }
2378
2379        // A connection that already opened client routes must not also register as
2380        // a module: cleanup would then release only one side and leak the other.
2381        if self
2382            .forwarding
2383            .connection_has_client_routes(connection_id)
2384            .map_err(RouterError::Forwarding)?
2385        {
2386            return Ok(vec![control_error_frame(
2387                &frame,
2388                "invalid_hello",
2389                "connection has open client routes and cannot also register as a module",
2390            )?]);
2391        }
2392
2393        // Kept for scope sync authority, which goes only to the connection that
2394        // presented the module's current launch nonce. Recorded before the
2395        // registration is attempted: a connection whose registration then fails
2396        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2397        self.hello_launch_nonces
2398            .lock()
2399            .unwrap_or_else(|poisoned| poisoned.into_inner())
2400            .record(connection_id, hello.launch_nonce.as_deref());
2401        let control_ops = effective_module_control_ops(hello.control_ops);
2402        // Built before anything is registered so an encoding failure leaves no
2403        // registry or forwarding state behind.
2404        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2405        if swap_candidate {
2406            return self.register_swap_candidate(
2407                connection_id,
2408                sink,
2409                &frame,
2410                hello.manifest,
2411                negotiated_ver,
2412                control_ops,
2413                hello_ack,
2414            );
2415        }
2416        // Registered unpublished: the module id is claimed, so a duplicate HELLO
2417        // is refused, but `catalog.list` does not show the module and the
2418        // catalog generation does not move until it is published below, once
2419        // its forwarding endpoint exists. Listing it now would advertise a
2420        // module that `route.open` still refuses for having no forwarding
2421        // connection.
2422        let registration = match self.registry.register_unpublished_with_control_ops(
2423            hello.manifest,
2424            negotiated_ver,
2425            connection_id,
2426            control_ops,
2427        ) {
2428            Ok(registration) => registration,
2429            Err(RegistryError::DuplicateModuleId { module_id }) => {
2430                return Ok(vec![control_error_frame(
2431                    &frame,
2432                    "duplicate_module_id",
2433                    format!(
2434                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2435                    ),
2436                )?])
2437            }
2438            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2439                return Ok(vec![control_error_frame(
2440                    &frame,
2441                    "invalid_module_id",
2442                    err.to_string(),
2443                )?])
2444            }
2445            Err(err) => {
2446                return Ok(vec![control_error_frame(
2447                    &frame,
2448                    "registry_error",
2449                    err.to_string(),
2450                )?])
2451            }
2452        };
2453        #[cfg(test)]
2454        self.hold_between_registry_and_forwarding();
2455
2456        let reply = if let Some(sink) = sink {
2457            // The forwarding table's module store is also the daemon-to-module
2458            // control-RPC lane, so every HELLO gets a live endpoint even when the
2459            // manifest has no routable provider role. Non-routable modules still
2460            // cannot receive route.bind in production: `handle_route_open` checks
2461            // the registry manifest with `target_has_required_role` before the
2462            // only production call to `begin_route_bind_relay_for` below that
2463            // route.open path. The remaining direct relay callers are unit tests
2464            // and benchmark harnesses that construct forwarding state explicitly.
2465            //
2466            // The HELLO_ACK is queued by the forwarding table itself, before the
2467            // endpoint becomes visible, and is NOT returned as a reply. A module
2468            // reads HELLO_ACK first and exits on anything else; a reply is only
2469            // written after this handler returns, by which time a route.open on
2470            // another connection could already have queued a route.bind request
2471            // for this module ahead of it.
2472            let concurrency = manifest_concurrency(&registration.manifest);
2473            if let Err(err) = self.forwarding.register_module_connection_acked(
2474                connection_id,
2475                registration.manifest.module_id.clone(),
2476                negotiated_ver,
2477                concurrency,
2478                sink,
2479                hello_ack,
2480            ) {
2481                // Forwarding registration failed, so there is no forwarding
2482                // state to tear down. Remove the registry entry, which was never
2483                // published and so never listed or counted in the catalog
2484                // generation, and signal the release watch directly.
2485                if matches!(
2486                    self.deregister_connection(
2487                        connection_id,
2488                        RegistrationEndReason::RegistrationFailed,
2489                    ),
2490                    Ok(r) if !r.is_empty()
2491                ) {
2492                    crate::supervise::notify_registration_release();
2493                }
2494                return Ok(vec![control_error_frame(
2495                    &frame,
2496                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2497                        "invalid_hello"
2498                    } else {
2499                        forwarding_error_code(&err)
2500                    },
2501                    err.to_string(),
2502                )?]);
2503            }
2504            Vec::new()
2505        } else {
2506            // No sink means no forwarding endpoint, so nothing can be routed
2507            // ahead of the ack; it goes out as the reply.
2508            vec![hello_ack]
2509        };
2510
2511        // The module is routable now (or, without a sink, as routable as it
2512        // will ever be), so the catalog may list it. Publishing bumps the
2513        // catalog generation in the same registry critical section that makes
2514        // the module listed.
2515        if !self
2516            .registry
2517            .publish_registration(connection_id)
2518            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2519        {
2520            debug!(
2521                module_id = %registration.manifest.module_id,
2522                connection_id = connection_id.get(),
2523                "registration was removed before it could be published"
2524            );
2525        }
2526
2527        // Exposure over assumption: Concurrency's serde default is pinned to the
2528        // pre-field behavior (ModuleManaged), so a management surface that is
2529        // genuinely Serial and just never declared it inherits concurrent
2530        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2531        // turns "no module has been bitten yet" into the checkable claim "no
2532        // module is exposed" -- one read of the boot log instead of a fleet
2533        // audit. Detected from the raw HELLO bytes because the serde default
2534        // deliberately erases the absent/declared distinction from the type.
2535        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2536            info!(
2537                module_id = %registration.manifest.module_id,
2538                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2539            );
2540        }
2541
2542        self.apply_registration_capabilities(&registration);
2543
2544        self.supervisor
2545            .record_activity_registration(&registration.manifest.module_id);
2546
2547        info!(
2548            module_id = %registration.manifest.module_id,
2549            module_version = %registration.manifest.module_version,
2550            negotiated_ver,
2551            routable_provider = manifest_provides_routable_role(&registration.manifest),
2552            connection_id = connection_id.get(),
2553            "module registered"
2554        );
2555
2556        Ok(reply)
2557    }
2558
2559    /// Register a HELLO the swap gate admitted into the candidate slot of the
2560    /// registry and of forwarding, where it is reachable over its own
2561    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2562    /// nothing routes to it until the supervisor cuts over.
2563    ///
2564    /// Registry first (unpublished), then forwarding, then publication, the
2565    /// same order as an ordinary HELLO; a forwarding failure removes the
2566    /// registry entry again. The swap supervisor treats a published, ready
2567    /// candidate as registered and promotes it, and promotion needs the
2568    /// candidate's forwarding endpoint, so the candidate is published only
2569    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
2570    /// recompute capability verdicts or run the capability deny census
2571    /// (`enforce_capability_denies`): both are about the module in the active
2572    /// slot, which `route.open` can target. A candidate sits in the candidate
2573    /// slot, so it neither provides capabilities to other modules nor can be
2574    /// routed to until promotion moves it into the active slot.
2575    #[allow(clippy::too_many_arguments)]
2576    fn register_swap_candidate(
2577        &self,
2578        connection_id: ConnectionId,
2579        sink: Option<crate::FrameSink>,
2580        frame: &Frame,
2581        manifest: ModuleManifest,
2582        negotiated_ver: u8,
2583        control_ops: Vec<String>,
2584        hello_ack: Frame,
2585    ) -> Result<Vec<Frame>, RouterError> {
2586        let module_id = manifest.module_id.clone();
2587        let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2588            manifest,
2589            negotiated_ver,
2590            connection_id,
2591            control_ops,
2592        ) {
2593            Ok(registration) => registration,
2594            Err(RegistryError::DuplicateModuleId { module_id }) => {
2595                return Ok(vec![control_error_frame(
2596                    frame,
2597                    "duplicate_module_id",
2598                    format!(
2599                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2600                    ),
2601                )?])
2602            }
2603            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2604                return Ok(vec![control_error_frame(
2605                    frame,
2606                    "invalid_module_id",
2607                    err.to_string(),
2608                )?])
2609            }
2610            Err(err) => {
2611                return Ok(vec![control_error_frame(
2612                    frame,
2613                    "registry_error",
2614                    err.to_string(),
2615                )?])
2616            }
2617        };
2618        #[cfg(test)]
2619        self.hold_between_registry_and_forwarding();
2620        let reply = if let Some(sink) = sink {
2621            // Same ordering as an ordinary HELLO: the forwarding table queues
2622            // the HELLO_ACK before the candidate endpoint is inserted, because
2623            // a module exits if its first frame after HELLO is anything else.
2624            let concurrency = manifest_concurrency(&registration.manifest);
2625            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2626                connection_id,
2627                module_id.clone(),
2628                negotiated_ver,
2629                concurrency,
2630                sink,
2631                hello_ack,
2632            ) {
2633                if matches!(
2634                    self.deregister_connection(
2635                        connection_id,
2636                        RegistrationEndReason::RegistrationFailed,
2637                    ),
2638                    Ok(r) if !r.is_empty()
2639                ) {
2640                    crate::supervise::notify_registration_release();
2641                }
2642                return Ok(vec![control_error_frame(
2643                    frame,
2644                    forwarding_error_code(&err),
2645                    err.to_string(),
2646                )?]);
2647            }
2648            Vec::new()
2649        } else {
2650            vec![hello_ack]
2651        };
2652        if !self
2653            .registry
2654            .publish_registration(connection_id)
2655            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2656        {
2657            debug!(
2658                module_id = %module_id,
2659                connection_id = connection_id.get(),
2660                "swap candidate registration was removed before it could be published"
2661            );
2662        }
2663        self.supervisor.mark_swap_candidate_admitted(&module_id);
2664        info!(
2665            module_id = %module_id,
2666            module_version = %registration.manifest.module_version,
2667            negotiated_ver,
2668            ready = registration.ready,
2669            connection_id = connection_id.get(),
2670            "swap candidate registered; not routable until cutover"
2671        );
2672        Ok(reply)
2673    }
2674
2675    fn build_hello_ack(
2676        &self,
2677        frame: &Frame,
2678        negotiated_ver: u8,
2679        module_id: &str,
2680    ) -> Result<Frame, RouterError> {
2681        let ack = ModuleHelloAckBody {
2682            negotiated_ver,
2683            subc_ops: module_subc_ops(),
2684            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2685            storage: self
2686                .storage_config
2687                .as_ref()
2688                .map(|cfg| cfg.descriptor_for(module_id)),
2689            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2690        };
2691        let body = serde_json::to_vec(&ack).map_err(|err| {
2692            RouterError::backend(
2693                0,
2694                frame.header.corr,
2695                format!("failed to encode HELLO_ACK: {err}"),
2696            )
2697        })?;
2698
2699        Frame::build_with_version(
2700            negotiated_ver,
2701            FrameType::HelloAck,
2702            control_flags(),
2703            0,
2704            0,
2705            frame.header.corr,
2706            body,
2707        )
2708        .map_err(RouterError::FrameBuild)
2709    }
2710
2711    async fn handle_client_control_request(
2712        &self,
2713        ctx: &RouteCtx,
2714        frame: Frame,
2715        request: ClientControlRequest,
2716        route_open_timings: &mut Option<RouteOpenTimings>,
2717    ) -> Result<Vec<Frame>, RouterError> {
2718        if let Some(operation) = privileged_operation(&request) {
2719            let caller = self.resolve_caller(ctx.connection_id).await;
2720            let op = client_control_request_op(&request);
2721            let text = prompt_text(operation, &caller);
2722            match self.privileged_operator_prompt {
2723                PrivilegedOperatorPrompt::Log => {
2724                    info!(target: "subc_daemon::privileged_prompt", operation = op,
2725                        target = ?operation, caller = %caller.description(), summary = %text,
2726                        "privileged request would prompt the operator");
2727                }
2728                PrivilegedOperatorPrompt::Enforce => {
2729                    let key =
2730                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2731                    let outcome = self
2732                        .forwarding
2733                        .operator_confirms()
2734                        .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2735                        .await;
2736                    if outcome.code != "confirmed" {
2737                        return Ok(vec![outcome.frame(&frame)]);
2738                    }
2739                }
2740            }
2741        }
2742        match request {
2743            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2744            ClientControlRequest::CatalogList { module_id } => {
2745                self.handle_catalog_list(frame, module_id)
2746            }
2747            ClientControlRequest::RouteOpen {
2748                target,
2749                identity,
2750                consumer_identity,
2751                consumer_capabilities,
2752                role_versions,
2753                admission_facts,
2754                scope,
2755            } => {
2756                self.handle_route_open(
2757                    ctx,
2758                    frame,
2759                    RouteOpenRequest {
2760                        target,
2761                        identity,
2762                        consumer_identity,
2763                        consumer_capabilities,
2764                        role_versions,
2765                        admission_facts,
2766                        scope,
2767                    },
2768                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2769                )
2770                .await
2771            }
2772            ClientControlRequest::RoutePoll {
2773                route_channel,
2774                route_epoch,
2775                kind,
2776            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2777            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2778            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2779                self.handle_supervisor_spawn_snapshot(frame)
2780            }
2781            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2782                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2783            }
2784            ClientControlRequest::SupervisorRestart {
2785                module_id,
2786                drain_timeout_ms,
2787            } => {
2788                self.log_supervisor_request_received(
2789                    ctx,
2790                    frame.header.corr,
2791                    ops::SUPERVISOR_RESTART,
2792                    Some(&module_id),
2793                    None,
2794                )?;
2795                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2796                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2797                    .await
2798            }
2799            ClientControlRequest::SupervisorSwap {
2800                module_id,
2801                ready_timeout_ms,
2802            } => {
2803                self.log_supervisor_request_received(
2804                    ctx,
2805                    frame.header.corr,
2806                    ops::SUPERVISOR_SWAP,
2807                    Some(&module_id),
2808                    None,
2809                )?;
2810                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2811                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2812                    .await
2813            }
2814            ClientControlRequest::SupervisorReload { module_id } => {
2815                self.log_supervisor_request_received(
2816                    ctx,
2817                    frame.header.corr,
2818                    ops::SUPERVISOR_RELOAD,
2819                    Some(&module_id),
2820                    None,
2821                )?;
2822                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2823                self.handle_supervisor_reload(frame, module_id, requester)
2824                    .await
2825            }
2826            ClientControlRequest::SupervisorRescan { preview } => {
2827                if !preview {
2828                    self.log_supervisor_request_received(
2829                        ctx,
2830                        frame.header.corr,
2831                        ops::SUPERVISOR_RESCAN,
2832                        None,
2833                        None,
2834                    )?;
2835                }
2836                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2837                self.handle_supervisor_rescan(frame, preview, requester)
2838                    .await
2839            }
2840            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2841                self.log_supervisor_request_received(
2842                    ctx,
2843                    frame.header.corr,
2844                    ops::SUPERVISOR_RELEASE_RESERVED,
2845                    Some(&module_id),
2846                    None,
2847                )?;
2848                self.handle_supervisor_release_reserved(frame, module_id)
2849                    .await
2850            }
2851            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2852                self.log_supervisor_request_received(
2853                    ctx,
2854                    frame.header.corr,
2855                    ops::SUPERVISOR_SET_ENABLED,
2856                    Some(&module_id),
2857                    Some(enabled),
2858                )?;
2859                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2860                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2861                    .await
2862            }
2863            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2864                self.handle_supervisor_health_probe(frame, module_id).await
2865            }
2866            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2867            ClientControlRequest::SupervisorRoutes { module_id } => {
2868                self.handle_supervisor_routes(frame, module_id)
2869            }
2870            ClientControlRequest::SupervisorProvenance { module_id } => {
2871                self.handle_supervisor_provenance(frame, module_id).await
2872            }
2873            ClientControlRequest::SupervisorStderrTail {
2874                module_id,
2875                max_lines,
2876                max_bytes,
2877            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2878            ClientControlRequest::SupervisorTerminals { module_id } => {
2879                self.handle_supervisor_terminals(frame, module_id).await
2880            }
2881        }
2882    }
2883
2884    fn log_supervisor_request_received(
2885        &self,
2886        ctx: &RouteCtx,
2887        corr: u64,
2888        op: &'static str,
2889        module_id: Option<&str>,
2890        enabled: Option<bool>,
2891    ) -> Result<(), RouterError> {
2892        let caller = self.supervisor_requester(ctx, corr)?;
2893        let caller = principal_label(&caller);
2894        // `Option` fields are recorded only when present, so a request with no
2895        // module id (rescan) or no enabled flag simply omits that field.
2896        info!(
2897            target: "control",
2898            op,
2899            module_id,
2900            enabled,
2901            connection_id = ctx.connection_id.get(),
2902            caller = %caller,
2903            "supervisor request received"
2904        );
2905        Ok(())
2906    }
2907
2908    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2909        Ok(self
2910            .registry
2911            .get_module_by_connection(ctx.connection_id)
2912            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2913            .map(|registration| Principal::Reserved {
2914                module_id: registration.manifest.module_id,
2915            })
2916            .unwrap_or(Principal::Direct))
2917    }
2918
2919    fn handle_module_control_request(
2920        &self,
2921        connection_id: ConnectionId,
2922        frame: Frame,
2923        request: ModuleControlRequestFromModule,
2924    ) -> Result<Vec<Frame>, RouterError> {
2925        match request {
2926            ModuleControlRequestFromModule::CatalogUpdate {
2927                provides,
2928                capabilities,
2929                ready,
2930            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2931            ModuleControlRequestFromModule::LiveRoots {} => {
2932                let registered = self
2933                    .registry
2934                    .get_module_by_connection(connection_id)
2935                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2936                let Some(registration) = registered else {
2937                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2938                };
2939                let response = self
2940                    .forwarding
2941                    .live_roots(&registration.manifest.module_id)
2942                    .map_err(RouterError::Forwarding)?;
2943                Ok(vec![control_response_body_frame(
2944                    &frame,
2945                    &response,
2946                    "ModuleControlResponseToModule::LiveRoots",
2947                )?])
2948            }
2949            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2950                self.handle_scope_sync(connection_id, frame, generation, scopes)
2951            }
2952            ModuleControlRequestFromModule::ScopeApply {
2953                generation,
2954                upsert,
2955                end,
2956            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2957            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2958                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2959            }
2960        }
2961    }
2962
2963    fn handle_operator_confirm(
2964        &self,
2965        ctx: &RouteCtx,
2966        frame: Frame,
2967    ) -> Result<Vec<Frame>, RouterError> {
2968        use crate::operator_confirm::{audit, Outcome};
2969        let registration = self
2970            .registry
2971            .get_module_by_connection(ctx.connection_id)
2972            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2973        let Some(registration) = registration else {
2974            let outcome = Outcome::refusal("not_registered");
2975            audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2976            return Ok(vec![outcome.frame(&frame)]);
2977        };
2978        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2979            Ok(request) => request,
2980            Err(_) => {
2981                let outcome = Outcome::refusal("invalid_control_body");
2982                audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2983                return Ok(vec![outcome.frame(&frame)]);
2984            }
2985        };
2986        let module_id = registration.manifest.module_id;
2987        // Read the launch nonce (under its own lock) before taking the forwarding
2988        // table's lock below: holding forwarding while waiting on another daemon
2989        // lock risks a lock-order deadlock with paths that take them the other way.
2990        let nonce = self
2991            .hello_launch_nonces
2992            .lock()
2993            .unwrap_or_else(|p| p.into_inner())
2994            .nonce(ctx.connection_id)
2995            .map(str::to_owned);
2996        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2997            self.supervisor
2998                .spawned_consumer_authorized(&module_id, nonce)
2999        });
3000        let confirms = self.forwarding.operator_confirms();
3001        self.forwarding
3002            .with_operator_route(
3003                ctx.connection_id,
3004                request.route_channel,
3005                request.route_epoch,
3006                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3007            )
3008            .map_err(RouterError::Forwarding)
3009    }
3010
3011    /// `scope.sync`: the owner is the module registered on this connection.
3012    /// A connection with no registration (every client connection, `direct`
3013    /// included) is refused `not_registered` before the table is consulted.
3014    fn handle_scope_sync(
3015        &self,
3016        connection_id: ConnectionId,
3017        frame: Frame,
3018        generation: u64,
3019        scopes: Vec<ScopeRecord>,
3020    ) -> Result<Vec<Frame>, RouterError> {
3021        self.handle_scope_change(connection_id, frame, generation, scopes, None)
3022    }
3023
3024    fn handle_scope_change(
3025        &self,
3026        connection_id: ConnectionId,
3027        frame: Frame,
3028        generation: u64,
3029        scopes: Vec<ScopeRecord>,
3030        end: Option<Vec<ScopeEnd>>,
3031    ) -> Result<Vec<Frame>, RouterError> {
3032        let is_apply = end.is_some();
3033        let Some(registration) = self
3034            .registry
3035            .get_module_by_connection(connection_id)
3036            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3037        else {
3038            return Ok(vec![control_error_frame(
3039                &frame,
3040                "not_registered",
3041                "scope.sync and scope.apply require an active module registration owned by this connection",
3042            )?]);
3043        };
3044        let owner = registration.manifest.module_id;
3045        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3046        let is_current_launch = |connection: ConnectionId| {
3047            self.hello_launch_nonces
3048                .lock()
3049                .unwrap_or_else(|poisoned| poisoned.into_inner())
3050                .presented(connection, current_nonce.as_deref())
3051        };
3052        // Lock order is the scope table, then the forwarding table: the new
3053        // tags are published, and the routes the change closes are selected,
3054        // while the scope table is still write-locked, so no admission can read
3055        // a record whose tag is not yet published.
3056        let mut table = self
3057            .scopes
3058            .write()
3059            .unwrap_or_else(|poisoned| poisoned.into_inner());
3060        let outcome = match end {
3061            Some(end) => table.apply(
3062                &owner,
3063                connection_id,
3064                is_current_launch,
3065                generation,
3066                scopes,
3067                end,
3068            ),
3069            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3070        };
3071        let drained = match &outcome {
3072            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3073            Err(_) => Vec::new(),
3074        };
3075        let has_deadlines = outcome.is_ok() && table.has_deadlines();
3076        drop(table);
3077        if has_deadlines {
3078            self.scope_deadline_added.notify_one();
3079        }
3080        match outcome {
3081            Ok(applied) => {
3082                let counts = ScopeOutcomeCounts::of(&applied.results);
3083                info!(
3084                    owner = %owner,
3085                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3086                    generation,
3087                    records = applied.results.len(),
3088                    created = counts.created,
3089                    replaced = counts.replaced,
3090                    updated = counts.updated,
3091                    unchanged = counts.unchanged,
3092                    refused = counts.refused,
3093                    ended = applied.ended.len(),
3094                    tag_changes = applied.tag_changes.len(),
3095                    routes_closed = drained.len(),
3096                    "scope change accepted"
3097                );
3098                // An accepted scope change can still refuse individual records, and the
3099                // owner is the only party that sees the reply. Name them here so
3100                // an operator can tell a refused session from a missing one
3101                // without the owner's logs. Capped so a call that refuses
3102                // thousands cannot flood the log; the count above is complete.
3103                for refused in applied
3104                    .results
3105                    .iter()
3106                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3107                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3108                {
3109                    warn!(
3110                        owner = %owner,
3111                        generation,
3112                        scope_ref = %refused.scope_ref,
3113                        scope_epoch = refused.scope_epoch,
3114                        code = refused.code.as_deref().unwrap_or(""),
3115                        "scope record refused"
3116                    );
3117                }
3118                self.close_scope_drained_routes(drained);
3119                let response = if is_apply {
3120                    ModuleControlResponseToModule::ScopeApply {
3121                        generation,
3122                        results: applied.results,
3123                        end_results: applied.end_results,
3124                        ended: applied.ended,
3125                    }
3126                } else {
3127                    ModuleControlResponseToModule::ScopeSync {
3128                        generation,
3129                        results: applied.results,
3130                        ended: applied.ended,
3131                    }
3132                };
3133                Ok(vec![control_response_body_frame(
3134                    &frame,
3135                    &response,
3136                    "scope change response",
3137                )?])
3138            }
3139            Err(refusal) => {
3140                info!(
3141                    owner = %owner,
3142                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3143                    generation,
3144                    code = refusal.code,
3145                    "scope change refused"
3146                );
3147                Ok(vec![control_error_frame(
3148                    &frame,
3149                    refusal.code,
3150                    refusal.message,
3151                )?])
3152            }
3153        }
3154    }
3155
3156    /// Tell both ends of each route a scope change closed. The module gets a
3157    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
3158    /// the client's route handle. The client also gets `route.closed` with the
3159    /// scope reason, one push per module and reason, so it can tell a revoked
3160    /// route from an ordinary close and not reopen it.
3161    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3162        if drained.is_empty() {
3163            return;
3164        }
3165        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3166            BTreeMap::new();
3167        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3168        for route in drained {
3169            warn!(
3170                module_id = %route.module_id,
3171                reason = ?route.reason,
3172                client_connection_id = route.client.connection_id.get(),
3173                route_channel = route.client.channel,
3174                "closing route because its scope changed"
3175            );
3176            pushes
3177                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3178                .or_insert_with(|| (route.reason, Vec::new()))
3179                .1
3180                .push(EndpointRoute {
3181                    goodbye_target: route.client.clone(),
3182                    principal: Principal::Unverified,
3183                    bound_at: Instant::now(),
3184                    draining: false,
3185                    drain_reason: None,
3186                });
3187            goodbyes.push(route.module);
3188            goodbyes.push(route.client);
3189        }
3190        for ((module_id, _), (reason, routes)) in pushes {
3191            send_route_control_pushes(
3192                &self.forwarding,
3193                routes,
3194                ClientControlPush::RouteClosed {
3195                    module_id,
3196                    channels: Vec::new(),
3197                    reason,
3198                    drained: false,
3199                    abandoned: 0,
3200                    excluded_subscriptions: 0,
3201                    terminal: Some(false),
3202                },
3203            );
3204        }
3205        self.emit_route_goodbyes(goodbyes);
3206    }
3207
3208    /// `scope.describe`: any registered module may read any scope, because a
3209    /// provider must read the scope a route it serves is stamped with.
3210    fn handle_scope_describe(
3211        &self,
3212        connection_id: ConnectionId,
3213        frame: Frame,
3214        owner: Principal,
3215        scope_ref: String,
3216    ) -> Result<Vec<Frame>, RouterError> {
3217        let registered = self
3218            .registry
3219            .get_module_by_connection(connection_id)
3220            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3221        if registered.is_none() {
3222            return Ok(vec![control_error_frame(
3223                &frame,
3224                "not_registered",
3225                "scope.describe requires an active module registration owned by this connection",
3226            )?]);
3227        }
3228        let description = self
3229            .scopes
3230            .read()
3231            .unwrap_or_else(|poisoned| poisoned.into_inner())
3232            .describe(&owner, &scope_ref);
3233        let owner_configured = match &owner {
3234            // Ask whether the owner is configured (`is_configured`), not
3235            // whether it is on the roster (`get(..).is_some()`): a supervised
3236            // module's process can register and describe a scope before the
3237            // supervisor has put it on the roster.
3238            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3239            _ => false,
3240        };
3241        let response = ModuleControlResponseToModule::ScopeDescribe {
3242            status: description.status,
3243            scope_epoch: description.scope_epoch,
3244            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3245            owner_synced: description.owner_synced,
3246            owner_configured,
3247            scope: description.stamp,
3248        };
3249        Ok(vec![control_response_body_frame(
3250            &frame,
3251            &response,
3252            "ModuleControlResponseToModule::ScopeDescribe",
3253        )?])
3254    }
3255
3256    fn handle_catalog_update(
3257        &self,
3258        connection_id: ConnectionId,
3259        frame: Frame,
3260        provides: Vec<ProviderRole>,
3261        capabilities: Option<CapabilityDeclarations>,
3262        ready: Option<bool>,
3263    ) -> Result<Vec<Frame>, RouterError> {
3264        self.refresh_capability_requirements();
3265        let Some(registration) = self
3266            .registry
3267            .get_module_by_connection(connection_id)
3268            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3269        else {
3270            return Ok(vec![control_error_frame(
3271                &frame,
3272                "not_registered",
3273                "catalog.update requires an active module registration owned by this connection",
3274            )?]);
3275        };
3276
3277        if let Some(message) =
3278            catalog_update_frozen_field_message(&registration.manifest, &provides)
3279        {
3280            return Ok(vec![control_error_frame(
3281                &frame,
3282                "catalog_update_frozen_field",
3283                message,
3284            )?]);
3285        }
3286
3287        let mut candidate = registration.manifest.clone();
3288        candidate.provides = provides.clone();
3289        candidate.capabilities = capabilities
3290            .clone()
3291            .or_else(|| registration.manifest.capabilities.clone());
3292        if let Err(err) = candidate.validate_capability_grammar() {
3293            return Ok(vec![control_error_frame(
3294                &frame,
3295                "invalid_capability_grammar",
3296                err.to_string(),
3297            )?]);
3298        }
3299
3300        // Updates must honor the same reserved owner as initial registration;
3301        // otherwise an empty HELLO could acquire the claim after admission.
3302        let mut conflicts = self
3303            .capability_evaluator
3304            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3305        if let Some(conflict) = conflicts.first() {
3306            let message = format!(
3307                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3308                conflict.capability, conflict.claimants[0], candidate.module_id
3309            );
3310            for conflict in &mut conflicts {
3311                conflict.source = DuplicateClaimSource::CatalogUpdate;
3312            }
3313            log_duplicate_claim_events(conflicts);
3314            return Ok(vec![control_error_frame(
3315                &frame,
3316                "reserved_capability",
3317                message,
3318            )?]);
3319        }
3320
3321        let updated = self
3322            .registry
3323            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3324            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3325        if updated.is_none() {
3326            return Ok(vec![control_error_frame(
3327                &frame,
3328                "not_registered",
3329                "catalog.update requires an active module registration owned by this connection",
3330            )?]);
3331        }
3332        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3333            log_duplicate_claim_events(
3334                self.capability_evaluator
3335                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3336            );
3337        }
3338        if capability_census_trigger(
3339            registration.manifest.capabilities.as_ref(),
3340            updated
3341                .as_ref()
3342                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3343        ) {
3344            self.enforce_capability_denies();
3345        }
3346        self.refresh_capability_requirements();
3347
3348        let response = ModuleControlResponseToModule::CatalogUpdate {};
3349        control_response_body_frame(
3350            &frame,
3351            &response,
3352            "ModuleControlResponseToModule::CatalogUpdate",
3353        )
3354        .map(|frame| vec![frame])
3355    }
3356
3357    pub(crate) fn with_activity_ledger_config(
3358        mut self,
3359        run_dir: &Path,
3360        incarnation: String,
3361        config: crate::activity::ActivityConfig,
3362    ) -> Self {
3363        self.activity = Some(crate::activity::ActivityLedger::open_config(
3364            run_dir.join("activity.jsonl"),
3365            incarnation,
3366            config,
3367        ));
3368        self.supervisor.set_activity(self.activity.clone());
3369        self
3370    }
3371
3372    pub(crate) async fn configure_activity(
3373        &self,
3374        config: crate::activity::ActivityConfig,
3375    ) -> Result<(), &'static str> {
3376        match &self.activity {
3377            Some(ledger) => ledger.configure(config).await,
3378            None => Ok(()),
3379        }
3380    }
3381
3382    fn cancel_activity_subscription(
3383        &self,
3384        ctx: &RouteCtx,
3385        frame: &Frame,
3386    ) -> Result<bool, RouterError> {
3387        let task = self
3388            .activity_streams
3389            .lock()
3390            .unwrap_or_else(|p| p.into_inner())
3391            .remove(&(ctx.connection_id, frame.header.corr));
3392        let Some(task) = task else {
3393            return Ok(false);
3394        };
3395        task.abort();
3396        let end = Frame::build_with_version(
3397            response_version(frame),
3398            FrameType::StreamEnd,
3399            control_flags(),
3400            0,
3401            0,
3402            frame.header.corr,
3403            Vec::new(),
3404        )
3405        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3406        let sink = ctx.egress.clone();
3407        tokio::spawn(async move {
3408            let _ = sink.send(end).await;
3409        });
3410        Ok(true)
3411    }
3412
3413    async fn handle_activity_read(
3414        &self,
3415        ctx: &RouteCtx,
3416        frame: Frame,
3417    ) -> Result<Vec<Frame>, RouterError> {
3418        use subc_control::{
3419            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3420            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3421        };
3422        #[derive(Deserialize)]
3423        struct Identity {
3424            #[serde(default)]
3425            consumer_identity: Option<ConsumerIdentity>,
3426        }
3427        // A registered module connection already identifies its caller. Ignore
3428        // consumer_identity there so a module cannot borrow a mirror's identity.
3429        // Only separate consumer connections use the Direct/nonce trust boundary.
3430        let principal = match self
3431            .forwarding
3432            .module_id_for_connection(ctx.connection_id)
3433            .map_err(RouterError::Forwarding)?
3434        {
3435            Some(module_id) => Ok(Principal::Reserved { module_id }),
3436            None => {
3437                let identity: Identity = match serde_json::from_slice(&frame.body) {
3438                    Ok(identity) => identity,
3439                    Err(_) => {
3440                        return Ok(vec![control_error_frame(
3441                            &frame,
3442                            ACTIVITY_MALFORMED,
3443                            "malformed activity read",
3444                        )?])
3445                    }
3446                };
3447                self.route_open_principal(&frame, identity.consumer_identity)?
3448            }
3449        };
3450        let admitted = matches!(principal, Ok(Principal::Direct))
3451            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3452        if !admitted {
3453            return Ok(vec![control_error_frame(
3454                &frame,
3455                ACTIVITY_READ_FORBIDDEN,
3456                "activity reads require an operator or trusted mirror principal",
3457            )?]);
3458        }
3459        let Some(ledger) = &self.activity else {
3460            return Ok(vec![control_error_frame(
3461                &frame,
3462                ACTIVITY_UNAVAILABLE,
3463                "activity ledger unavailable",
3464            )?]);
3465        };
3466        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3467            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3468        {
3469            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3470                Ok(request) => ledger.query(request).await,
3471                Err(_) => Err(ACTIVITY_MALFORMED),
3472            };
3473            return match result {
3474                Ok(reply) => Ok(vec![control_response_body_frame(
3475                    &frame,
3476                    &reply,
3477                    ops::ACTIVITY_QUERY,
3478                )?]),
3479                Err(code) => Ok(vec![control_error_frame(
3480                    &frame,
3481                    code,
3482                    "activity query refused",
3483                )?]),
3484            };
3485        }
3486        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3487            Ok(request) => request,
3488            Err(_) => {
3489                return Ok(vec![control_error_frame(
3490                    &frame,
3491                    ACTIVITY_MALFORMED,
3492                    "malformed activity subscription",
3493                )?])
3494            }
3495        };
3496        let mut subscription = match ledger.subscribe(request.since).await {
3497            Ok(subscription) => subscription,
3498            Err(code) => {
3499                return Ok(vec![control_error_frame(
3500                    &frame,
3501                    code,
3502                    "activity subscription refused",
3503                )?])
3504            }
3505        };
3506        let key = (ctx.connection_id, frame.header.corr);
3507        let sink = ctx.egress.clone();
3508        let mut streams = self
3509            .activity_streams
3510            .lock()
3511            .unwrap_or_else(|p| p.into_inner());
3512        streams.retain(|_, task| !task.is_finished());
3513        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3514            return Ok(vec![control_error_frame(
3515                &frame,
3516                "duplicate_subscription",
3517                "activity subscription correlation id already in use",
3518            )?]);
3519        }
3520        let task = tokio::spawn(async move {
3521            loop {
3522                let entry = if let Some(entry) = subscription.replay.pop_front() {
3523                    Some(entry)
3524                } else {
3525                    subscription.receiver.recv().await
3526                };
3527                let Some(entry) = entry else {
3528                    break;
3529                };
3530                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3531                    return;
3532                };
3533                let Ok(next) = Frame::build_with_version(
3534                    response_version(&frame),
3535                    FrameType::StreamData,
3536                    control_flags(),
3537                    0,
3538                    0,
3539                    frame.header.corr,
3540                    body,
3541                ) else {
3542                    return;
3543                };
3544                if sink.send(next).await.is_err() {
3545                    return;
3546                }
3547            }
3548            // Dropping a full subscriber's sender drains its accepted entries
3549            // before the terminal error, preserving their order on the wire.
3550            if let Some(code) = subscription.terminal_code() {
3551                if let Ok(end) =
3552                    control_error_frame(&frame, code, "activity subscriber fell behind")
3553                {
3554                    let _ = sink.send(end).await;
3555                }
3556            }
3557        });
3558        streams.insert(key, task.abort_handle());
3559        Ok(Vec::new())
3560    }
3561
3562    async fn handle_activity_record(
3563        &self,
3564        ctx: &RouteCtx,
3565        frame: Frame,
3566    ) -> Result<Vec<Frame>, RouterError> {
3567        use subc_control::{
3568            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3569            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3570        };
3571        let result = async {
3572            let module = self
3573                .forwarding
3574                .module_id_for_connection(ctx.connection_id)
3575                .ok()
3576                .flatten()
3577                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3578            let request: ActivityRecordRequest =
3579                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3580            crate::activity::validate(&module, &request)?;
3581            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3582            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3583            let at_ms = std::time::SystemTime::now()
3584                .duration_since(std::time::UNIX_EPOCH)
3585                .unwrap_or_default()
3586                .as_millis() as u64;
3587            ledger.record(module, request, actor, at_ms).await
3588        }
3589        .await;
3590        match result {
3591            Ok(entry) => Ok(vec![control_response_body_frame(
3592                &frame,
3593                &ActivityRecordResponse { seq: entry.seq },
3594                "activity.record",
3595            )?]),
3596            Err(code) => Ok(vec![control_error_frame(
3597                &frame,
3598                code,
3599                "activity recording refused",
3600            )?]),
3601        }
3602    }
3603
3604    fn activity_actor(
3605        &self,
3606        connection: ConnectionId,
3607        module: &str,
3608        on: Option<&subc_control::ActivityOn>,
3609    ) -> Result<subc_control::ActivityActor, &'static str> {
3610        use crate::forwarding::{DataRoute, DataRouteState};
3611        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3612        let principal = |id: &str| Principal::Reserved {
3613            module_id: id.to_owned(),
3614        };
3615        let mut actor = ActivityActor {
3616            principal: principal(module),
3617            scope_owner: None,
3618            agent_id: None,
3619            run_id: None,
3620            flow_id: None,
3621            project_root: None,
3622        };
3623        let scope = match on {
3624            None => return Ok(actor),
3625            Some(ActivityOn::Scope(scope)) => {
3626                let selector = subc_protocol::scope::ScopeSelector::new(
3627                    principal(module),
3628                    scope.scope_ref.clone(),
3629                    Some(scope.epoch),
3630                );
3631                self.scopes
3632                    .read()
3633                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3634                    .attribute(&principal(module), module, &selector, true)
3635                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3636            }
3637            Some(ActivityOn::Route(route)) => {
3638                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3639                    .forwarding
3640                    .lookup_data_route(connection, route.channel, route.epoch)
3641                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3642                else {
3643                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3644                };
3645                if binding.module_id != module {
3646                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3647                }
3648                actor.principal = binding.principal.clone();
3649                actor.project_root = binding
3650                    .project_root
3651                    .as_ref()
3652                    .map(|root| root.as_path().to_path_buf());
3653                let Some(bound) = &binding.scope else {
3654                    return Ok(actor);
3655                };
3656                let owner = principal(&bound.owner);
3657                let selector = subc_protocol::scope::ScopeSelector::new(
3658                    owner.clone(),
3659                    bound.scope_ref.clone(),
3660                    Some(bound.tag.scope_epoch),
3661                );
3662                let admitted = self
3663                    .scopes
3664                    .read()
3665                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3666                    .attribute(&owner, module, &selector, true)
3667                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3668                if admitted.tag != bound.tag {
3669                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3670                }
3671                admitted
3672            }
3673        };
3674        actor.scope_owner = Some(scope.stamp.owner);
3675        actor.agent_id = scope.stamp.attributes.agent_id;
3676        actor.run_id = scope.stamp.attributes.run_id;
3677        actor.flow_id = scope.stamp.attributes.flow_id;
3678        Ok(actor)
3679    }
3680
3681    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3682        self.refresh_capability_requirements();
3683        // A bare connection count is ambiguous between many clients holding a
3684        // route each and one client accumulating hundreds, so publish the
3685        // concentration alongside it. Route state is best-effort here: a
3686        // diagnostic endpoint must still answer if the forwarding lock is
3687        // contended.
3688        let mut counters = self.counters.snapshot();
3689        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3690            obj.insert(
3691                "activity_unparseable_lines".into(),
3692                ledger.read_errors().into(),
3693            );
3694            obj.insert(
3695                "activity_lifecycle_append_failures".into(),
3696                ledger.lifecycle_failures().into(),
3697            );
3698            obj.insert(
3699                "activity_started_digest_skips".into(),
3700                ledger.lifecycle_digest_skips().into(),
3701            );
3702        }
3703        if let (Ok((connections_with_routes, max)), Some(obj)) = (
3704            self.forwarding.client_route_concentration(),
3705            counters.as_object_mut(),
3706        ) {
3707            obj.insert(
3708                "client_connections_with_routes".into(),
3709                connections_with_routes.into(),
3710            );
3711            obj.insert("max_routes_on_one_connection".into(), max.into());
3712        }
3713        // A module that is being fast-refused and a module that is fine look
3714        // identical from a client that retries and succeeds, so name the open
3715        // breakers here. This rides the existing free-form counters object
3716        // rather than a new wire field, so no sibling that deserializes
3717        // `ServerDescribe` has to be rebuilt to keep reading it.
3718        if let (Some(open_breakers), Some(obj)) = (
3719            self.route_bind_breakers.open_snapshot(),
3720            counters.as_object_mut(),
3721        ) {
3722            obj.insert("route_bind_breakers_open".into(), open_breakers);
3723        }
3724        let response = ClientControlResponse::ServerDescribe {
3725            protocol_ver: PROTOCOL_VERSION,
3726            subc_ops: subc_ops(),
3727            capabilities: self.subc_capabilities.as_ref().to_vec(),
3728            connected_clients: self.connected_clients.count(),
3729            counters: Some(counters),
3730            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3731            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3732            capability_requirements: self.capability_requirement_statuses(),
3733            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3734        };
3735        Ok(vec![control_response_body_frame(
3736            &frame,
3737            &response,
3738            "ClientControlResponse::ServerDescribe",
3739        )?])
3740    }
3741
3742    fn handle_catalog_list(
3743        &self,
3744        frame: Frame,
3745        module_id: Option<String>,
3746    ) -> Result<Vec<Frame>, RouterError> {
3747        let (generation, modules) = self.registry.list_modules().map_err(|err| {
3748            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3749        })?;
3750        let entries = modules
3751            .into_iter()
3752            .filter(|registration| {
3753                module_id
3754                    .as_deref()
3755                    .map(|wanted| registration.manifest.module_id == wanted)
3756                    .unwrap_or(true)
3757            })
3758            .map(|registration| {
3759                let not_ready = self.not_ready_reason(&registration);
3760                let roles = registration.manifest.provides;
3761                CatalogEntry::new(
3762                    registration.manifest.module_id,
3763                    roles,
3764                    registration.control_ops,
3765                )
3766                .with_ready(not_ready.is_none())
3767                .with_not_ready(not_ready)
3768                .with_module_version(Some(registration.manifest.module_version))
3769                .with_capabilities(registration.manifest.capabilities)
3770                .with_self_signals(registration.manifest.self_signals)
3771                .with_events(registration.manifest.events)
3772            })
3773            .collect();
3774        let response = ClientControlResponse::CatalogList {
3775            generation,
3776            modules: entries,
3777            subc_ops: subc_ops(),
3778        };
3779        Ok(vec![control_response_body_frame(
3780            &frame,
3781            &response,
3782            "ClientControlResponse::CatalogList",
3783        )?])
3784    }
3785
3786    fn route_open_principal(
3787        &self,
3788        frame: &Frame,
3789        consumer_identity: Option<ConsumerIdentity>,
3790    ) -> Result<Result<Principal, Frame>, RouterError> {
3791        let Some(consumer_identity) = consumer_identity else {
3792            return Ok(Ok(Principal::Direct));
3793        };
3794
3795        if self.supervisor.spawned_consumer_authorized(
3796            &consumer_identity.module_id,
3797            &consumer_identity.launch_nonce,
3798        ) {
3799            return Ok(Ok(Principal::Reserved {
3800                module_id: consumer_identity.module_id,
3801            }));
3802        }
3803
3804        Ok(Err(control_error_frame(
3805            frame,
3806            "bad_consumer_identity",
3807            format!(
3808                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3809                consumer_identity.module_id
3810            ),
3811        )?))
3812    }
3813
3814    /// Ordinary `route.open` refusals go through here; admission and breaker
3815    /// refusals log separately with their capacity or breaker state. The daemon can
3816    /// attest which code it sent: without the event, a client's "the daemon
3817    /// refused me" and the daemon's own view could only be reconciled by
3818    /// argument. Malformed input (`invalid_project_root`) does not come here;
3819    /// rejecting a request that was never a valid open is not a refusal of one.
3820    fn route_open_refusal_frame(
3821        &self,
3822        ctx: &RouteCtx,
3823        frame: &Frame,
3824        module_id: &str,
3825        reason: &'static str,
3826        code: &'static str,
3827        message: impl Into<String>,
3828    ) -> Result<Frame, RouterError> {
3829        self.observe_route_open_refusal(ctx, module_id, reason, code);
3830        control_error_frame(frame, code, message.into())
3831    }
3832
3833    /// Refuse a `route.open` because the target module's bind-relay breaker is
3834    /// open, without attempting the relay.
3835    ///
3836    /// The wire code is `module_timeout`, which is the truth (the module has
3837    /// not been answering binds) and which both SDKs already classify as
3838    /// retryable with capped backoff. Reusing it is what keeps this change out
3839    /// of both SDKs; the daemon-side distinction lives in the counter key
3840    /// instead.
3841    ///
3842    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
3843    /// While a breaker is open this fires on every open to that module, and the
3844    /// stall written up in `docs/designs/route-open-head-of-line.md` already
3845    /// produced 261 lines about a single module inside 3000 lines of daemon
3846    /// log. The rare transitions are logged at warn/info instead and the volume
3847    /// is carried by the counter, so the evidence survives without the flood.
3848    /// The debug line keeps a per-refusal record reachable for whoever turns
3849    /// the level up.
3850    fn route_open_breaker_refusal_frame(
3851        &self,
3852        ctx: &RouteCtx,
3853        frame: &Frame,
3854        module_id: &str,
3855        consecutive_timeouts: u32,
3856        retry_in: Duration,
3857        probe_in_flight: bool,
3858    ) -> Result<Frame, RouterError> {
3859        self.counters
3860            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3861        debug!(
3862            target: "control",
3863            code = "module_timeout",
3864            module_id = ?module_id,
3865            connection_id = ctx.connection_id.get(),
3866            consecutive_timeouts,
3867            retry_in_ms = retry_in.as_millis() as u64,
3868            probe_in_flight,
3869            "route.open refused by open bind-relay breaker"
3870        );
3871        // Say what a caller can act on. An open bind-relay breaker means the
3872        // module timed out accepting several new routes in a row. The module
3873        // is still running and its established routes keep working; only new
3874        // route.open requests are refused until the cooldown ends and one
3875        // test route (the probe) gets through. A message that only counts
3876        // failed relays reads as "the module is down" to a worker that sees it.
3877        let detail = if probe_in_flight {
3878            "one test route is already being tried; retry once it settles".to_string()
3879        } else {
3880            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3881        };
3882        control_error_frame(
3883            frame,
3884            "module_timeout",
3885            format!(
3886                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3887                 timed out in a row); {detail}; its established routes are unaffected"
3888            ),
3889        )
3890    }
3891
3892    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
3893    /// requester's bytes (an unknown target is whatever the client sent) and
3894    /// is Debug-formatted so control characters land in the log escaped
3895    /// rather than as terminal sequences for whoever tails it.
3896    ///
3897    /// `reason` names the check that refused, because one wire code has
3898    /// several senders: after a module registers, `target_unavailable` can
3899    /// come from a missing role, an inactive registration, a supervisor that
3900    /// has not marked the process live, a missing forwarding connection, or a
3901    /// failed relay, and a log that records only the code cannot say which of
3902    /// them fired. It is a static, daemon-chosen label per branch, so it is
3903    /// safe to print plainly and stays a closed set.
3904    fn observe_route_open_refusal(
3905        &self,
3906        ctx: &RouteCtx,
3907        module_id: &str,
3908        reason: &'static str,
3909        code: &'static str,
3910    ) {
3911        self.counters.increment_route_open_refused(code);
3912        info!(
3913            target: "control",
3914            code,
3915            reason,
3916            module_id = ?module_id,
3917            connection_id = ctx.connection_id.get(),
3918            "route.open refused"
3919        );
3920        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3921            self.route_outages.record_not_serving(module_id, reason);
3922        }
3923    }
3924
3925    /// Record an ACCEPTED route.open.
3926    ///
3927    /// Refusals have been logged and counted since the attestation work; accepts
3928    /// were invisible, so the daemon knew every principal it stamped and wrote
3929    /// none of them down. The party that attests the identity was the only party
3930    /// not recording it, which left a credential vault unable to name the sender
3931    /// of a call that reached it (claustrum #43) and left the launch-nonce
3932    /// concurrency question unanswerable from the outside.
3933    ///
3934    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
3935    /// `code`/`module_id`/`connection_id` returns both directions of the same
3936    /// decision rather than two shapes a reader has to join by hand.
3937    ///
3938    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
3939    /// and the difference carries information rather than being an
3940    /// inconsistency. This line is only reachable after a successful bind to a
3941    /// REGISTERED module, so the value has already passed HELLO validation
3942    /// including the path-hazard refusal and cannot contain control bytes. A
3943    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
3944    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
3945    ///
3946    /// Bare is also what every other daemon line already emits (`module
3947    /// registered`, `configured module supervised`). Shipping `?module_id` here
3948    /// made this instrument the only one in the file whose ids did not answer
3949    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
3950    /// line whose whole purpose is being grepped beside its sibling.
3951    ///
3952    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
3953    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
3954    /// forwards every field type through it and a bare `&str` and a `?`-escaped
3955    /// one are recorded identically. A test written against that harness passes
3956    /// either way -- I wrote one, measured it, and deleted it rather than ship a
3957    /// green assertion that cannot fail. The same limit applies to the escaping
3958    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
3959    /// it reads as a guard on the Debug escaping and cannot detect its removal.
3960    /// Fencing either needs the real formatter, not the capture layer.
3961    ///
3962    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
3963    /// unix-socket options and subc is loopback TCP, so there is no peer identity
3964    /// to record. The ephemeral port would decay within minutes and answer only a
3965    /// live question. The identity question is instead answered by counting
3966    /// distinct live connections presenting one module's `consumer_identity` --
3967    /// "is anyone else holding this secret" rather than "is this the right
3968    /// process".
3969    fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3970        self.route_outages.record_accepted(module_id);
3971        self.counters.increment_route_open_accepted(principal);
3972        info!(
3973            target: "control",
3974            principal,
3975            module_id,
3976            connection_id = ctx.connection_id.get(),
3977            "route.open accepted"
3978        );
3979    }
3980
3981    fn supervised_absent_route_open_refusal_frame(
3982        &self,
3983        ctx: &RouteCtx,
3984        frame: &Frame,
3985        module_id: &str,
3986        code: &'static str,
3987        status: &crate::supervise::ModuleStatus,
3988    ) -> Result<Frame, RouterError> {
3989        self.counters.increment_route_open_refused(code);
3990        info!(
3991            target: "control",
3992            code,
3993            reason = "supervised_not_registered",
3994            module_id = ?module_id,
3995            connection_id = ctx.connection_id.get(),
3996            state = %status.state,
3997            enabled = status.enabled,
3998            live = status.live,
3999            "route.open refused"
4000        );
4001        // A supervised module whose process has not registered is not
4002        // serving, whatever the reason; the supervisor knows this id, so it is
4003        // safe to track.
4004        self.route_outages
4005            .record_not_serving(module_id, "supervised_not_registered");
4006        let message = if let Some(pid) = status.waiting_for_previous_pid {
4007            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4008                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4009            })
4010        } else {
4011            format!(
4012                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4013                status.state, status.enabled, status.live
4014            )
4015        };
4016        control_error_frame(frame, code, message)
4017    }
4018
4019    async fn handle_route_open(
4020        &self,
4021        ctx: &RouteCtx,
4022        frame: Frame,
4023        request: RouteOpenRequest,
4024        timings: &mut RouteOpenTimings,
4025    ) -> Result<Vec<Frame>, RouterError> {
4026        let RouteOpenRequest {
4027            target,
4028            mut identity,
4029            consumer_identity,
4030            consumer_capabilities,
4031            role_versions,
4032            admission_facts,
4033            scope,
4034        } = request;
4035        let target_module_id = target_module_id(&target).to_string();
4036        timings.module_id.clone_from(&target_module_id);
4037        if self
4038            .registry
4039            .get_module_by_connection(ctx.connection_id)
4040            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4041            .is_some()
4042        {
4043            return Ok(vec![control_error_frame(
4044                &frame,
4045                "invalid_request",
4046                "module connections cannot open client routes",
4047            )?]);
4048        }
4049        debug!(
4050            connection_id = ctx.connection_id.get(),
4051            corr = frame.header.corr,
4052            module_id = %target_module_id,
4053            "handling route.open"
4054        );
4055
4056        // A malformed declaration is refused first, before anything about the
4057        // target is looked up: the same body would be refused against any
4058        // module, so the caller learns nothing by retrying or waiting. An empty
4059        // map declares nothing and travels as no field at all, so a provider
4060        // only ever sees a missing field or a non-empty one.
4061        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4062        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4063            self.observe_route_open_refusal(
4064                ctx,
4065                &target_module_id,
4066                "invalid_role_versions",
4067                error_codes::INVALID_REQUEST,
4068            );
4069            return Ok(vec![control_error_body_frame(
4070                &frame,
4071                ErrorBody {
4072                    code: error_codes::INVALID_REQUEST.to_string(),
4073                    message: error.to_string(),
4074                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4075                },
4076            )?]);
4077        }
4078
4079        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4080            if status.waiting_for_previous_pid.is_some() {
4081                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4082                    ctx,
4083                    &frame,
4084                    &target_module_id,
4085                    "module_warming",
4086                    &status,
4087                )?]);
4088            }
4089        }
4090
4091        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
4092        // opposite. Below, a caller learns whether a module is unregistered,
4093        // supervised-but-down (with state/enabled/live), or registered without the
4094        // requested role. Elsewhere that is an enumeration leak: a probe learning
4095        // the shape of a fleet it cannot otherwise see.
4096        //
4097        // It is not one here, and the reason is the ACCESS MODEL rather than
4098        // anything about these errors. Reaching route.open requires the
4099        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
4100        // connection file, so any caller who completes it already runs as this
4101        // user -- and can read subc.jsonc for the module list and `ck module
4102        // status` for live state. The reply discloses nothing the caller cannot
4103        // read more easily from disk, while the precision is load-bearing:
4104        // `unknown_module` is retryable and a missing role is not.
4105        //
4106        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
4107        // remote transport, a sandboxed caller, a shared-host mode -- THAT
4108        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
4109        let Some(registration) = self
4110            .registry
4111            .get_module(&target_module_id)
4112            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4113        else {
4114            if let Some((status, warming)) =
4115                self.supervisor_status(&target_module_id, frame.header.corr)?
4116            {
4117                // BEFORE the two availability codes below, because for a module
4118                // that speaks no subc wire both of them are false comfort: they
4119                // say "not right now" and are retried, and this module will
4120                // never register no matter how long the caller waits. The
4121                // absence here is the declaration being honoured, not a module
4122                // that is late.
4123                if status.protocol == ModuleProtocol::None {
4124                    return Ok(vec![self.route_open_refusal_frame(
4125                        ctx,
4126                        &frame,
4127                        &target_module_id,
4128                        "protocol_none",
4129                        error_codes::MODULE_NO_PROTOCOL,
4130                        format!(
4131                            "module_id '{target_module_id}' is declared protocol: none; \
4132                             it speaks no subc wire and serves no routes"
4133                        ),
4134                    )?]);
4135                }
4136                let code = if warming {
4137                    "module_warming"
4138                } else {
4139                    "target_unavailable"
4140                };
4141                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4142                    ctx,
4143                    &frame,
4144                    &target_module_id,
4145                    code,
4146                    &status,
4147                )?]);
4148            }
4149            if let Some(removed_ago_ms) =
4150                self.supervisor.removal_tombstone_age_ms(&target_module_id)
4151            {
4152                return Ok(vec![self.route_open_refusal_frame(
4153                    ctx,
4154                    &frame,
4155                    &target_module_id,
4156                    "removed",
4157                    error_codes::MODULE_REMOVED,
4158                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4159                )?]);
4160            }
4161            return Ok(vec![self.route_open_refusal_frame(
4162                ctx,
4163                &frame,
4164                &target_module_id,
4165                "not_registered",
4166                error_codes::UNKNOWN_MODULE,
4167                format!("module_id '{target_module_id}' is not registered"),
4168            )?]);
4169        };
4170
4171        // Best-effort only: registry readiness and forwarding reservation use
4172        // different locks, so a module can flip readiness between this read and
4173        // the relay. Modules must still tolerate an `on_bind` while not ready.
4174        if !registration.ready {
4175            self.counters
4176                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4177            info!(
4178                target: "control",
4179                code = error_codes::MODULE_WARMING,
4180                module_id = ?target_module_id,
4181                connection_id = ctx.connection_id.get(),
4182                reason = "declared_not_ready",
4183                "route.open refused"
4184            );
4185            // The module is registered but says it cannot take work, which is
4186            // an outage from the caller's side even though its process is up.
4187            self.route_outages
4188                .record_not_serving(&target_module_id, "declared_not_ready");
4189            return Ok(vec![control_error_body_frame(
4190                &frame,
4191                ErrorBody {
4192                    code: error_codes::MODULE_WARMING.to_string(),
4193                    message: format!(
4194                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4195                    ),
4196                    detail: Some(serde_json::json!({
4197                        "reason": "declared_not_ready"
4198                    })),
4199                },
4200            )?]);
4201        }
4202
4203        // Effective readiness, second half: a module that declares a capability
4204        // `need: required` is not routable while that capability has no
4205        // registered provider. It is enforced HERE, as a retryable routing
4206        // refusal, and deliberately not as spawn ordering or a boot block. The
4207        // module is still started and registered and can make its own calls;
4208        // spawn ordering is a promise that cannot be kept once a provider
4209        // crashes at runtime, and refusing to boot would stop the whole
4210        // machine, including the tools needed to fix its configuration.
4211        //
4212        // "Provided" is the evaluator's verdict, which counts a provider as
4213        // soon as it has REGISTERED, not once it is ready. Two modules that
4214        // require each other's capabilities are therefore both routable once
4215        // both register; counting readiness instead would deadlock them.
4216        //
4217        // Only new opens are refused. Routes already bound when a provider
4218        // goes away stay bound: nothing here tears them down, and the module
4219        // answers them as it can. Like the readiness read above this is
4220        // best-effort against a provider registering or leaving concurrently.
4221        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
4222            self.counters
4223                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4224            info!(
4225                target: "control",
4226                code = error_codes::MODULE_WARMING,
4227                module_id = ?target_module_id,
4228                connection_id = ctx.connection_id.get(),
4229                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4230                capability = %capability,
4231                "route.open refused"
4232            );
4233            return Ok(vec![control_error_body_frame(
4234                &frame,
4235                ErrorBody {
4236                    code: error_codes::MODULE_WARMING.to_string(),
4237                    message: format!(
4238                        "module_id '{target_module_id}' requires capability '{capability}', \
4239                         which no registered module provides; retry"
4240                    ),
4241                    detail: Some(serde_json::json!({
4242                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4243                        "capability": capability,
4244                    })),
4245                },
4246            )?]);
4247        }
4248
4249        if !target_has_required_role(&target, &registration.manifest.provides) {
4250            return Ok(vec![self.route_open_refusal_frame(
4251                ctx,
4252                &frame,
4253                &target_module_id,
4254                "role_not_provided",
4255                "target_unavailable",
4256                format!("module_id '{target_module_id}' does not provide the requested target"),
4257            )?]);
4258        }
4259
4260        if registration.state != ChannelState::Active {
4261            return Ok(vec![self.route_open_refusal_frame(
4262                ctx,
4263                &frame,
4264                &target_module_id,
4265                "registration_not_active",
4266                "target_unavailable",
4267                format!("module_id '{target_module_id}' is not active"),
4268            )?]);
4269        }
4270
4271        if self
4272            .forwarding
4273            .module_is_draining(&target_module_id)
4274            .map_err(RouterError::Forwarding)?
4275        {
4276            return Ok(vec![self.route_open_refusal_frame(
4277                ctx,
4278                &frame,
4279                &target_module_id,
4280                "reloading",
4281                "module_reloading",
4282                format!("module_id '{target_module_id}' is reloading"),
4283            )?]);
4284        }
4285
4286        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4287            process_liveness.process_live(&target_module_id) == Some(false)
4288        }) {
4289            // A module the supervisor is restarting or reloading can still hold
4290            // a registration: the old process before its connection closes, or
4291            // a new one that registered while the supervisor was draining. The
4292            // forwarding table does not see that as draining, but the consumer
4293            // should still be told to retry soon, exactly as for the drain
4294            // above, rather than that the target is unavailable.
4295            if process_liveness.process_replacing(&target_module_id) {
4296                return Ok(vec![self.route_open_refusal_frame(
4297                    ctx,
4298                    &frame,
4299                    &target_module_id,
4300                    "reloading",
4301                    "module_reloading",
4302                    format!("module_id '{target_module_id}' is reloading"),
4303                )?]);
4304            }
4305            return Ok(vec![self.route_open_refusal_frame(
4306                ctx,
4307                &frame,
4308                &target_module_id,
4309                "supervisor_not_live",
4310                "target_unavailable",
4311                format!("module_id '{target_module_id}' is not live"),
4312            )?]);
4313        }
4314
4315        if !self
4316            .forwarding
4317            .has_live_module_connection(&target_module_id)
4318            .map_err(RouterError::Forwarding)?
4319        {
4320            return Ok(vec![self.route_open_refusal_frame(
4321                ctx,
4322                &frame,
4323                &target_module_id,
4324                "no_forwarding_connection",
4325                "target_unavailable",
4326                format!("module_id '{target_module_id}' has no live forwarding connection"),
4327            )?]);
4328        }
4329
4330        if let Some(error) =
4331            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4332        {
4333            self.observe_route_open_refusal(
4334                ctx,
4335                &target_module_id,
4336                "op_not_allowed",
4337                "op_not_allowed",
4338            );
4339            return Ok(vec![error]);
4340        }
4341
4342        let principal = match self.route_open_principal(&frame, consumer_identity)? {
4343            Ok(principal) => principal,
4344            Err(error) => {
4345                self.observe_route_open_refusal(
4346                    ctx,
4347                    &target_module_id,
4348                    "bad_consumer_identity",
4349                    "bad_consumer_identity",
4350                );
4351                return Ok(vec![error]);
4352            }
4353        };
4354        timings.principal = principal_label(&principal);
4355
4356        // This is attested, control-plane policy for supervised module origins.
4357        // Keep it before route reservation and out of the opaque forwarding hot
4358        // path: data frames must never acquire a per-frame capability check.
4359        if let Principal::Reserved {
4360            module_id: opening_module_id,
4361        } = &principal
4362        {
4363            if let Some(opening_registration) = self
4364                .registry
4365                .get_module(opening_module_id)
4366                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4367            {
4368                if let Some(capability) =
4369                    denied_capability(&opening_registration.manifest, &registration.manifest)
4370                {
4371                    warn!(
4372                        opening_module_id,
4373                        target_module_id,
4374                        capability,
4375                        "refusing route.open because an attested capability deny edge matches"
4376                    );
4377                    return Ok(vec![self.route_open_refusal_frame(
4378                        ctx,
4379                        &frame,
4380                        &target_module_id,
4381                        "capability_deny_edge",
4382                        "capability_forbidden",
4383                        format!(
4384                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4385                        ),
4386                    )?]);
4387                }
4388            }
4389        }
4390
4391        if admission_facts.is_some() {
4392            let carrier_matches = matches!(
4393                &principal,
4394                Principal::Reserved { module_id }
4395                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4396            );
4397            if !carrier_matches {
4398                return Ok(vec![self.route_open_refusal_frame(
4399                    ctx,
4400                    &frame,
4401                    &target_module_id,
4402                    "admission_facts_carrier_not_permitted",
4403                    "admission_facts_not_permitted",
4404                    "admission facts may only be carried by the configured reserved module",
4405                )?]);
4406            }
4407
4408            let target_allowed = self
4409                .admission_facts_targets
4410                .as_ref()
4411                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4412            if !target_allowed {
4413                return Ok(vec![self.route_open_refusal_frame(
4414                    ctx,
4415                    &frame,
4416                    &target_module_id,
4417                    "admission_facts_target_not_listed",
4418                    "admission_facts_target_not_allowed",
4419                    format!(
4420                        "admission facts are not permitted for target module_id '{target_module_id}'"
4421                    ),
4422                )?]);
4423            }
4424
4425            // Keep the value opaque to subc. The downstream admission validator owns
4426            // schema and semantic checks; this daemon only enforces carrier authority
4427            // and the configured destination allowlist.
4428        }
4429
4430        // Scope admission, on the attested principal above and never on the
4431        // request body. The tag read here travels with the pending bind and is
4432        // compared with the published one at commit, so a sync between here
4433        // and the module's ack refuses the open instead of binding a stamp
4434        // that is no longer true.
4435        let (bound_scope, mut scope_stamp) = match &scope {
4436            None => (None, None),
4437            Some(selector) => {
4438                let owner_configured = match &selector.owner {
4439                    // A reserved owner counts as configured from before its
4440                    // process is spawned (see `SupervisorHandle::is_configured`).
4441                    // So an owner that has not synced its scopes yet is refused
4442                    // as retryable (`scope_not_synced`), not as one that will
4443                    // never sync.
4444                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4445                    _ => false,
4446                };
4447                let mut scopes = self
4448                    .scopes
4449                    .write()
4450                    .unwrap_or_else(|poisoned| poisoned.into_inner());
4451                let admitted = scopes
4452                    .admit(&principal, &target_module_id, selector, owner_configured)
4453                    .and_then(|admission| {
4454                        crate::scopes::check_target_flow_support(
4455                            &admission.stamp,
4456                            &target_module_id,
4457                            registration.manifest.capabilities.as_ref(),
4458                        )?;
4459                        crate::scopes::check_target_agent_run_support(
4460                            &admission.stamp,
4461                            &target_module_id,
4462                            registration.manifest.capabilities.as_ref(),
4463                        )?;
4464                        let bound = scopes.reserve_bind(
4465                            &admission,
4466                            &principal,
4467                            ctx.connection_id,
4468                            selector.hold_presence,
4469                        );
4470                        Ok((admission, bound))
4471                    });
4472                match admitted {
4473                    Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4474                    Err(refusal) => {
4475                        return Ok(vec![self.route_open_refusal_frame(
4476                            ctx,
4477                            &frame,
4478                            &target_module_id,
4479                            refusal.code,
4480                            refusal.code,
4481                            refusal.message,
4482                        )?]);
4483                    }
4484                }
4485            }
4486        };
4487
4488        // Bind admits a root that no longer exists on disk, because refusing here
4489        // closes the only exit from a paused run: cancel needs a bound route, and a
4490        // renamed or reclaimed directory makes that route unopenable forever. The
4491        // run itself is intact and still addressable by its recorded identity.
4492        //
4493        // This does NOT relax the rule the strict constructor protects. That rule is
4494        // that no root is ever aliased into NEW durable state -- a missing component
4495        // can reappear as a symlink elsewhere, which would move the identity and
4496        // split a session's history across two of them. The engine now refuses the
4497        // two operations that create such state (send and import) at admission,
4498        // which is a narrower way to hold the same invariant: reads and terminations
4499        // are admitted, writes are not. That refusal had to ship before this line
4500        // changed, or there is an interval where a send commits under a provisional
4501        // identity -- the exact failure the original policy existed to prevent.
4502        //
4503        // Resolution follows realpath rather than lexical cleanup: the longest
4504        // existing ancestor is canonicalized and the missing tail re-appended, so a
4505        // live root is unchanged and a vanished leaf keeps the identity it was
4506        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
4507        // same caller the moment the directory vanished, which strands the run more
4508        // quietly than refusing it.
4509        timings.enter(RouteOpenPhase::ProjectRoot);
4510        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4511            Ok(project_root) => project_root,
4512            Err(err) => {
4513                return Ok(vec![control_error_frame(
4514                    &frame,
4515                    "invalid_project_root",
4516                    err.to_string(),
4517                )?])
4518            }
4519        };
4520        timings.enter(RouteOpenPhase::Admission);
4521        identity.project_root = project_root.as_path().to_path_buf();
4522
4523        // Last gate before any relay work, and deliberately after the cheap
4524        // registry and availability checks above: those name a more precise
4525        // condition (unknown, removed, reloading) and a caller is better served
4526        // by the precise code than by this one.
4527        //
4528        // Everything below this point costs an egress permit, a reserved handle
4529        // pair and, if the module does not answer, the whole relay budget. The
4530        // reader no longer waits for that budget, so cap each target explicitly;
4531        // serial dispatch used to provide the accidental cap of one relay per
4532        // connection. Admission is a mutex-protected count and never waits.
4533        let _concurrency_guard = match self
4534            .route_bind_concurrency
4535            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4536        {
4537            Ok(guard) => guard,
4538            Err(in_flight) => {
4539                return Ok(vec![self.route_open_target_capacity_refusal(
4540                    ctx,
4541                    &frame,
4542                    &target_module_id,
4543                    in_flight,
4544                )?]);
4545            }
4546        };
4547
4548        // A module that has already burned the whole budget `threshold` times
4549        // in a row does not get to charge it again until a probe says it recovered.
4550        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4551            RouteBindAdmission::Admitted { guard, probe } => {
4552                if probe {
4553                    info!(
4554                        module_id = %target_module_id,
4555                        connection_id = ctx.connection_id.get(),
4556                        "route.bind breaker half-open: admitting one probe"
4557                    );
4558                }
4559                guard
4560            }
4561            RouteBindAdmission::Refused {
4562                consecutive_timeouts,
4563                retry_in,
4564                probe_in_flight,
4565            } => {
4566                return Ok(vec![self.route_open_breaker_refusal_frame(
4567                    ctx,
4568                    &frame,
4569                    &target_module_id,
4570                    consecutive_timeouts,
4571                    retry_in,
4572                    probe_in_flight,
4573                )?]);
4574            }
4575        };
4576
4577        // Resolve the per-module budget here so the wait matches the operator's
4578        // intent for this specific target. A per-module override in
4579        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
4580        // daemons) wins over the daemon-wide default.
4581        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4582        let relay_deadline = Instant::now() + route_bind_relay_timeout;
4583        timings.enter(RouteOpenPhase::EgressReservation);
4584        let pending = match self
4585            .forwarding
4586            .begin_route_bind_relay_for(
4587                ctx.connection_id,
4588                ctx.egress.clone(),
4589                response_version(&frame),
4590                frame.header.corr,
4591                &target_module_id,
4592                principal.clone(),
4593                bound_scope,
4594                Some(project_root),
4595                relay_deadline,
4596            )
4597            .await
4598        {
4599            Ok(pending) => pending,
4600            Err(err) => {
4601                return Ok(vec![self.route_open_refusal_frame(
4602                    ctx,
4603                    &frame,
4604                    &target_module_id,
4605                    "relay_reservation_failed",
4606                    forwarding_error_code(&err),
4607                    err.to_string(),
4608                )?])
4609            }
4610        };
4611        timings.enter(RouteOpenPhase::Admission);
4612        let crate::forwarding::PendingRouteBindRelay {
4613            endpoint,
4614            module_sink,
4615            negotiated_ver,
4616            client_channel,
4617            client_epoch,
4618            module_channel,
4619            module_epoch,
4620            corr: relay_corr,
4621            receiver,
4622        } = pending;
4623        let mut reservation =
4624            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4625
4626        // Reserving egress can wait while a module reconnects or a swap cuts
4627        // over. Check the relay's captured connection, not the earlier by-id
4628        // lookup: the original module's flow/run capabilities cannot authorize
4629        // its replacement. The captured sink stays bound to that connection.
4630        let relay_registration = self
4631            .registry
4632            .get_module_by_connection(endpoint.connection_id)
4633            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4634        let relay_capabilities = relay_registration
4635            .as_ref()
4636            .and_then(|registration| registration.manifest.capabilities.as_ref());
4637        if let Some(selector) = &scope {
4638            // Egress reservation may wait across a scope update. Recheck the
4639            // proof against the current record before sending the captured bind.
4640            if let Err(refusal) = self
4641                .scopes
4642                .read()
4643                .unwrap_or_else(|poisoned| poisoned.into_inner())
4644                .check_bind_proof(&principal, selector)
4645            {
4646                reservation.release_and_disarm();
4647                return Ok(vec![self.route_open_refusal_frame(
4648                    ctx,
4649                    &frame,
4650                    &target_module_id,
4651                    refusal.code,
4652                    refusal.code,
4653                    refusal.message,
4654                )?]);
4655            }
4656        }
4657        if let Some(stamp) = scope_stamp
4658            .as_ref()
4659            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4660        {
4661            if let Err(refusal) = crate::scopes::check_target_flow_support(
4662                stamp,
4663                &target_module_id,
4664                relay_capabilities,
4665            )
4666            .and_then(|()| {
4667                crate::scopes::check_target_agent_run_support(
4668                    stamp,
4669                    &target_module_id,
4670                    relay_capabilities,
4671                )
4672            }) {
4673                reservation.release_and_disarm();
4674                return Ok(vec![self.route_open_refusal_frame(
4675                    ctx,
4676                    &frame,
4677                    &target_module_id,
4678                    refusal.code,
4679                    refusal.code,
4680                    refusal.message,
4681                )?]);
4682            }
4683        }
4684
4685        // Older readers may reject unknown fields. Select both new fields from
4686        // the captured endpoint's declaration, including on unscoped binds.
4687        let call_mode = if relay_capabilities
4688            .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4689        {
4690            Some(derive_call_mode(scope_stamp.as_ref()))
4691        } else {
4692            if let Some(stamp) = scope_stamp.as_mut() {
4693                stamp.expires_at_ms = None;
4694            }
4695            None
4696        };
4697
4698        debug!(
4699            connection_id = ctx.connection_id.get(),
4700            client_channel,
4701            client_epoch,
4702            module_channel,
4703            module_epoch,
4704            "reserved route handle pair"
4705        );
4706        // Rendered BEFORE the move into the relay, because the accept arm below
4707        // is where it is logged and the principal is gone by then.
4708        let principal_label = principal_label(&principal);
4709        let relay = ModuleControlRequest::RouteBind {
4710            route_channel: module_channel,
4711            epoch: module_epoch,
4712            target,
4713            identity,
4714            principal: Some(principal),
4715            consumer_capabilities,
4716            role_versions,
4717            admission_facts,
4718            scope: scope_stamp,
4719            call_mode,
4720        };
4721        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4722            RouterError::backend(
4723                0,
4724                frame.header.corr,
4725                format!("failed to encode route.bind request: {err}"),
4726            )
4727        })?;
4728        let relay_frame = Frame::build_with_version(
4729            negotiated_ver,
4730            FrameType::Request,
4731            control_flags(),
4732            0,
4733            0,
4734            relay_corr,
4735            relay_body,
4736        )
4737        .map_err(RouterError::FrameBuild)?;
4738
4739        timings.enter(RouteOpenPhase::BindSend);
4740        let sent = module_sink.send(relay_frame).await;
4741        timings.enter(RouteOpenPhase::Admission);
4742        if let Err(err) = sent {
4743            reservation.release_and_disarm();
4744            return Ok(vec![self.route_open_refusal_frame(
4745                ctx,
4746                &frame,
4747                &target_module_id,
4748                "relay_send_failed",
4749                "target_unavailable",
4750                err.to_string(),
4751            )?]);
4752        }
4753
4754        #[cfg(test)]
4755        if let Some(hold) = &self.route_bind_relay_sent_hold {
4756            // First wait: tell the test the bind is queued. Second wait: stay
4757            // here until the test has driven whatever it wants to happen
4758            // between queueing the bind and marking it enqueued.
4759            hold.wait().await;
4760            hold.wait().await;
4761        }
4762
4763        // A missing relay is not by itself an abandoned bind: the module may
4764        // already have acked it and the route may be committed. Only a relay
4765        // dropped without any module reply needs this GOODBYE; one a reply
4766        // settled has already had its GOODBYE decided by the reply path.
4767        if self
4768            .forwarding
4769            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4770            .map_err(RouterError::Forwarding)?
4771            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4772        {
4773            self.send_abandoned_route_bind_goodbye(
4774                &module_sink,
4775                negotiated_ver,
4776                module_channel,
4777                module_epoch,
4778            );
4779        }
4780
4781        timings.enter(RouteOpenPhase::BindAckWait);
4782        let outcome = timeout_at(relay_deadline, receiver).await;
4783        timings.enter(RouteOpenPhase::Admission);
4784        match outcome {
4785            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4786                reservation.disarm();
4787                if breaker.record_accepted() {
4788                    info!(
4789                        module_id = %target_module_id,
4790                        "route.bind breaker closed: the probe was accepted"
4791                    );
4792                }
4793                self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4794                Ok(Vec::new())
4795            }
4796            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4797                reservation.release_and_disarm();
4798                // A module that says no in microseconds is healthy. Rejection
4799                // is a different condition with its own refusal and must not
4800                // move the breaker.
4801                breaker.record_inconclusive();
4802                // The daemon's own commit re-check refused the bind because the
4803                // scope ended or changed after admission. The module accepted;
4804                // counting it as a module rejection would blame the module.
4805                let scope_code = match body.code.as_str() {
4806                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4807                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4808                    _ => None,
4809                };
4810                if let Some(code) = scope_code {
4811                    self.observe_route_open_refusal(
4812                        ctx,
4813                        &target_module_id,
4814                        "scope_changed_before_commit",
4815                        code,
4816                    );
4817                    return Ok(vec![control_error_body_frame(&frame, body)?]);
4818                }
4819                self.counters
4820                    .increment_route_open_refused("module_rejected");
4821                info!(
4822                    target: "control",
4823                    code = "module_rejected",
4824                    module_code = ?body.code,
4825                    module_id = ?target_module_id,
4826                    connection_id = ctx.connection_id.get(),
4827                    "route.open refused"
4828                );
4829                Ok(vec![control_error_body_frame(&frame, body)?])
4830            }
4831            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4832                reservation.release_and_disarm();
4833                breaker.record_inconclusive();
4834                // Fires when the module's connection closes while a relayed
4835                // bind is pending -- typically a caller racing a module restart
4836                // whose bind was relayed BEFORE the drain mark went up. Logged
4837                // because the caller sees only its own error and the fleet has
4838                // already spent one diagnosis round unable to tell this arm
4839                // from a relay timeout without daemon-side evidence.
4840                tracing::warn!(
4841                    module_id = %target_module_id,
4842                    "route.bind relay abandoned: {message}"
4843                );
4844                Ok(vec![self.route_open_refusal_frame(
4845                    ctx,
4846                    &frame,
4847                    &target_module_id,
4848                    "relay_abandoned",
4849                    "target_unavailable",
4850                    message,
4851                )?])
4852            }
4853            Ok(Err(_)) => {
4854                reservation.release_and_disarm();
4855                breaker.record_inconclusive();
4856                Ok(vec![self.route_open_refusal_frame(
4857                    ctx,
4858                    &frame,
4859                    &target_module_id,
4860                    "relay_waiter_canceled",
4861                    "target_unavailable",
4862                    "route.bind relay waiter was canceled before the module responded",
4863                )?])
4864            }
4865            Err(_) => {
4866                reservation.release_and_disarm();
4867                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
4868                // answer at all is the one condition a fast refusal can
4869                // usefully stand in for; every other arm already answered.
4870                if let Some(opened) = breaker.record_timeout(
4871                    self.route_bind_breaker_threshold,
4872                    self.route_bind_breaker_cooldown,
4873                ) {
4874                    warn!(
4875                        module_id = %target_module_id,
4876                        consecutive_timeouts = opened.consecutive_timeouts,
4877                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4878                        reopened_after_probe = opened.reopened_after_probe,
4879                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4880                    );
4881                }
4882                // The generous budget just burned to no answer: the module is
4883                // registered and its connection is up, but its bind handler sat
4884                // on the ack for the full budget (warm-on-bind, cold configure,
4885                // or a wedged handler). Every earlier unavailability shape
4886                // fast-refuses BEFORE the relay, so this arm firing means the
4887                // slowness is module-side -- log it so the per-module timeline
4888                // is reconstructable without client audit rows.
4889                tracing::warn!(
4890                    module_id = %target_module_id,
4891                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4892                    "route.bind relay timed out: module did not ack within budget"
4893                );
4894                Ok(vec![self.route_open_refusal_frame(
4895                    ctx,
4896                    &frame,
4897                    &target_module_id,
4898                    "relay_timed_out",
4899                    "module_timeout",
4900                    format!(
4901                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
4902                        route_bind_relay_timeout
4903                    ),
4904                )?])
4905            }
4906        }
4907    }
4908
4909    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4910        let response = ClientControlResponse::SupervisorSpawnSnapshot {
4911            snapshot: self.supervisor.spawn_snapshot(),
4912        };
4913        Ok(vec![control_response_body_frame(
4914            &frame,
4915            &response,
4916            "ClientControlResponse::SupervisorSpawnSnapshot",
4917        )?])
4918    }
4919
4920    fn handle_supervisor_spawn_subscribe(
4921        &self,
4922        ctx: &RouteCtx,
4923        frame: Frame,
4924        since: Option<SpawnCursor>,
4925    ) -> Result<Vec<Frame>, RouterError> {
4926        match self.supervisor.subscribe_spawns(
4927            ctx.connection_id,
4928            frame.header.corr,
4929            response_version(&frame),
4930            since,
4931            ctx.egress.clone(),
4932        ) {
4933            Ok(()) => Ok(Vec::new()),
4934            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4935                Ok(vec![control_error_body_frame(
4936                    &frame,
4937                    ErrorBody {
4938                        code: "spawn_cursor_incarnation_mismatch".to_string(),
4939                        message: "spawn cursor belongs to a different daemon incarnation"
4940                            .to_string(),
4941                        detail: Some(serde_json::json!({
4942                            "current_daemon_incarnation": current
4943                        })),
4944                    },
4945                )?])
4946            }
4947            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4948                &frame,
4949                ErrorBody {
4950                    code: "spawn_cursor_too_old".to_string(),
4951                    message: "spawn cursor predates the retained event ring".to_string(),
4952                    detail: Some(serde_json::json!({
4953                        "oldest_retained_cursor": oldest
4954                    })),
4955                },
4956            )?]),
4957            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4958                0,
4959                frame.header.corr,
4960                format!("failed to open supervisor spawn subscription: {error}"),
4961            )),
4962        }
4963    }
4964
4965    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4966        let generation = self
4967            .registry
4968            .generation()
4969            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4970        let mut modules = Vec::new();
4971        for module in self.supervisor.list() {
4972            let status = module.status_for_control("list").map_err(|err| {
4973                RouterError::backend(
4974                    0,
4975                    frame.header.corr,
4976                    format!("failed to read supervisor status: {err}"),
4977                )
4978            })?;
4979            let (configured, _) = module.configuration().map_err(|err| {
4980                RouterError::backend(
4981                    0,
4982                    frame.header.corr,
4983                    format!("failed to read module configuration: {err}"),
4984                )
4985            })?;
4986            // Status and configuration snapshots release their locks before the image probe awaits.
4987            let image = module.running_image_agreement().await;
4988            // Read per request so the figure is current when the operator asks;
4989            // the daemon samples nothing in between.
4990            let resources = Some(module.child_resource_usage());
4991            let pending_reload = Some(reload_verdict(
4992                &configured.program,
4993                status.spawned_from.as_deref(),
4994                image,
4995            ));
4996            // Keep the retired policy field on the wire for one release so
4997            // existing status consumers still receive the platform policy.
4998            modules.push(
4999                SupervisorEntry::new(
5000                    status.module_id,
5001                    status.state.to_string(),
5002                    status.enabled,
5003                    status.live,
5004                    status.health.status,
5005                )
5006                .with_launch_nonce_env(Some(!cfg!(unix)))
5007                .with_launch_nonce_source(status.launch_nonce_source)
5008                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5009                .with_protocol(status.protocol)
5010                .with_pending_reload(pending_reload)
5011                .with_last_probe_ms(status.health.last_probe_ms)
5012                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5013                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5014                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5015                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5016                .with_restart_count(Some(status.restart_count))
5017                .with_max_restarts(Some(status.max_restarts))
5018                .with_lifetime_restarts(Some(status.lifetime_restarts))
5019                .with_spawn_generation(Some(status.spawn_generation))
5020                .with_restart_window_secs(Some(status.restart_window.as_secs()))
5021                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5022                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5023                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5024                .with_resources(resources),
5025            );
5026        }
5027        let response = ClientControlResponse::SupervisorList {
5028            generation,
5029            modules,
5030        };
5031        Ok(vec![control_response_body_frame(
5032            &frame,
5033            &response,
5034            "ClientControlResponse::SupervisorList",
5035        )?])
5036    }
5037
5038    fn handle_supervisor_stderr_tail(
5039        &self,
5040        frame: Frame,
5041        module_id: String,
5042        max_lines: Option<u32>,
5043        max_bytes: Option<u32>,
5044    ) -> Result<Vec<Frame>, RouterError> {
5045        let Some(module) = self.supervisor.get(&module_id) else {
5046            return Ok(vec![control_error_frame(
5047                &frame,
5048                "unknown_module",
5049                format!("module_id '{module_id}' is not supervised"),
5050            )?]);
5051        };
5052
5053        let snapshot = module.stderr_tail(
5054            max_lines.map(|value| value as usize),
5055            max_bytes.map(|value| value as usize),
5056        );
5057
5058        let response = ClientControlResponse::SupervisorStderrTail {
5059            module_id,
5060            tail: StderrTail {
5061                capture: match snapshot.capture {
5062                    CaptureState::Captured => StderrCaptureState::Captured,
5063                    CaptureState::Incomplete { reason } => {
5064                        StderrCaptureState::Incomplete { reason }
5065                    }
5066                    CaptureState::NotCaptured { reason } => {
5067                        StderrCaptureState::NotCaptured { reason }
5068                    }
5069                },
5070                entries: snapshot
5071                    .entries
5072                    .into_iter()
5073                    .map(|entry| match entry {
5074                        TailEntry::Line {
5075                            text,
5076                            truncated,
5077                            at_ms,
5078                        } => StderrTailEntry::Line {
5079                            text,
5080                            truncated,
5081                            at_ms,
5082                        },
5083                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5084                    })
5085                    .collect(),
5086                dropped_lines: snapshot.dropped_lines,
5087            },
5088        };
5089        Ok(vec![control_response_body_frame(
5090            &frame,
5091            &response,
5092            "ClientControlResponse::SupervisorStderrTail",
5093        )?])
5094    }
5095
5096    async fn handle_supervisor_terminals(
5097        &self,
5098        frame: Frame,
5099        module_id: String,
5100    ) -> Result<Vec<Frame>, RouterError> {
5101        let Some(module) = self.supervisor.get(&module_id) else {
5102            return Ok(vec![control_error_frame(
5103                &frame,
5104                "unknown_module",
5105                format!("module_id '{module_id}' is not supervised"),
5106            )?]);
5107        };
5108
5109        // The journal read runs on a blocking thread: it can be megabytes of
5110        // file I/O and must not occupy a runtime worker.
5111        let terminals = module
5112            .read_durable_terminal_history()
5113            .await
5114            .map_err(|error| {
5115                RouterError::backend(
5116                    0,
5117                    frame.header.corr,
5118                    format!("failed to read terminal history: {error}"),
5119                )
5120            })?;
5121        let response = ClientControlResponse::SupervisorTerminals {
5122            module_id,
5123            terminals,
5124        };
5125        Ok(vec![control_response_body_frame(
5126            &frame,
5127            &response,
5128            "ClientControlResponse::SupervisorTerminals",
5129        )?])
5130    }
5131
5132    fn handle_supervisor_routes(
5133        &self,
5134        frame: Frame,
5135        module_id: Option<String>,
5136    ) -> Result<Vec<Frame>, RouterError> {
5137        let modules = self
5138            .forwarding
5139            .route_census(module_id.as_deref())
5140            .map_err(RouterError::Forwarding)?
5141            .into_iter()
5142            .map(|(module_id, routes)| SupervisorRouteModule {
5143                module_id,
5144                routes: routes
5145                    .into_iter()
5146                    .map(|route| SupervisorRoute {
5147                        consumer: match route.principal {
5148                            Principal::Reserved { module_id } => {
5149                                SupervisorRouteConsumer::Reserved { module_id }
5150                            }
5151                            Principal::Direct | Principal::Unverified => {
5152                                SupervisorRouteConsumer::Direct {
5153                                    connection_id: route.goodbye_target.connection_id.get(),
5154                                }
5155                            }
5156                        },
5157                        age_ms: Instant::now()
5158                            .saturating_duration_since(route.bound_at)
5159                            .as_millis()
5160                            .try_into()
5161                            .unwrap_or(u64::MAX),
5162                        draining: route.draining,
5163                        drain_reason: route.drain_reason,
5164                    })
5165                    .collect(),
5166            })
5167            .collect();
5168        let response = ClientControlResponse::SupervisorRoutes { modules };
5169        Ok(vec![control_response_body_frame(
5170            &frame,
5171            &response,
5172            "ClientControlResponse::SupervisorRoutes",
5173        )?])
5174    }
5175
5176    async fn handle_supervisor_provenance(
5177        &self,
5178        frame: Frame,
5179        module_id: Option<String>,
5180    ) -> Result<Vec<Frame>, RouterError> {
5181        let mut selected = if let Some(module_id) = module_id {
5182            let Some(module) = self.supervisor.get(&module_id) else {
5183                return Ok(vec![control_error_frame(
5184                    &frame,
5185                    "unknown_module",
5186                    format!("module_id '{module_id}' is not supervised"),
5187                )?]);
5188            };
5189            vec![module]
5190        } else {
5191            self.supervisor.list()
5192        };
5193
5194        let mut modules = Vec::with_capacity(selected.len());
5195        for module in selected.drain(..) {
5196            let (status, observed_image) = module
5197                .status_and_running_image_agreement()
5198                .await
5199                .map_err(|err| {
5200                    RouterError::backend(
5201                        0,
5202                        frame.header.corr,
5203                        format!("failed to read supervisor status: {err}"),
5204                    )
5205                })?;
5206            let module_declared = self
5207                .registry
5208                .get_module(&status.module_id)
5209                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5210                .and_then(|registration| registration.manifest.provenance)
5211                .map(|build| ModuleDeclaredProvenance::Reported { build })
5212                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5213            #[cfg(test)]
5214            let running_image = match &self.provenance_probe_override {
5215                Some(result) => result.clone(),
5216                None => observed_image,
5217            };
5218            #[cfg(not(test))]
5219            let running_image = observed_image;
5220            modules.push(SupervisorModuleProvenance {
5221                module_id: status.module_id,
5222                module_declared,
5223                daemon_observed: SupervisorObservedProcess {
5224                    launch_nonce_source: status.launch_nonce_source,
5225                    pid: status.pid,
5226                    spawned_at_ms: status.spawned_at_ms,
5227                    spawned_from: status.spawned_from,
5228                    running_image,
5229                },
5230            });
5231        }
5232        let daemon = SupervisorDaemonProvenance {
5233            daemon_build: self.daemon_provenance.build.clone(),
5234            daemon_observed: DaemonObservedProcess {
5235                pid: self.daemon_provenance.pid,
5236                started_at_ms: self
5237                    .daemon_provenance
5238                    .start_clock
5239                    .map(|clock| clock.started_at_ms())
5240                    .or(self.daemon_provenance.started_at_ms),
5241                running_image: self
5242                    .daemon_provenance
5243                    .probe
5244                    .observe(
5245                        self.daemon_provenance.pid,
5246                        self.daemon_provenance.executable_path.as_deref(),
5247                        self.daemon_provenance.executable_identity.clone(),
5248                        self.daemon_provenance.process_start_time,
5249                    )
5250                    .await,
5251            },
5252        };
5253        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5254        Ok(vec![control_response_body_frame(
5255            &frame,
5256            &response,
5257            "ClientControlResponse::SupervisorProvenance",
5258        )?])
5259    }
5260
5261    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5262        self.refresh_capability_requirements();
5263        let generation = self
5264            .registry
5265            .generation()
5266            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5267        let modules = self
5268            .supervisor
5269            .list()
5270            .into_iter()
5271            .map(|module| {
5272                let status = module.status_for_control("health").map_err(|err| {
5273                    RouterError::backend(
5274                        0,
5275                        frame.header.corr,
5276                        format!("failed to read supervisor health: {err}"),
5277                    )
5278                })?;
5279                let module_id = status.module_id;
5280                let capability_detail = self
5281                    .capability_evaluator
5282                    .required_problem_detail(&module_id);
5283                Ok(SupervisorHealthEntry {
5284                    module_id,
5285                    status: status.health.status,
5286                    detail: append_capability_problem_detail(
5287                        status.health.detail,
5288                        capability_detail,
5289                    ),
5290                    metrics: status.health.metrics,
5291                    consecutive_failures: status.health.consecutive_failures,
5292                    late_answer_count: status.health.late_answer_count,
5293                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5294                    last_action: status.health.last_action,
5295                    last_action_ms: status.health.last_action_ms,
5296                    last_probe_ms: status.health.last_probe_ms,
5297                })
5298            })
5299            .collect::<Result<Vec<_>, RouterError>>()?;
5300        let response = ClientControlResponse::SupervisorHealth {
5301            generation,
5302            modules,
5303        };
5304        Ok(vec![control_response_body_frame(
5305            &frame,
5306            &response,
5307            "ClientControlResponse::SupervisorHealth",
5308        )?])
5309    }
5310
5311    async fn handle_supervisor_restart(
5312        &self,
5313        frame: Frame,
5314        module_id: String,
5315        drain_timeout_ms: Option<u64>,
5316        requester: Principal,
5317    ) -> Result<Vec<Frame>, RouterError> {
5318        let operation_lock = self.supervisor.operation_lock();
5319        let _operation_guard = operation_lock.lock().await;
5320        let Some(module) = self.supervisor.get(&module_id) else {
5321            return Ok(vec![control_error_frame(
5322                &frame,
5323                "unknown_module",
5324                format!("module_id '{module_id}' is not supervised"),
5325            )?]);
5326        };
5327
5328        self.route_outages.mark_operator_action(&module_id);
5329        if let Err(err) = module
5330            .restart_requested(drain_timeout_ms, Some(requester))
5331            .await
5332        {
5333            self.route_outages
5334                .operator_action_ended_unrefused(&module_id);
5335            let (code, message) = match err {
5336                crate::supervise::SuperviseError::Disabled { .. } => {
5337                    ("module_disabled", err.to_string())
5338                }
5339                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5340                    ("swap_in_progress", err.to_string())
5341                }
5342                _ => (
5343                    "target_unavailable",
5344                    format!("failed to restart module_id '{module_id}': {err}"),
5345                ),
5346            };
5347            return Ok(vec![control_error_frame(&frame, code, message)?]);
5348        }
5349
5350        let response = ClientControlResponse::SupervisorAck {
5351            module_id,
5352            applied: true,
5353        };
5354        Ok(vec![control_response_body_frame(
5355            &frame,
5356            &response,
5357            "ClientControlResponse::SupervisorAck",
5358        )?])
5359    }
5360
5361    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
5362    /// when the old process has finished draining: a caller whose own lane
5363    /// rides the old process must get its reply before that drain waits on it.
5364    async fn handle_supervisor_swap(
5365        &self,
5366        frame: Frame,
5367        module_id: String,
5368        ready_timeout_ms: Option<u64>,
5369        requester: Principal,
5370    ) -> Result<Vec<Frame>, RouterError> {
5371        // The daemon-wide operation lock is held only to resolve the handle,
5372        // not across the swap. The swap can take its whole readiness budget,
5373        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
5374        // holding it here would park an operator's stop behind the swap it is
5375        // meant to abort. A rescan or stop that reaches the module during the
5376        // swap is served by the swap itself (see `supervise_swap`).
5377        let module = {
5378            let operation_lock = self.supervisor.operation_lock();
5379            let _operation_guard = operation_lock.lock().await;
5380            self.supervisor.get(&module_id)
5381        };
5382        let Some(module) = module else {
5383            return Ok(vec![control_error_frame(
5384                &frame,
5385                "unknown_module",
5386                format!("module_id '{module_id}' is not supervised"),
5387            )?]);
5388        };
5389
5390        self.route_outages.mark_operator_action(&module_id);
5391        if let Err(err) = module
5392            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5393            .await
5394        {
5395            self.route_outages
5396                .operator_action_ended_unrefused(&module_id);
5397            use crate::supervise::SuperviseError;
5398            let message = err.to_string();
5399            let error = match err {
5400                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5401                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5402                    code: "swap_refused".to_string(),
5403                    message,
5404                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5405                },
5406                SuperviseError::SwapFailed {
5407                    arm,
5408                    candidate_exit,
5409                    ..
5410                } => ErrorBody {
5411                    code: "swap_failed".to_string(),
5412                    message,
5413                    detail: Some(serde_json::json!({
5414                        "arm": arm.as_str(),
5415                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5416                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5417                    })),
5418                },
5419                _ => ErrorBody::new(
5420                    "target_unavailable",
5421                    format!("failed to swap module_id '{module_id}': {message}"),
5422                ),
5423            };
5424            return Ok(vec![control_error_body_frame(&frame, error)?]);
5425        }
5426        // A completed swap kept the incumbent serving until cutover, so it
5427        // usually opened no outage; a mark left behind would make the next,
5428        // unrelated outage read as requested.
5429        self.route_outages
5430            .operator_action_ended_unrefused(&module_id);
5431
5432        let response = ClientControlResponse::SupervisorAck {
5433            module_id,
5434            applied: true,
5435        };
5436        Ok(vec![control_response_body_frame(
5437            &frame,
5438            &response,
5439            "ClientControlResponse::SupervisorAck",
5440        )?])
5441    }
5442
5443    async fn handle_supervisor_reload(
5444        &self,
5445        frame: Frame,
5446        module_id: String,
5447        requester: Principal,
5448    ) -> Result<Vec<Frame>, RouterError> {
5449        let operation_lock = self.supervisor.operation_lock();
5450        let _operation_guard = operation_lock.lock().await;
5451        let Some(module) = self.supervisor.get(&module_id) else {
5452            return Ok(vec![control_error_frame(
5453                &frame,
5454                "unknown_module",
5455                format!("module_id '{module_id}' is not supervised"),
5456            )?]);
5457        };
5458
5459        self.route_outages.mark_operator_action(&module_id);
5460        if let Err(err) = module.reload_requested(Some(requester)).await {
5461            self.route_outages
5462                .operator_action_ended_unrefused(&module_id);
5463            let (code, message) = match err {
5464                crate::supervise::SuperviseError::Disabled { .. } => {
5465                    ("module_disabled", err.to_string())
5466                }
5467                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5468                    ("swap_in_progress", err.to_string())
5469                }
5470                _ => (
5471                    "reload_failed",
5472                    format!("failed to reload module_id '{module_id}': {err}"),
5473                ),
5474            };
5475            return Ok(vec![control_error_frame(&frame, code, message)?]);
5476        }
5477
5478        let response = ClientControlResponse::SupervisorAck {
5479            module_id,
5480            applied: true,
5481        };
5482        Ok(vec![control_response_body_frame(
5483            &frame,
5484            &response,
5485            "ClientControlResponse::SupervisorAck",
5486        )?])
5487    }
5488
5489    async fn handle_supervisor_rescan(
5490        &self,
5491        frame: Frame,
5492        preview: bool,
5493        requester: Principal,
5494    ) -> Result<Vec<Frame>, RouterError> {
5495        let Some(context) = self.rescan.clone() else {
5496            return Ok(vec![control_error_frame(
5497                &frame,
5498                "rescan_unavailable",
5499                "the daemon was not started with a reloadable config path".to_string(),
5500            )?]);
5501        };
5502
5503        let operation_lock = self.supervisor.operation_lock();
5504        let _operation_guard = operation_lock.lock().await;
5505        let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5506            Ok(config) => config,
5507            Err(err) => {
5508                return Ok(vec![control_error_frame(
5509                    &frame,
5510                    "invalid_daemon_config",
5511                    format!("supervisor rescan rejected daemon config: {err}"),
5512                )?])
5513            }
5514        };
5515        // `load` reports a missing file as Ok(None), which is correct at boot
5516        // (no config, nothing to supervise) and catastrophic here: rescan treats
5517        // "not in the config" as "remove it", so an absent file would read as an
5518        // empty module list and retire the entire running fleet. An editor
5519        // writing via write-new-then-rename, or a half-finished edit, is enough
5520        // to open that window. Refuse instead: a config that cannot be read
5521        // carries no instruction to remove anything.
5522        let Some((config, activity_config)) = loaded else {
5523            return Ok(vec![control_error_frame(
5524                &frame,
5525                "invalid_daemon_config",
5526                format!(
5527                    "daemon config not found at {}; refusing to rescan (an absent config would \
5528                     retire every supervised module)",
5529                    context.config_path.display()
5530                ),
5531            )?]);
5532        };
5533        let (
5534            configured_port,
5535            storage_config,
5536            admission_facts_carrier_module_id,
5537            admission_facts_targets,
5538            scope_authority_owners,
5539            privileged_operator_prompt,
5540            modules,
5541            reserved_capabilities,
5542        ) = (
5543            config.port,
5544            config.storage,
5545            config.admission_facts_carrier_module_id,
5546            config.admission_facts_targets,
5547            config.scope_authority_owners,
5548            config.privileged_operator_prompt,
5549            config.modules,
5550            config.reserved_capabilities,
5551        );
5552
5553        // Collect the sections rescan cannot apply, so the REPLY carries them.
5554        //
5555        // The warning below has always been correct and has always gone only to
5556        // the journal -- addressed to whoever reads logs, while the person who
5557        // just edited the config is looking at the CLI. Naming each section
5558        // individually rather than setting a flag: "something outside modules
5559        // changed" sends the operator back to diffing their own file, which is
5560        // the work this is meant to save.
5561        let mut restart_required = Vec::new();
5562        for section in RestartRequiredSection::ALL {
5563            let changed = match section {
5564                RestartRequiredSection::Port => configured_port != context.configured_port,
5565                RestartRequiredSection::Storage => storage_config != context.storage_config,
5566                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5567                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5568                }
5569                RestartRequiredSection::AdmissionFactsTargets => {
5570                    admission_facts_targets != context.admission_facts_targets
5571                }
5572                RestartRequiredSection::ScopeAuthorityOwners => {
5573                    scope_authority_owners != context.scope_authority_owners
5574                }
5575                RestartRequiredSection::PrivilegedOperatorPrompt => {
5576                    privileged_operator_prompt != context.privileged_operator_prompt
5577                }
5578            };
5579            if changed {
5580                restart_required.push(section.label().to_string());
5581            }
5582        }
5583        if !restart_required.is_empty() {
5584            warn!(
5585                config_path = %context.config_path.display(),
5586                sections = %restart_required.join(", "),
5587                "daemon config changed outside the modules section; restart the daemon to apply those changes"
5588            );
5589        }
5590
5591        for configured in &modules {
5592            if let Err(err) = validate_spec(&configured.module_spec()) {
5593                return Ok(vec![control_error_frame(
5594                    &frame,
5595                    "invalid_daemon_config",
5596                    format!("supervisor rescan rejected daemon config: {err}"),
5597                )?]);
5598            }
5599        }
5600
5601        let configured_capabilities = modules
5602            .iter()
5603            .map(|module| (module.module_id.clone(), module.enabled))
5604            .collect::<Vec<_>>();
5605        let preview_capability_warnings = if preview {
5606            let (_, registrations) = self.runtime_capability_snapshot()?;
5607            let current_modules = self
5608                .supervisor
5609                .list()
5610                .into_iter()
5611                .map(|module| module.module_id().to_string())
5612                .collect::<BTreeSet<_>>();
5613            let resulting_modules = configured_capabilities.clone();
5614            let removed = current_modules
5615                .into_iter()
5616                .filter(|module_id| {
5617                    !resulting_modules
5618                        .iter()
5619                        .any(|(configured_id, _)| configured_id == module_id)
5620                })
5621                .collect::<Vec<_>>();
5622            self.capability_evaluator.preview_removal_warnings(
5623                resulting_modules,
5624                &removed,
5625                &registrations,
5626            )
5627        } else {
5628            Vec::new()
5629        };
5630        let result = match self
5631            .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5632            .await
5633        {
5634            Ok(result) => result,
5635            Err(message) => {
5636                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5637            }
5638        };
5639        if !preview {
5640            if let Some(ledger) = &self.activity {
5641                ledger.module_rescanned(&result.added, &result.removed, requester);
5642            }
5643            if let Err(code) = self.configure_activity(activity_config).await {
5644                warn!(
5645                    code,
5646                    "activity retention could not be applied during rescan"
5647                );
5648            }
5649            self.capability_evaluator
5650                .configure(configured_capabilities, reserved_capabilities);
5651            self.capability_evaluator.wake_deadline_loop();
5652            self.refresh_capability_requirements();
5653        }
5654        let mut result = result;
5655        result.restart_required = restart_required;
5656        result.capability_warnings = preview_capability_warnings;
5657        let response = ClientControlResponse::SupervisorRescan { result };
5658        Ok(vec![control_response_body_frame(
5659            &frame,
5660            &response,
5661            "ClientControlResponse::SupervisorRescan",
5662        )?])
5663    }
5664
5665    async fn handle_supervisor_release_reserved(
5666        &self,
5667        frame: Frame,
5668        module_id: String,
5669    ) -> Result<Vec<Frame>, RouterError> {
5670        let Some(context) = self.rescan.clone() else {
5671            return Ok(vec![control_error_frame(
5672                &frame,
5673                "release_unavailable",
5674                "reserved-id release requires a daemon started with a reloadable config path",
5675            )?]);
5676        };
5677        let operation_lock = self.supervisor.operation_lock();
5678        let _operation_guard = operation_lock.lock().await;
5679        let loaded = match crate::daemon_config::load(&context.config_path) {
5680            Ok(Some(config)) => config,
5681            Ok(None) => {
5682                return Ok(vec![control_error_frame(
5683                    &frame,
5684                    "invalid_daemon_config",
5685                    format!(
5686                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5687                        context.config_path.display()
5688                    ),
5689                )?])
5690            }
5691            Err(err) => {
5692                return Ok(vec![control_error_frame(
5693                    &frame,
5694                    "invalid_daemon_config",
5695                    format!("unable to verify reserved-id release against daemon config: {err}"),
5696                )?])
5697            }
5698        };
5699        if loaded
5700            .modules
5701            .iter()
5702            .any(|configured| configured.module_id == module_id)
5703        {
5704            return Ok(vec![control_error_frame(
5705                &frame,
5706                "reserved_module_configured",
5707                format!(
5708                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5709                ),
5710            )?]);
5711        }
5712        if !self.supervisor.release_retained_reserved_gate(&module_id) {
5713            return Ok(vec![control_error_frame(
5714                &frame,
5715                "reserved_gate_not_retained",
5716                format!(
5717                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5718                ),
5719            )?]);
5720        }
5721
5722        let response = ClientControlResponse::SupervisorAck {
5723            module_id,
5724            applied: true,
5725        };
5726        Ok(vec![control_response_body_frame(
5727            &frame,
5728            &response,
5729            "ClientControlResponse::SupervisorAck",
5730        )?])
5731    }
5732
5733    /// Reconcile the running module set against the configured one.
5734    ///
5735    /// With `preview` set, the diff is computed and returned WITHOUT applying any
5736    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
5737    /// deliberately shares this function with the executing path rather than
5738    /// computing the same diff somewhere else -- two implementations of one
5739    /// decision agree until they do not, and the whole value of a preview is that
5740    /// it describes the operation that will actually run.
5741    async fn reconcile_supervised_modules(
5742        &self,
5743        supervisor: &Supervisor,
5744        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5745        preview: bool,
5746        requester: Principal,
5747    ) -> Result<SupervisorRescanResult, String> {
5748        let mut current = BTreeMap::new();
5749        for module in self.supervisor.list() {
5750            let (spec, health) = module.configuration().map_err(|err| {
5751                format!(
5752                    "failed to read configuration for module_id '{}': {err}",
5753                    module.module_id()
5754                )
5755            })?;
5756            let enabled = module
5757                .status()
5758                .map_err(|err| {
5759                    format!(
5760                        "failed to read status for module_id '{}': {err}",
5761                        module.module_id()
5762                    )
5763                })?
5764                .enabled;
5765            current.insert(
5766                module.module_id().to_string(),
5767                (module, spec, health, enabled),
5768            );
5769        }
5770        let configured = configured_modules
5771            .into_iter()
5772            .map(|module| (module.module_id.clone(), module))
5773            .collect::<BTreeMap<_, _>>();
5774
5775        let added = configured
5776            .keys()
5777            .filter(|module_id| !current.contains_key(*module_id))
5778            .cloned()
5779            .collect::<Vec<_>>();
5780        let removed = current
5781            .keys()
5782            .filter(|module_id| !configured.contains_key(*module_id))
5783            .cloned()
5784            .collect::<Vec<_>>();
5785        let mut changed_pending_reload = Vec::new();
5786        let mut configuration_changes = BTreeSet::new();
5787        let mut enabled_changes = BTreeSet::new();
5788        let mut unchanged = 0_u32;
5789
5790        for (module_id, configured_module) in &configured {
5791            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5792            else {
5793                continue;
5794            };
5795            // Compare the whole launch spec so a future launch field cannot
5796            // accidentally become a live-only policy change. Health is stored
5797            // separately and applies live without replacing the process.
5798            let launch_changed = *current_spec != configured_module.module_spec();
5799            let configuration_changed =
5800                launch_changed || *current_health != configured_module.health;
5801            let enabled_changed = *current_enabled != configured_module.enabled;
5802            if configuration_changed {
5803                configuration_changes.insert(module_id.clone());
5804            }
5805            if launch_changed {
5806                changed_pending_reload.push(module_id.clone());
5807            }
5808            if enabled_changed {
5809                enabled_changes.insert(module_id.clone());
5810            }
5811            if !configuration_changed && !enabled_changed {
5812                unchanged = unchanged.saturating_add(1);
5813            }
5814        }
5815
5816        // Everything above this point is pure computation over two snapshots.
5817        // Everything below MUTATES. The preview returns here so the boundary is a
5818        // single early return rather than a condition repeated at each mutation
5819        // site, where one missed guard would apply part of a change the caller was
5820        // told would not happen.
5821        if preview {
5822            return Ok(SupervisorRescanResult {
5823                added,
5824                removed,
5825                changed_pending_reload,
5826                enabled_changes: enabled_changes.iter().cloned().collect(),
5827                unchanged,
5828                preview: true,
5829                // Filled by the caller on both paths, so the preview reports
5830                // restart-required sections identically to an executed rescan --
5831                // the preview is where an operator is most likely to be looking.
5832                restart_required: Vec::new(),
5833                capability_warnings: Vec::new(),
5834            });
5835        }
5836
5837        for module_id in &removed {
5838            let module = &current
5839                .get(module_id)
5840                .expect("removed module came from current supervisor state")
5841                .0;
5842            module
5843                .retire_requested(Some(requester.clone()))
5844                .await
5845                .map_err(|err| {
5846                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
5847                })?;
5848            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
5849            //
5850            // `handle_route_open` resolves an absent module in three steps:
5851            // registry, then supervisor status, then tombstone. Retiring first
5852            // opens a window where ALL THREE ARE ABSENT -- the registry entry
5853            // went with the teardown above, the supervisor entry went with
5854            // `retire`, and the tombstone does not exist yet -- so a route.open
5855            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
5856            // it") for a module that was deliberately removed and whose caller
5857            // should get `module_removed` (TERMINAL, carrying a removal age).
5858            //
5859            // Writing the tombstone first closes it: during the window the
5860            // supervisor entry still answers, so the caller gets
5861            // `target_unavailable` -- retryable, and TRUE, because the module
5862            // is mid-teardown. After both statements it is `module_removed`.
5863            // No instant remains where a removed module reads as one that
5864            // never existed.
5865            //
5866            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
5867            // the absence of a test beside a fix invites deletion: these are
5868            // two sync statements with no await between them, so reaching the
5869            // window needs a second worker thread to land exactly between them
5870            // and there is no hook to force it. MEASURED: the 25 daemon_config
5871            // tests pass identically with the old order and the new one, so
5872            // the existing suite cannot see this and a green run is not
5873            // evidence either way. What the suite does hold is the
5874            // post-condition -- a removed module answers `module_removed` --
5875            // which this preserves.
5876            //
5877            // Found by an Athena panel reading the shipped tree against a
5878            // design note (2026-09-19), as the one concrete instance of that
5879            // note's class that survived contact with source. Direction is
5880            // benign: retryable where terminal was intended, never the reverse.
5881            self.supervisor.record_rescan_removal(module_id);
5882            self.supervisor.retire(module_id);
5883            self.route_outages.forget(module_id);
5884        }
5885
5886        for module_id in configured.keys() {
5887            let Some((module, _, _, _)) = current.get(module_id) else {
5888                continue;
5889            };
5890            let configured_module = configured
5891                .get(module_id)
5892                .expect("configured module id came from configured map");
5893            if configuration_changes.contains(module_id) {
5894                module
5895                    .update_configuration(
5896                        configured_module.module_spec(),
5897                        configured_module.health.clone(),
5898                        configured_module.drain_timeout_ms,
5899                    )
5900                    .await
5901                    .map_err(|err| {
5902                        format!(
5903                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
5904                        )
5905                    })?;
5906            }
5907            if enabled_changes.contains(module_id) {
5908                // A rescan that starts or stops a module applies an operator's
5909                // edit to the config, so the resulting outage was asked for.
5910                self.route_outages.mark_operator_action(module_id);
5911                module
5912                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
5913                    .await
5914                    .map_err(|err| {
5915                        self.route_outages.operator_action_ended_unrefused(module_id);
5916                        format!(
5917                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5918                            configured_module.enabled
5919                        )
5920                    })?;
5921            }
5922        }
5923
5924        for module_id in &added {
5925            let configured_module = configured
5926                .get(module_id)
5927                .expect("added module id came from configured map");
5928            supervisor
5929                .supervise_configured_requested(
5930                    configured_module.module_spec(),
5931                    configured_module.enabled,
5932                    configured_module.health.clone(),
5933                    configured_module.drain_timeout_ms,
5934                    configured_module.restart,
5935                    Some(requester.clone()),
5936                )
5937                .map_err(|err| {
5938                    format!("failed to add module_id '{module_id}' during rescan: {err}")
5939                })?;
5940        }
5941
5942        Ok(SupervisorRescanResult {
5943            added,
5944            removed,
5945            changed_pending_reload,
5946            enabled_changes: enabled_changes.iter().cloned().collect(),
5947            unchanged,
5948            preview: false,
5949            // Filled by the caller, which is the only layer that can see the
5950            // previous config to diff against.
5951            restart_required: Vec::new(),
5952            capability_warnings: Vec::new(),
5953        })
5954    }
5955
5956    async fn handle_supervisor_set_enabled(
5957        &self,
5958        frame: Frame,
5959        module_id: String,
5960        enabled: bool,
5961        requester: Principal,
5962    ) -> Result<Vec<Frame>, RouterError> {
5963        let operation_lock = self.supervisor.operation_lock();
5964        let _operation_guard = operation_lock.lock().await;
5965        let Some(module) = self.supervisor.get(&module_id) else {
5966            return Ok(vec![control_error_frame(
5967                &frame,
5968                "unknown_module",
5969                format!("module_id '{module_id}' is not supervised"),
5970            )?]);
5971        };
5972
5973        // Enabling counts as well as disabling: a module an operator starts
5974        // is refused until it registers, and that wait was asked for.
5975        self.route_outages.mark_operator_action(&module_id);
5976        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
5977            Ok(applied) => applied,
5978            Err(err) => {
5979                self.route_outages
5980                    .operator_action_ended_unrefused(&module_id);
5981                return Ok(vec![control_error_frame(
5982                    &frame,
5983                    "target_unavailable",
5984                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5985                )?]);
5986            }
5987        };
5988        if !applied {
5989            // Already in the requested state: nothing was made unavailable,
5990            // so the mark must not outlive this request.
5991            self.route_outages
5992                .operator_action_ended_unrefused(&module_id);
5993        }
5994
5995        self.capability_evaluator.wake_deadline_loop();
5996        self.refresh_capability_requirements();
5997        let response = ClientControlResponse::SupervisorAck { module_id, applied };
5998        Ok(vec![control_response_body_frame(
5999            &frame,
6000            &response,
6001            "ClientControlResponse::SupervisorAck",
6002        )?])
6003    }
6004
6005    async fn handle_supervisor_health_probe(
6006        &self,
6007        frame: Frame,
6008        module_id: String,
6009    ) -> Result<Vec<Frame>, RouterError> {
6010        self.refresh_capability_requirements();
6011        let Some(registration) = self
6012            .registry
6013            .get_module(&module_id)
6014            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6015        else {
6016            return Ok(vec![control_error_frame(
6017                &frame,
6018                "unknown_module",
6019                format!("module_id '{module_id}' is not registered"),
6020            )?]);
6021        };
6022
6023        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
6024        // named for it. Making `module_registration_grants_op` return false
6025        // unconditionally reddens five tests, and every one is named for something
6026        // else -- capability relay, probe/bind demultiplexing, supervision-only
6027        // probing. They exercise a successful advertisement check on the way to their
6028        // own subject.
6029        //
6030        // Real protection, fragile in a specific way: narrowing any of those tests to
6031        // focus on its stated subject would silently remove coverage nobody knows
6032        // they are carrying. Recorded here rather than as a sixth test, because the
6033        // useful fact is WHICH tests hold the guard up -- a new test would add
6034        // coverage without telling the next person what the existing ones quietly do.
6035        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6036        {
6037            return Ok(vec![control_error_frame(
6038                &frame,
6039                "health_not_advertised",
6040                format!("module_id '{module_id}' did not advertise health.check"),
6041            )?]);
6042        }
6043
6044        let deadline = Instant::now() + self.health_probe_timeout;
6045        let pending = match self.forwarding.begin_module_control_rpc_for(
6046            &module_id,
6047            MODULE_CONTROL_OP_HEALTH_CHECK,
6048            deadline,
6049        ) {
6050            Ok(pending) => pending,
6051            Err(err) => {
6052                return Ok(vec![control_error_frame(
6053                    &frame,
6054                    forwarding_error_code(&err),
6055                    err.to_string(),
6056                )?])
6057            }
6058        };
6059
6060        let PendingModuleControlRpc {
6061            endpoint,
6062            module_sink,
6063            negotiated_ver,
6064            corr: probe_corr,
6065            receiver,
6066        } = pending;
6067        let mut guard =
6068            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6069        let probe_body =
6070            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6071                RouterError::backend(
6072                    0,
6073                    frame.header.corr,
6074                    format!("failed to encode health.check request: {err}"),
6075                )
6076            })?;
6077        let probe_frame = Frame::build_with_version(
6078            negotiated_ver,
6079            FrameType::Request,
6080            control_flags(),
6081            0,
6082            0,
6083            probe_corr,
6084            probe_body,
6085        )
6086        .map_err(RouterError::FrameBuild)?;
6087
6088        // Queue admission and the reply share one deadline, so a module that
6089        // stops reading cannot park the requesting connection's reader forever.
6090        match timeout_at(deadline, module_sink.send(probe_frame)).await {
6091            Ok(Ok(())) => {}
6092            Ok(Err(err)) => {
6093                return Ok(vec![control_error_frame(
6094                    &frame,
6095                    "target_unavailable",
6096                    err.to_string(),
6097                )?]);
6098            }
6099            Err(_) => {
6100                return Ok(vec![control_error_frame(
6101                    &frame,
6102                    "module_timeout",
6103                    format!(
6104                        "module_id '{module_id}' did not answer health.check within {:?}",
6105                        self.health_probe_timeout
6106                    ),
6107                )?]);
6108            }
6109        }
6110
6111        match timeout_at(deadline, receiver).await {
6112            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6113                guard.disarm();
6114                let Some(report) = response.health_report() else {
6115                    return Ok(vec![control_error_frame(
6116                        &frame,
6117                        "invalid_control_body",
6118                        "health.check RPC returned a non-health response",
6119                    )?]);
6120                };
6121                // Metrics go out whole here. The supervisor's cached snapshot
6122                // caps this blob (see truncate_health_metrics), and this path
6123                // exists precisely to answer without that cap -- so applying it
6124                // here would leave no way to see what the cached view drops.
6125                let HealthReport {
6126                    status,
6127                    detail,
6128                    metrics,
6129                } = report;
6130                let capability_detail = self
6131                    .capability_evaluator
6132                    .required_problem_detail(&module_id);
6133                let response = ClientControlResponse::SupervisorHealthProbe {
6134                    module_id,
6135                    status,
6136                    detail: append_capability_problem_detail(detail, capability_detail),
6137                    metrics,
6138                };
6139                Ok(vec![control_response_body_frame(
6140                    &frame,
6141                    &response,
6142                    "ClientControlResponse::SupervisorHealthProbe",
6143                )?])
6144            }
6145            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6146                guard.disarm();
6147                Ok(vec![control_error_body_frame(&frame, body)?])
6148            }
6149            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6150                guard.disarm();
6151                Ok(vec![control_error_frame(
6152                    &frame,
6153                    "target_unavailable",
6154                    message,
6155                )?])
6156            }
6157            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6158                guard.disarm();
6159                Ok(vec![control_error_frame(
6160                    &frame,
6161                    "invalid_control_body",
6162                    message,
6163                )?])
6164            }
6165            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6166                guard.disarm();
6167                Ok(vec![control_error_frame(
6168                    &frame,
6169                    "invalid_control_body",
6170                    format!("expected module-control op '{expected}', got '{actual}'"),
6171                )?])
6172            }
6173            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6174                guard.disarm();
6175                Ok(vec![control_error_frame(
6176                    &frame,
6177                    "module_timeout",
6178                    format!(
6179                        "module_id '{module_id}' answered health.check after {:?}",
6180                        self.health_probe_timeout
6181                    ),
6182                )?])
6183            }
6184            Ok(Err(_)) => Ok(vec![control_error_frame(
6185                &frame,
6186                "target_unavailable",
6187                "health.check waiter was canceled before the module responded",
6188            )?]),
6189            Err(_) => Ok(vec![control_error_frame(
6190                &frame,
6191                "module_timeout",
6192                format!(
6193                    "module_id '{module_id}' did not answer health.check within {:?}",
6194                    self.health_probe_timeout
6195                ),
6196            )?]),
6197        }
6198    }
6199
6200    fn supervisor_status(
6201        &self,
6202        module_id: &str,
6203        corr: u64,
6204    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6205        self.supervisor
6206            .get(module_id)
6207            .map(|module| {
6208                let warming = module.is_warming_for_control("status").map_err(|err| {
6209                    RouterError::backend(
6210                        0,
6211                        corr,
6212                        format!(
6213                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
6214                        ),
6215                    )
6216                })?;
6217                module.status_for_control("status").map_err(|err| {
6218                    RouterError::backend(
6219                        0,
6220                        corr,
6221                        format!(
6222                            "failed to read supervisor status for module_id '{module_id}': {err}"
6223                        ),
6224                    )
6225                }).map(|status| (status, warming))
6226            })
6227            .transpose()
6228    }
6229
6230    fn guard_module_control_op(
6231        &self,
6232        frame: &Frame,
6233        module_id: &str,
6234        op: &str,
6235    ) -> Result<Option<Frame>, RouterError> {
6236        if self.module_grants_op(module_id, op, frame.header.corr)? {
6237            return Ok(None);
6238        }
6239
6240        Ok(Some(control_error_frame(
6241            frame,
6242            "op_not_allowed",
6243            format!("module_id '{module_id}' did not grant control op '{op}'"),
6244        )?))
6245    }
6246
6247    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6248        let Some(registration) = self
6249            .registry
6250            .get_module(module_id)
6251            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6252        else {
6253            return Ok(false);
6254        };
6255        Ok(module_registration_grants_op(&registration.control_ops, op))
6256    }
6257
6258    fn handle_status_update(
6259        &self,
6260        endpoint: ModuleEndpointId,
6261        frame: Frame,
6262    ) -> Result<Vec<Frame>, RouterError> {
6263        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6264            Ok(update) => update,
6265            Err(err) => {
6266                // Forward-compat: a newer module may push a channel-0 op this subc
6267                // version doesn't know. The control contract says unknown push ops
6268                // are IGNORED, never answered with an error. Only a malformed body
6269                // for an op we DO know is a real error worth surfacing.
6270                if is_known_module_push_op(&frame.body) {
6271                    return Ok(vec![control_error_frame(
6272                        &frame,
6273                        "invalid_control_body",
6274                        format!("malformed module control push body: {err}"),
6275                    )?]);
6276                }
6277                return Ok(Vec::new());
6278            }
6279        };
6280
6281        match update {
6282            ModuleControlPush::RouteStatus {
6283                route_channel,
6284                route_epoch,
6285                status,
6286            } => {
6287                self.forwarding
6288                    .cache_status(endpoint, route_channel, route_epoch, status)
6289                    .map_err(RouterError::Forwarding)?;
6290            }
6291        }
6292        Ok(Vec::new())
6293    }
6294
6295    fn handle_route_poll(
6296        &self,
6297        ctx: &RouteCtx,
6298        frame: Frame,
6299        route_channel: u16,
6300        route_epoch: u32,
6301        kind: PollKind,
6302    ) -> Result<Vec<Frame>, RouterError> {
6303        let snapshot = self
6304            .forwarding
6305            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6306            .map_err(RouterError::Forwarding)?;
6307        let response = match (kind, snapshot) {
6308            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6309                ClientControlResponse::RoutePoll {
6310                    route_channel,
6311                    route_epoch,
6312                    status,
6313                    live: None,
6314                }
6315            }
6316            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6317                route_channel,
6318                route_epoch,
6319                status: None,
6320                live: None,
6321            },
6322            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6323                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
6324                // is what makes reporting `true` correct rather than a
6325                // confident guess. `process_live` returns None only when the
6326                // module id has no supervisor snapshot at all -- an
6327                // externally-started module the daemon did not spawn -- and
6328                // for those the supervisor has no opinion to offer, ever. It
6329                // is never None for a supervised module in an unknown state:
6330                // a supervised module always has a snapshot, and the answer
6331                // comes from `state == Running && process_alive`.
6332                //
6333                // The route is Bound, so the module completed a HELLO on a
6334                // live connection; "the process this route points at is
6335                // running" is therefore attested by the binding rather than
6336                // assumed. Reporting `false` for an unsupervised module would
6337                // be the actual lie -- it would tell a client its healthy
6338                // route is dead because the daemon does not manage the
6339                // process.
6340                //
6341                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
6342                // module whose liveness is genuinely unknown, e.g. a snapshot
6343                // that has not been populated yet -- THIS DEFAULT BECOMES
6344                // WRONG and must split: unsupervised stays true, unknown
6345                // becomes null so the client can tell the two apart. The
6346                // response field is already `Option<bool>`, so the wire can
6347                // carry that distinction today.
6348                let live = self
6349                    .process_liveness
6350                    .as_ref()
6351                    .and_then(|source| source.process_live(&module_id))
6352                    .unwrap_or(true);
6353                ClientControlResponse::RoutePoll {
6354                    route_channel,
6355                    route_epoch,
6356                    status: None,
6357                    live: Some(live),
6358                }
6359            }
6360            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6361                route_channel,
6362                route_epoch,
6363                status: None,
6364                live: Some(false),
6365            },
6366        };
6367
6368        Ok(vec![control_response_body_frame(
6369            &frame,
6370            &response,
6371            "ClientControlResponse::RoutePoll",
6372        )?])
6373    }
6374
6375    pub(crate) fn observe_module_control_completion(
6376        &self,
6377        completion: ModuleControlRpcCompletion,
6378    ) -> bool {
6379        match completion {
6380            ModuleControlRpcCompletion::Unknown => false,
6381            ModuleControlRpcCompletion::Settled => true,
6382            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6383                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6384                info!(
6385                    module_id = %module_id,
6386                    latency_ms,
6387                    "late health.check answer proves the module is alive"
6388                );
6389                match self
6390                    .supervisor
6391                    .record_late_health_answer(&module_id, latency_ms)
6392                {
6393                    Ok(true) => {}
6394                    Ok(false) => debug!(
6395                        module_id = %module_id,
6396                        latency_ms,
6397                        "late health.check answer has no active supervisor snapshot"
6398                    ),
6399                    Err(err) => warn!(
6400                        module_id = %module_id,
6401                        latency_ms,
6402                        error = %err,
6403                        "failed to record late health.check answer"
6404                    ),
6405                }
6406                true
6407            }
6408        }
6409    }
6410
6411    /// Decide whether a failure while settling a relayed `route.bind` belongs to
6412    /// the module connection whose frame is being handled, or to the client that
6413    /// relay was opened for.
6414    ///
6415    /// This runs on the MODULE connection's frame handler, where returning `Err`
6416    /// ends that connection -- and a module connection carries every client's
6417    /// routes to that module, so ending it costs the whole fleet its tools.
6418    /// `ConnectionClosing` carries the id of the connection that is closing, and
6419    /// when that id is a CLIENT's, the condition is entirely about that one
6420    /// client's route.open. A client-scoped condition has no authority over a
6421    /// shared module connection, so it is logged and the single relay is dropped:
6422    /// the client is going away, and `complete_pending_relay` already removed the
6423    /// relay before failing, so there is nothing left to settle. Anything that
6424    /// relay still reserved is released by that client's own connection teardown,
6425    /// which is already under way -- that is what "closing" means.
6426    ///
6427    /// Every other failure is a statement about THIS connection and stays fatal:
6428    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
6429    /// id in `ConnectionClosing` all mean this connection cannot keep serving
6430    /// frames correctly.
6431    fn refuse_to_end_module_connection_for_a_client(
6432        &self,
6433        module_connection_id: ConnectionId,
6434        corr: u64,
6435        err: ForwardingError,
6436    ) -> Result<(), RouterError> {
6437        if let ForwardingError::ConnectionClosing { connection_id } = err {
6438            if connection_id != module_connection_id {
6439                warn!(
6440                    module_connection_id = module_connection_id.get(),
6441                    client_connection_id = connection_id.get(),
6442                    corr,
6443                    "dropping a route.bind response for a closing client; the module connection keeps serving"
6444                );
6445                return Ok(());
6446            }
6447        }
6448        Err(RouterError::Forwarding(err))
6449    }
6450
6451    fn handle_module_relay_response(
6452        &self,
6453        connection_id: ConnectionId,
6454        frame: Frame,
6455    ) -> Result<Vec<Frame>, RouterError> {
6456        let mut secondary_error = None;
6457        let outcome = match frame.header.ty {
6458            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6459                Ok(probe) if probe.op == "route.bind" => {
6460                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6461                        Ok(ModuleControlResponse::RouteBindAck {}) => {
6462                            RouteBindRelayOutcome::Accepted
6463                        }
6464                        Ok(other) => {
6465                            let message =
6466                                format!("route.bind response carried unexpected body: {other:?}");
6467                            secondary_error = Some(control_error_frame(
6468                                &frame,
6469                                "invalid_control_body",
6470                                message.clone(),
6471                            )?);
6472                            RouteBindRelayOutcome::ModuleGone(message)
6473                        }
6474                        Err(err) => {
6475                            let message = format!("malformed route.bind response body: {err}");
6476                            secondary_error = Some(control_error_frame(
6477                                &frame,
6478                                "invalid_control_body",
6479                                message.clone(),
6480                            )?);
6481                            RouteBindRelayOutcome::ModuleGone(message)
6482                        }
6483                    }
6484                }
6485                Ok(probe) => {
6486                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6487                    {
6488                        Ok(response) => ModuleControlRpcOutcome::Response(response),
6489                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6490                            "malformed {} response body: {err}",
6491                            probe.op
6492                        )),
6493                    };
6494                    let completion = self
6495                        .forwarding
6496                        .complete_module_control_rpc(
6497                            connection_id,
6498                            frame.header.corr,
6499                            Some(&probe.op),
6500                            outcome,
6501                        )
6502                        .map_err(RouterError::Forwarding)?;
6503                    if !self.observe_module_control_completion(completion) {
6504                        debug!(
6505                            connection_id = connection_id.get(),
6506                            corr = frame.header.corr,
6507                            op = %probe.op,
6508                            "dropping late or unknown module-control RPC response"
6509                        );
6510                    }
6511                    return Ok(Vec::new());
6512                }
6513                Err(err) => {
6514                    if let Some(expected_op) = self
6515                        .forwarding
6516                        .pending_module_control_op(connection_id, frame.header.corr)
6517                        .map_err(RouterError::Forwarding)?
6518                    {
6519                        let completion = self
6520                            .forwarding
6521                            .complete_module_control_rpc(
6522                                connection_id,
6523                                frame.header.corr,
6524                                None,
6525                                ModuleControlRpcOutcome::MalformedResponse(format!(
6526                                    "malformed {expected_op} response body: {err}"
6527                                )),
6528                            )
6529                            .map_err(RouterError::Forwarding)?;
6530                        if !self.observe_module_control_completion(completion) {
6531                            debug!(
6532                                connection_id = connection_id.get(),
6533                                corr = frame.header.corr,
6534                                "dropping late malformed module-control RPC response"
6535                            );
6536                        }
6537                        return Ok(Vec::new());
6538                    }
6539                    let message = format!("malformed route.bind response body: {err}");
6540                    secondary_error = Some(control_error_frame(
6541                        &frame,
6542                        "invalid_control_body",
6543                        message.clone(),
6544                    )?);
6545                    RouteBindRelayOutcome::ModuleGone(message)
6546                }
6547            },
6548            FrameType::Error => {
6549                if self
6550                    .forwarding
6551                    .pending_module_control_op(connection_id, frame.header.corr)
6552                    .map_err(RouterError::Forwarding)?
6553                    .is_some()
6554                {
6555                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6556                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6557                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6558                            "malformed module-control ERROR body: {err}"
6559                        )),
6560                    };
6561                    let completion = self
6562                        .forwarding
6563                        .complete_module_control_rpc(
6564                            connection_id,
6565                            frame.header.corr,
6566                            None,
6567                            outcome,
6568                        )
6569                        .map_err(RouterError::Forwarding)?;
6570                    if !self.observe_module_control_completion(completion) {
6571                        debug!(
6572                            connection_id = connection_id.get(),
6573                            corr = frame.header.corr,
6574                            "dropping late or unknown module-control RPC error"
6575                        );
6576                    }
6577                    return Ok(Vec::new());
6578                }
6579                match serde_json::from_slice::<ErrorBody>(&frame.body) {
6580                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
6581                    Err(err) => {
6582                        let message = format!("malformed route.bind ERROR body: {err}");
6583                        secondary_error = Some(control_error_frame(
6584                            &frame,
6585                            "invalid_control_body",
6586                            message.clone(),
6587                        )?);
6588                        RouteBindRelayOutcome::ModuleGone(message)
6589                    }
6590                }
6591            }
6592            ty => {
6593                return Ok(vec![control_error_frame(
6594                    &frame,
6595                    "unsupported_control_frame",
6596                    format!("unsupported module channel-0 frame {ty:?}"),
6597                )?])
6598            }
6599        };
6600
6601        let settled =
6602            self.forwarding
6603                .complete_pending_relay(connection_id, frame.header.corr, outcome);
6604        let completion = match settled {
6605            Ok(completion) => completion,
6606            Err(err) => {
6607                self.refuse_to_end_module_connection_for_a_client(
6608                    connection_id,
6609                    frame.header.corr,
6610                    err,
6611                )?;
6612                return Ok(secondary_error.into_iter().collect());
6613            }
6614        };
6615        if let Some(target) = completion.abandoned.as_ref() {
6616            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6617        }
6618        if !completion.settled {
6619            debug!(
6620                connection_id = connection_id.get(),
6621                corr = frame.header.corr,
6622                frame_type = ?frame.header.ty,
6623                "dropping late or unknown route.bind relay response"
6624            );
6625        }
6626        Ok(secondary_error.into_iter().collect())
6627    }
6628
6629    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6630        debug!(connection_id = connection_id.get(), "handling GOODBYE");
6631        // GOODBYE ends the connection's logical session even when its socket
6632        // stays open. Use disconnect teardown so verdicts, client notices and
6633        // scope authority are released at the same lifecycle boundary.
6634        self.cleanup_connection_with_end_reason(
6635            connection_id,
6636            RegistrationEndReason::ExplicitGoodbye,
6637        )
6638        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6639        Ok(Vec::new())
6640    }
6641}
6642
6643impl Default for ControlHandler {
6644    fn default() -> Self {
6645        Self::new(Arc::new(Registry::default()))
6646    }
6647}
6648
6649impl crate::supervise::SwapPromotionObserver for ControlHandler {
6650    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6651        self.apply_registration_capabilities(registration);
6652    }
6653}
6654
6655fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6656    CapabilityRequirementStatus {
6657        consumer: status.consumer,
6658        capability: status.capability,
6659        need: match status.need {
6660            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6661            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6662        },
6663        verdict: status.verdict.as_str().to_string(),
6664        episode_seq: status.episode_seq,
6665        config_satisfiable: status.config_satisfiable,
6666        runtime_available: status.runtime_available,
6667        detail: status.detail,
6668    }
6669}
6670
6671fn append_capability_problem_detail(
6672    detail: Option<String>,
6673    capability_detail: Option<String>,
6674) -> Option<String> {
6675    match (detail, capability_detail) {
6676        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6677        (Some(detail), None) => Some(detail),
6678        (None, Some(capability_detail)) => Some(capability_detail),
6679        (None, None) => None,
6680    }
6681}
6682
6683fn subc_ops() -> Vec<String> {
6684    SUBC_CONTROL_OPS
6685        .iter()
6686        .map(|op| (*op).to_string())
6687        .collect()
6688}
6689
6690fn module_subc_ops() -> Vec<String> {
6691    SUBC_CONTROL_OPS
6692        .iter()
6693        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6694        .map(|op| (*op).to_string())
6695        .collect()
6696}
6697
6698#[cfg(test)]
6699fn module_baseline_control_ops() -> Vec<String> {
6700    MODULE_BASELINE_CONTROL_OPS
6701        .iter()
6702        .map(|op| (*op).to_string())
6703        .collect()
6704}
6705
6706fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6707    let mut seen = HashSet::new();
6708    let mut effective = Vec::new();
6709    for op in MODULE_BASELINE_CONTROL_OPS {
6710        if seen.insert((*op).to_string()) {
6711            effective.push((*op).to_string());
6712        }
6713    }
6714    for op in declared.unwrap_or_default() {
6715        if seen.insert(op.clone()) {
6716            effective.push(op);
6717        }
6718    }
6719    effective
6720}
6721
6722fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6723    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6724}
6725
6726fn target_module_id(target: &RouteTarget) -> &str {
6727    match target {
6728        RouteTarget::ToolProvider { module_id }
6729        | RouteTarget::ManagementSurface { module_id }
6730        | RouteTarget::InternalService { module_id, .. } => module_id,
6731    }
6732}
6733
6734fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6735    roles.iter().any(|role| match (target, role) {
6736        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6737        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6738        (
6739            RouteTarget::InternalService { service_id, .. },
6740            ProviderRole::InternalService {
6741                service_id: provided,
6742                ..
6743            },
6744        ) => service_id == provided,
6745        _ => false,
6746    })
6747}
6748
6749fn is_routable_role(role: &ProviderRole) -> bool {
6750    matches!(
6751        role,
6752        ProviderRole::ToolProvider { .. }
6753            | ProviderRole::ManagementSurface { .. }
6754            | ProviderRole::InternalService { .. }
6755    )
6756}
6757
6758#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6759enum ControlRequestBodyError {
6760    UnknownOp,
6761    InvalidBody,
6762}
6763
6764#[derive(Debug, Deserialize)]
6765struct ControlOpProbe {
6766    op: String,
6767}
6768
6769/// Channel-0 push ops this subc version understands. A push whose `op` is not in
6770/// this set is treated as a forward-compat unknown and ignored rather than errored.
6771const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6772
6773fn is_known_module_push_op(body: &[u8]) -> bool {
6774    serde_json::from_slice::<ControlOpProbe>(body)
6775        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6776        .unwrap_or(false)
6777}
6778
6779fn is_known_module_request_op(body: &[u8]) -> bool {
6780    serde_json::from_slice::<ControlOpProbe>(body)
6781        .map(|probe| is_module_to_subc_op(&probe.op))
6782        .unwrap_or(false)
6783}
6784
6785fn is_module_to_subc_op(op: &str) -> bool {
6786    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6787}
6788
6789fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6790    debug!(
6791        op = %op,
6792        connection_id = connection_id.get(),
6793        corr,
6794        "control dispatch"
6795    );
6796}
6797
6798fn log_slow_control_dispatch(
6799    dispatch_started_at: Option<StdInstant>,
6800    op: &'static str,
6801    connection_id: ConnectionId,
6802    corr: u64,
6803    route_open_timings: Option<&RouteOpenTimings>,
6804) {
6805    let Some(dispatch_started_at) = dispatch_started_at else {
6806        return;
6807    };
6808    // Use the same monotonic clock as the phase timers and relay deadline.
6809    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
6810    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6811        if let Some(timings) = route_open_timings {
6812            warn!(
6813                op = %op,
6814                connection_id = connection_id.get(),
6815                corr,
6816                elapsed_ms = elapsed.as_millis() as u64,
6817                module_id = ?timings.module_id,
6818                principal = %timings.principal,
6819                admission_ms = timings.admission.as_millis() as u64,
6820                project_root_ms = timings.project_root.as_millis() as u64,
6821                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
6822                bind_send_ms = timings.bind_send.as_millis() as u64,
6823                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
6824                "slow control dispatch"
6825            );
6826            return;
6827        }
6828        warn!(
6829            op = %op,
6830            connection_id = connection_id.get(),
6831            corr,
6832            elapsed_ms = elapsed.as_millis() as u64,
6833            "slow control dispatch"
6834        );
6835    }
6836}
6837
6838/// A test stand-in for the OS caller lookup, given the recorded endpoints.
6839#[cfg(test)]
6840type TestCallerLookup =
6841    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
6842
6843/// Every new client operation needs an explicit privilege decision here.
6844fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
6845    match request {
6846        ClientControlRequest::SupervisorRestart { module_id, .. } => {
6847            Some(PrivilegedOperation::Restart { module_id })
6848        }
6849        ClientControlRequest::SupervisorSwap { module_id, .. } => {
6850            Some(PrivilegedOperation::Swap { module_id })
6851        }
6852        ClientControlRequest::SupervisorReload { module_id } => {
6853            Some(PrivilegedOperation::Reload { module_id })
6854        }
6855        ClientControlRequest::SupervisorRescan { preview } => {
6856            if *preview {
6857                None
6858            } else {
6859                Some(PrivilegedOperation::Rescan)
6860            }
6861        }
6862        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
6863            Some(PrivilegedOperation::SetEnabled {
6864                module_id,
6865                enabled: *enabled,
6866            })
6867        }
6868        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
6869            Some(PrivilegedOperation::ReleaseReserved { module_id })
6870        }
6871        ClientControlRequest::ServerDescribe {}
6872        | ClientControlRequest::CatalogList { .. }
6873        | ClientControlRequest::RouteOpen { .. }
6874        | ClientControlRequest::RoutePoll { .. }
6875        | ClientControlRequest::SupervisorList {}
6876        | ClientControlRequest::SupervisorHealth {}
6877        | ClientControlRequest::SupervisorHealthProbe { .. }
6878        | ClientControlRequest::SupervisorRoutes { .. }
6879        | ClientControlRequest::SupervisorProvenance { .. }
6880        | ClientControlRequest::SupervisorStderrTail { .. }
6881        | ClientControlRequest::SupervisorTerminals { .. }
6882        | ClientControlRequest::SupervisorSpawnSnapshot {}
6883        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
6884    }
6885}
6886
6887fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6888    match request {
6889        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6890        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6891        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6892        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6893        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6894        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6895        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6896        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6897        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6898        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6899        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6900        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6901        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6902        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6903        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6904        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6905        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6906        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6907        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6908    }
6909}
6910
6911fn principal_label(principal: &Principal) -> String {
6912    match principal {
6913        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6914        Principal::Direct => "direct".to_string(),
6915        other => format!("{other:?}"),
6916    }
6917}
6918
6919fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6920    match request {
6921        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6922        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6923        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6924        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6925        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6926    }
6927}
6928
6929fn parse_client_control_request(
6930    body: &[u8],
6931) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6932    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6933        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6934            Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6935                ControlRequestBodyError::InvalidBody
6936            }
6937            Ok(_) => ControlRequestBodyError::UnknownOp,
6938            Err(_) => ControlRequestBodyError::InvalidBody,
6939        };
6940        (err, classification)
6941    })
6942}
6943
6944fn parse_module_control_request_from_module(
6945    body: &[u8],
6946) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6947    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6948        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6949            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6950            Ok(_) => ControlRequestBodyError::UnknownOp,
6951            Err(_) => ControlRequestBodyError::InvalidBody,
6952        };
6953        (err, classification)
6954    })
6955}
6956
6957#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6958enum ProviderRoleKind {
6959    ToolProvider,
6960    PipelineStage,
6961    ManagementSurface,
6962    InternalService,
6963}
6964
6965fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6966    match role {
6967        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6968        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6969        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6970        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6971    }
6972}
6973
6974fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6975    roles.iter().map(provider_role_kind).collect()
6976}
6977
6978/// Most refused scope records named individually in the log per sync; the
6979/// `refused` count on the accepted line is always complete.
6980const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6981
6982/// Per-outcome counts of one accepted `scope.sync`, for its log line.
6983#[derive(Debug, Default, PartialEq, Eq)]
6984struct ScopeOutcomeCounts {
6985    created: usize,
6986    replaced: usize,
6987    updated: usize,
6988    unchanged: usize,
6989    refused: usize,
6990}
6991
6992impl ScopeOutcomeCounts {
6993    fn of(results: &[ScopeRecordResult]) -> Self {
6994        let mut counts = Self::default();
6995        for result in results {
6996            let slot = match result.outcome {
6997                ScopeRecordOutcome::Created => &mut counts.created,
6998                ScopeRecordOutcome::Replaced => &mut counts.replaced,
6999                ScopeRecordOutcome::Updated => &mut counts.updated,
7000                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7001                ScopeRecordOutcome::Refused => &mut counts.refused,
7002            };
7003            *slot += 1;
7004        }
7005        counts
7006    }
7007}
7008
7009#[cfg(test)]
7010mod scope_outcome_count_tests {
7011    use super::*;
7012
7013    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7014        ScopeRecordResult {
7015            scope_ref: "r".to_string(),
7016            scope_epoch: 1,
7017            outcome,
7018            code: None,
7019            message: None,
7020            version: None,
7021            parent_state: None,
7022        }
7023    }
7024
7025    /// Each outcome lands in its own count, so a refused record can never be
7026    /// hidden inside the total the log already printed.
7027    #[test]
7028    fn every_outcome_is_counted_in_its_own_field() {
7029        let results = [
7030            result(ScopeRecordOutcome::Created),
7031            result(ScopeRecordOutcome::Created),
7032            result(ScopeRecordOutcome::Replaced),
7033            result(ScopeRecordOutcome::Updated),
7034            result(ScopeRecordOutcome::Unchanged),
7035            result(ScopeRecordOutcome::Refused),
7036            result(ScopeRecordOutcome::Refused),
7037            result(ScopeRecordOutcome::Refused),
7038        ];
7039        assert_eq!(
7040            ScopeOutcomeCounts::of(&results),
7041            ScopeOutcomeCounts {
7042                created: 2,
7043                replaced: 1,
7044                updated: 1,
7045                unchanged: 1,
7046                refused: 3,
7047            }
7048        );
7049    }
7050}
7051
7052/// Return whether a catalog change can create a newly violating live route.
7053/// Removing an attested claim is intentionally excluded: it makes fewer routes
7054/// forbidden and therefore must leave the existing route census untouched.
7055fn capability_census_trigger(
7056    old: Option<&CapabilityDeclarations>,
7057    new: Option<&CapabilityDeclarations>,
7058) -> bool {
7059    let old_provides = old
7060        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7061        .unwrap_or_default();
7062    let old_denies = old
7063        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7064        .unwrap_or_default();
7065    let new = new.cloned().unwrap_or(CapabilityDeclarations {
7066        provides: Vec::new(),
7067        requires: Vec::new(),
7068        must_never_reach: Vec::new(),
7069    });
7070
7071    new.provides
7072        .iter()
7073        .any(|capability| !old_provides.contains(capability))
7074        || new
7075            .must_never_reach
7076            .iter()
7077            .any(|capability| !old_denies.contains(capability))
7078}
7079
7080/// Find the first capability an attested opener denies that an attested target
7081/// claims. Both manifests are live registry records, never cached or client data.
7082fn denied_capability<'a>(
7083    opening_manifest: &'a ModuleManifest,
7084    target_manifest: &ModuleManifest,
7085) -> Option<&'a str> {
7086    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7087    let target_capabilities = target_manifest.capabilities.as_ref()?;
7088    opening_capabilities
7089        .must_never_reach
7090        .iter()
7091        .find(|denied| {
7092            target_capabilities
7093                .provides
7094                .iter()
7095                .any(|provided| provided == *denied)
7096        })
7097        .map(String::as_str)
7098}
7099
7100fn catalog_update_frozen_field_message(
7101    registered: &ModuleManifest,
7102    provides: &[ProviderRole],
7103) -> Option<String> {
7104    let old_has_provides = !registered.provides.is_empty();
7105    let new_has_provides = !provides.is_empty();
7106    if old_has_provides != new_has_provides {
7107        return Some(format!(
7108            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7109            registered.module_id
7110        ));
7111    }
7112
7113    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
7114        return Some(format!(
7115            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7116            registered.module_id
7117        ));
7118    }
7119
7120    let registered_concurrency = manifest_concurrency(registered);
7121    let mut candidate = registered.clone();
7122    candidate.provides = provides.to_vec();
7123    let candidate_concurrency = manifest_concurrency(&candidate);
7124    if candidate_concurrency != registered_concurrency {
7125        return Some(format!(
7126            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7127            registered.module_id, registered_concurrency, candidate_concurrency
7128        ));
7129    }
7130
7131    // control_ops live beside the manifest in the HELLO body, not inside
7132    // ModuleManifest, so a provides-only catalog.update cannot change them.
7133    None
7134}
7135
7136fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7137    manifest.provides.iter().any(is_routable_role)
7138}
7139
7140/// Returns the routable-provider concurrency subc should enforce for this manifest.
7141///
7142/// ToolProvider and ManagementSurface store their delivery concurrency directly.
7143/// InternalService has no role-specific concurrency field, so it retains the
7144/// existing ModuleManaged default for backward compatibility.
7145fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7146    manifest
7147        .provides
7148        .iter()
7149        .find_map(|provider| match provider {
7150            ProviderRole::ToolProvider { concurrency, .. }
7151            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7152            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7153        })
7154        .unwrap_or(Concurrency::ModuleManaged)
7155}
7156
7157/// True when the manifest carries a ManagementSurface role whose concurrency
7158/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
7159/// bytes because the typed manifest deliberately erases that distinction: the
7160/// default exists for wire compatibility, and this probe exists so the default
7161/// stays observable. Any parse irregularity returns false -- the caller only
7162/// logs, and a malformed body already failed registration upstream.
7163fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7164    let has_management_surface = manifest
7165        .provides
7166        .iter()
7167        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7168    if !has_management_surface {
7169        return false;
7170    }
7171    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7172        return false;
7173    };
7174    let Some(provides) = raw
7175        .get("manifest")
7176        .and_then(|manifest| manifest.get("provides"))
7177        .and_then(serde_json::Value::as_array)
7178    else {
7179        return false;
7180    };
7181    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
7182    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
7183    // against the management_surface_manifest_without_concurrency golden, not
7184    // recalled (the externally-tagged guess was this function's first bug).
7185    provides.iter().any(|role| {
7186        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7187            && role.get("concurrency").is_none()
7188    })
7189}
7190
7191fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7192    if peer_version != PROTOCOL_VERSION {
7193        return Err(format!(
7194            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7195        ));
7196    }
7197    Ok(PROTOCOL_VERSION)
7198}
7199
7200fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7201    Frame::build_with_version(
7202        response_version(frame),
7203        FrameType::Pong,
7204        frame.header.flags,
7205        0,
7206        0,
7207        frame.header.corr,
7208        Vec::new(),
7209    )
7210    .map_err(RouterError::FrameBuild)
7211}
7212
7213pub(crate) fn control_error_frame(
7214    frame: &Frame,
7215    code: &'static str,
7216    message: impl Into<String>,
7217) -> Result<Frame, RouterError> {
7218    control_error_body_frame(
7219        frame,
7220        ErrorBody {
7221            code: code.to_string(),
7222            message: message.into(),
7223            detail: None,
7224        },
7225    )
7226}
7227
7228fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7229    let body = serde_json::to_vec(&error).map_err(|err| {
7230        RouterError::backend(
7231            0,
7232            frame.header.corr,
7233            format!("failed to encode control ERROR: {err}"),
7234        )
7235    })?;
7236
7237    Frame::build_with_version(
7238        response_version(frame),
7239        FrameType::Error,
7240        control_flags(),
7241        0,
7242        0,
7243        frame.header.corr,
7244        body,
7245    )
7246    .map_err(RouterError::FrameBuild)
7247}
7248
7249fn control_response_body_frame<T: Serialize>(
7250    frame: &Frame,
7251    reply: &T,
7252    label: &'static str,
7253) -> Result<Frame, RouterError> {
7254    let body = serde_json::to_vec(reply).map_err(|err| {
7255        RouterError::backend(
7256            0,
7257            frame.header.corr,
7258            format!("failed to encode {label}: {err}"),
7259        )
7260    })?;
7261
7262    Frame::build_with_version(
7263        response_version(frame),
7264        FrameType::Response,
7265        control_flags(),
7266        0,
7267        0,
7268        frame.header.corr,
7269        body,
7270    )
7271    .map_err(RouterError::FrameBuild)
7272}
7273
7274/// Map a forwarding failure to the wire code a client sees.
7275///
7276/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
7277/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
7278/// chosen here decides whether a caller retries or gives up.
7279///
7280/// That makes attribution the load-bearing property, not merely having a code. A
7281/// permanent fault published as a retryable one produces a fleet-wide retry storm
7282/// against something that can never recover; a transient fault published as
7283/// permanent gives up on work that would have succeeded. Both look correct in a
7284/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
7285/// the mapping per variant rather than merely asserting that some code exists.
7286///
7287/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
7288/// two codes on the same side of the boundary passes it. Measured rather than
7289/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
7290/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
7291/// a test named for something else that happens to assert the string.
7292///
7293/// That accidental coverage is deliberately left alone rather than promoted to a
7294/// named test, because it guards a property this function does not promise.
7295/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
7296/// specific code within a class, so identity is free to change and only the
7297/// partition is a contract. Splitting it out would assert a guarantee nothing
7298/// depends on — and a suite that promises more than the code does is the harder
7299/// thing to correct later, because the next reader cannot tell which assertions
7300/// are load-bearing.
7301///
7302/// Pin identity here the moment a consumer branches on a specific code.
7303fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7304    match err {
7305        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7306        ForwardingError::NoModuleConnection => "target_unavailable",
7307        ForwardingError::ModuleReloading { .. } => "module_reloading",
7308        ForwardingError::ClientRouteChannelExhausted { .. }
7309        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7310        ForwardingError::StaleModuleEndpoint
7311        | ForwardingError::UnknownReservation { .. }
7312        | ForwardingError::ConnectionClosing { .. }
7313        | ForwardingError::ClientEgressClosed { .. }
7314        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7315        // Only a swap candidate's registration can produce this, and it means
7316        // exactly what a second active HELLO for a live id means.
7317        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7318        ForwardingError::RelayCorrelationExhausted
7319        | ForwardingError::RouteOpenBuild(_)
7320        | ForwardingError::Poisoned => "forwarding_error",
7321    }
7322}
7323
7324fn response_version(frame: &Frame) -> u8 {
7325    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7326        frame.header.ver
7327    } else {
7328        PROTOCOL_VERSION
7329    }
7330}
7331
7332fn control_flags() -> Flags {
7333    Flags::new(false, Priority::Passive, false)
7334}
7335
7336/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
7337/// channel. The target is the module (a client never saw the route), so this
7338/// takes the module path: delivered late rather than dropped when the module's
7339/// queue is momentarily full, and never closing its connection.
7340fn send_goodbye_target_best_effort(
7341    counters: &DaemonCounters,
7342    target: &GoodbyeTarget,
7343    context: &'static str,
7344) {
7345    let Ok(frame) = Frame::build_with_version(
7346        target.negotiated_ver,
7347        FrameType::Goodbye,
7348        control_flags(),
7349        target.channel,
7350        target.epoch,
7351        0,
7352        Vec::new(),
7353    ) else {
7354        return;
7355    };
7356    crate::forwarding::send_module_route_goodbye(
7357        counters,
7358        &target.sink,
7359        frame,
7360        target.module_id.as_deref(),
7361        context,
7362    );
7363}
7364
7365pub(crate) fn send_route_control_pushes(
7366    forwarding: &ForwardingTable,
7367    routes: Vec<EndpointRoute>,
7368    push: ClientControlPush,
7369) {
7370    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7371    for route in routes {
7372        let target = route.goodbye_target;
7373        if let Some((existing, channels)) = targets
7374            .iter_mut()
7375            .find(|(existing, _)| existing.connection_id == target.connection_id)
7376        {
7377            debug_assert_eq!(
7378                existing.negotiated_ver, target.negotiated_ver,
7379                "one connection cannot negotiate multiple frame versions"
7380            );
7381            if !channels.contains(&target.channel) {
7382                channels.push(target.channel);
7383            }
7384            continue;
7385        }
7386        let channel = target.channel;
7387        targets.push((target, vec![channel]));
7388    }
7389    for (target, mut channels) in targets {
7390        channels.sort_unstable();
7391        let mut push = push.clone();
7392        match &mut push {
7393            ClientControlPush::RouteClosing {
7394                channels: covered, ..
7395            }
7396            | ClientControlPush::RouteClosed {
7397                channels: covered, ..
7398            } => *covered = channels,
7399        }
7400        let body = match serde_json::to_vec(&push) {
7401            Ok(body) => body,
7402            Err(err) => {
7403                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7404                continue;
7405            }
7406        };
7407        let frame = match Frame::build_with_version(
7408            target.negotiated_ver,
7409            FrameType::Push,
7410            control_flags(),
7411            0,
7412            0,
7413            0,
7414            body.clone(),
7415        ) {
7416            Ok(frame) => frame,
7417            Err(err) => {
7418                warn!(
7419                    route_channel = target.channel,
7420                    error = %err,
7421                    "failed to build route lifecycle control PUSH frame"
7422                );
7423                continue;
7424            }
7425        };
7426        if let Err(err) = target.sink.try_send(frame) {
7427            if target.close_on_delivery_failure() {
7428                warn!(
7429                    target_connection_id = target.connection_id.get(),
7430                    route_channel = target.channel,
7431                    error = %err,
7432                    "route lifecycle control PUSH was not delivered to client; closing target connection"
7433                );
7434                let _ = forwarding.escalate_client_delivery_failure(
7435                    target.connection_id,
7436                    target.channel,
7437                    target.epoch,
7438                    CloseReason::new(
7439                        "route_lifecycle_push_delivery_failed",
7440                        format!(
7441                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7442                            target.channel
7443                        ),
7444                    ),
7445                    crate::forwarding::UndeliveredFrame {
7446                        module_id: target.module_id.as_deref(),
7447                        sink: &target.sink,
7448                    },
7449                );
7450            }
7451        }
7452    }
7453}
7454
7455#[cfg(test)]
7456mod tests {
7457    mod activity_read {
7458        include!("activity_read_tests.rs");
7459    }
7460
7461    mod lifecycle_activity {
7462        include!("control_lifecycle_tests.rs");
7463    }
7464    #[cfg(unix)]
7465    #[tokio::test]
7466    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7467        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7468        let path = dir.join("subc.jsonc");
7469        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7470            "program":"/bin/sleep","args":["60"],"protocol":"none",
7471            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7472        }}}).to_string()).unwrap();
7473        let mut configured = crate::daemon_config::load(&path)
7474            .unwrap()
7475            .unwrap()
7476            .modules
7477            .pop()
7478            .unwrap();
7479        let registry = std::sync::Arc::new(crate::Registry::default());
7480        let handle = crate::SupervisorHandle::new();
7481        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7482            .with_handle(handle.clone());
7483        let module = supervisor
7484            .supervise_configured_with_health(
7485                configured.module_spec(),
7486                true,
7487                configured.health.clone(),
7488                None,
7489                configured.restart,
7490            )
7491            .unwrap();
7492        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7493        let before = module.status().unwrap().pid;
7494        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7495        configured.health.cadence = std::time::Duration::from_secs(3600);
7496        let health_only = handler
7497            .reconcile_supervised_modules(
7498                &supervisor,
7499                vec![configured.clone()],
7500                false,
7501                Principal::Direct,
7502            )
7503            .await
7504            .unwrap();
7505        assert!(
7506            health_only.changed_pending_reload.is_empty(),
7507            "health policy is already applied live"
7508        );
7509        assert_eq!(module.status().unwrap().pid, before);
7510        assert_eq!(
7511            module.configuration().unwrap().1.http,
7512            configured.health.http
7513        );
7514        configured.args = vec!["61".into()];
7515        let launch = handler
7516            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7517            .await
7518            .unwrap();
7519        assert_eq!(launch.changed_pending_reload, ["stock"]);
7520        assert_eq!(
7521            module.status().unwrap().pid,
7522            before,
7523            "a launch edit is stored until reload"
7524        );
7525        module.drain().await.unwrap();
7526    }
7527    use cortexkit_test_support::ScratchDir;
7528    use std::{
7529        collections::BTreeMap,
7530        fmt,
7531        path::PathBuf,
7532        sync::{Arc, Mutex},
7533        time::Duration,
7534    };
7535
7536    use serde_json::{json, Value};
7537    use subc_protocol::{
7538        manifest::{
7539            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7540            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7541        },
7542        session::HealthStatus,
7543        FrameType,
7544    };
7545
7546    use super::*;
7547    use crate::{
7548        forwarding::{DataRoute, DataRouteState},
7549        registry::ChannelState,
7550        router::FrameSink,
7551        stderr_tail::DEFAULT_MAX_LINE_BYTES,
7552        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7553        RouteCtx, Router,
7554    };
7555    use tokio::{
7556        sync::mpsc,
7557        time::{sleep, Instant},
7558    };
7559    use tracing::{
7560        field::{Field, Visit},
7561        Event, Subscriber,
7562    };
7563    use tracing_subscriber::{layer::Context, prelude::*, Layer};
7564
7565    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
7566    ///
7567    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
7568    /// is only populated for `tests/*.rs` integration test binaries -- this file
7569    /// compiles as part of the library target, which gets neither. This test's
7570    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
7571    /// and the sibling binary lives two directories up at
7572    /// `<target-dir>/<profile>/fake-aft-stub`.
7573    ///
7574    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
7575    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
7576    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
7577    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
7578    /// `NotFound`, which reads as a broken test rather than an unbuilt
7579    /// dependency -- so state the cause and the remedy instead. Deliberately a
7580    /// panic and not a silent skip: a test that quietly passes when it could not
7581    /// run is worse than one that fails, because it reports health it never
7582    /// verified.
7583    fn fake_aft_stub_path() -> PathBuf {
7584        let mut path = std::env::current_exe().expect("current_exe available in tests");
7585        path.pop(); // .../deps/
7586        path.pop(); // .../<profile>/
7587        path.push(if cfg!(windows) {
7588            "fake-aft-stub.exe"
7589        } else {
7590            "fake-aft-stub"
7591        });
7592        assert!(
7593            path.exists(),
7594            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7595             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7596            path.display()
7597        );
7598        path
7599    }
7600
7601    /// Whether clients retry `code` in place: the predicate itself, never a copy
7602    /// of its set. A copied list breaks silently when a code is added to or
7603    /// removed from the real one, and a stale copy here would let exactly the
7604    /// failure this test exists to catch pass.
7605    fn client_retries(code: &str) -> bool {
7606        subc_protocol::error_codes::is_retryable_route_open(code)
7607    }
7608
7609    /// A code is not a label — clients branch on it, so publishing the wrong KIND
7610    /// of failure is worse than publishing none. A permanent fault dressed as
7611    /// retryable makes every client in the fleet retry forever against something
7612    /// that cannot recover; a transient fault dressed as permanent abandons work
7613    /// that would have succeeded.
7614    ///
7615    /// Asserting "a code exists" cannot catch either, because the string is free
7616    /// to say anything. This enumerates every variant and pins which side of the
7617    /// retry boundary it lands on, so a new variant must be classified here
7618    /// deliberately rather than inheriting whichever arm it was appended to.
7619    #[test]
7620    fn retryability_of_forwarding_codes_matches_the_failure() {
7621        // Transient by nature: the target is booting, reloading, or its endpoint
7622        // was swapped mid-flight. Retrying is how these resolve.
7623        let transient = [
7624            ForwardingError::NoModuleConnection,
7625            ForwardingError::ModuleReloading {
7626                module_id: "m".into(),
7627            },
7628            ForwardingError::StaleModuleEndpoint,
7629            ForwardingError::UnknownReservation {
7630                client_channel: 1,
7631                module_channel: 1,
7632            },
7633            ForwardingError::ConnectionClosing {
7634                connection_id: ConnectionId::new(1),
7635            },
7636            ForwardingError::ClientEgressClosed {
7637                connection_id: ConnectionId::new(1),
7638            },
7639            ForwardingError::ModuleEgressUnavailable {
7640                connection_id: ConnectionId::new(1),
7641            },
7642        ];
7643        for err in transient {
7644            let code = forwarding_error_code(&err);
7645            assert!(
7646                client_retries(code),
7647                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7648            );
7649        }
7650
7651        // Not fixed by retrying. Channel and correlation exhaustion need the
7652        // caller to close routes, and a poisoned lock is a daemon that cannot
7653        // recover at all — the worst thing to advertise as retryable, since every
7654        // client would storm a daemon that will never answer.
7655        let permanent = [
7656            ForwardingError::ConnectionRoleConflict {
7657                connection_id: ConnectionId::new(1),
7658            },
7659            ForwardingError::ClientRouteChannelExhausted {
7660                connection_id: ConnectionId::new(1),
7661            },
7662            ForwardingError::ModuleRouteChannelExhausted {
7663                endpoint: ModuleEndpointId {
7664                    connection_id: ConnectionId::new(1),
7665                    generation: 1,
7666                },
7667            },
7668            ForwardingError::RelayCorrelationExhausted,
7669            ForwardingError::RouteOpenBuild("x".into()),
7670            ForwardingError::Poisoned,
7671        ];
7672        for err in permanent {
7673            let code = forwarding_error_code(&err);
7674            assert!(
7675                !client_retries(code),
7676                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7677            );
7678        }
7679    }
7680
7681    /// The principal is the daemon's answer to "who is calling", and modules
7682    /// branch on it: aft gates bash on it, cerebellum gates browser control,
7683    /// plexus gates connector invocation. So a stamp is an authorization input in
7684    /// another process, not a label — and both possible answers SUCCEED, which is
7685    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
7686    /// first-party capability to something that never proved it; a supervised one
7687    /// stamped `Direct` silently strips a module of capability it is entitled to.
7688    ///
7689    /// Neither shows up in a test that only checks the bind succeeded. Before this
7690    /// test the only coverage was accidental —
7691    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
7692    /// stamped principal on its way past, so narrowing that wire-shape test to its
7693    /// stated subject would have deleted the last assertion on this value. It
7694    /// still asserts the stamp, which is now redundancy rather than the only
7695    /// guard: both fail under the same mutation, and this one names the reason.
7696    /// SCOPE: this handler's supervisor has spawned nothing, so
7697    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
7698    /// is unreachable here. Both assertions below are refusals, and a mutant that
7699    /// refuses everything would satisfy them.
7700    ///
7701    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
7702    /// spawns a supervised consumer, reads its live nonce, and asserts the module
7703    /// observed `principal.kind == "reserved"` carrying that module_id — verified
7704    /// at source rather than assumed, since a citation is a claim about another
7705    /// file and ages like one. Recorded because a harness that structurally
7706    /// cannot reach an arm reports "none" for that arm identically to one that
7707    /// covers it and found nothing.
7708    #[tokio::test]
7709    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7710        let handler = ControlHandler::default();
7711        let frame =
7712            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7713
7714        // Absent consumer_identity is the ordinary case: a human at a terminal, or
7715        // any process holding the connection file. Nothing was proved, so nothing
7716        // may be granted beyond the unattested floor.
7717        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7718        assert_eq!(
7719            stamped,
7720            Principal::Direct,
7721            "a caller that proved nothing must not be stamped as a supervised module"
7722        );
7723
7724        // A claimed module_id with a nonce no supervised child was given is a
7725        // forgery attempt, not a weaker caller: it must be REFUSED rather than
7726        // quietly demoted to Direct, or an impersonation attempt looks identical
7727        // to an ordinary unattested connection.
7728        let forged = handler
7729            .route_open_principal(
7730                &frame,
7731                Some(ConsumerIdentity {
7732                    module_id: "aft".to_string(),
7733                    launch_nonce: "not-a-real-nonce".to_string(),
7734                }),
7735            )
7736            .unwrap();
7737        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7738        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7739    }
7740
7741    /// The test above hands `route_open_principal` an identity it built itself,
7742    /// which proves the stamping rule and nothing about where the identity comes
7743    /// from. The real producer is a wire body, and the two are joined by a serde
7744    /// field name that nothing else asserts.
7745    ///
7746    /// That join fails quietly in one specific way: an unrecognised key is simply
7747    /// absent after parsing, so a renamed or misspelled `consumer_identity`
7748    /// yields `None` and every supervised module silently drops to `Direct`.
7749    /// Capability-wise that is the safe direction, but it surfaces far from its
7750    /// cause — as a module mysteriously refused bash — and it would pass every
7751    /// test that builds its own input.
7752    ///
7753    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
7754    /// would break every client the moment the daemon gains a field, trading a
7755    /// quiet demotion for a hard refusal on additive change. Asserting the join
7756    /// instead means a rename breaks a test here rather than the fleet.
7757    #[test]
7758    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7759        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7760        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7761        let ClientControlRequest::RouteOpen {
7762            consumer_identity, ..
7763        } = parsed
7764        else {
7765            panic!("route.open body must parse as RouteOpen");
7766        };
7767        assert_eq!(
7768            consumer_identity,
7769            Some(ConsumerIdentity {
7770                module_id: "aft".to_string(),
7771                launch_nonce: "n".to_string(),
7772            }),
7773            "the wire field name must reach the value route_open_principal reads"
7774        );
7775    }
7776
7777    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7778        ModuleManifest::builder(module_id, "0.1.0")
7779            .protocol_ver(protocol_ver)
7780            .provides(vec![ProviderRole::ToolProvider {
7781                tools: vec![Tool {
7782                    name: "read".to_string(),
7783                    description: None,
7784                    execution_mode: ExecutionMode::Pure,
7785                    schema: json!({"type": "object"}),
7786                }],
7787                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7788                concurrency: Concurrency::ModuleManaged,
7789                emits_push: true,
7790                sub_supervises: true,
7791            }])
7792            .build()
7793    }
7794
7795    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7796        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7797    }
7798
7799    fn hello_frame_with_control_ops(
7800        module_id: &str,
7801        protocol_ver: u8,
7802        corr: u64,
7803        control_ops: Option<Vec<String>>,
7804    ) -> Frame {
7805        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7806    }
7807
7808    fn hello_frame_with_nonce(
7809        module_id: &str,
7810        protocol_ver: u8,
7811        corr: u64,
7812        launch_nonce: Option<&str>,
7813    ) -> Frame {
7814        hello_frame_full(
7815            module_id,
7816            protocol_ver,
7817            corr,
7818            None,
7819            launch_nonce.map(ToOwned::to_owned),
7820        )
7821    }
7822
7823    fn hello_frame_full(
7824        module_id: &str,
7825        protocol_ver: u8,
7826        corr: u64,
7827        control_ops: Option<Vec<String>>,
7828        launch_nonce: Option<String>,
7829    ) -> Frame {
7830        let body = serde_json::to_vec(&ModuleHelloBody {
7831            manifest: manifest(module_id, protocol_ver),
7832            protocol_ver,
7833            control_ops,
7834            launch_nonce,
7835        })
7836        .unwrap();
7837        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7838    }
7839
7840    fn non_routable_hello_frame_with_control_ops(
7841        module_id: &str,
7842        corr: u64,
7843        control_ops: Option<Vec<String>>,
7844    ) -> Frame {
7845        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7846        manifest.provides.clear();
7847        let body = serde_json::to_vec(&ModuleHelloBody {
7848            manifest,
7849            protocol_ver: PROTOCOL_VERSION,
7850            control_ops,
7851            launch_nonce: None,
7852        })
7853        .unwrap();
7854        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7855    }
7856
7857    fn capability_grammar_hello_frame(
7858        capabilities: Value,
7859        runtime_computed: Option<Value>,
7860        corr: u64,
7861    ) -> Frame {
7862        let mut body = serde_json::to_value(ModuleHelloBody {
7863            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7864            protocol_ver: PROTOCOL_VERSION,
7865            control_ops: None,
7866            launch_nonce: None,
7867        })
7868        .expect("HELLO body serializes");
7869        body["manifest"]["capabilities"] = capabilities;
7870        if let Some(runtime_computed) = runtime_computed {
7871            body["runtime_computed"] = runtime_computed;
7872        }
7873        Frame::build(
7874            FrameType::Hello,
7875            control_flags(),
7876            0,
7877            0,
7878            corr,
7879            serde_json::to_vec(&body).expect("HELLO body reserializes"),
7880        )
7881        .expect("HELLO frame builds")
7882    }
7883
7884    fn channel_request(channel: u16, corr: u64) -> Frame {
7885        Frame::build(
7886            FrameType::Request,
7887            Flags::new(true, Priority::Interactive, false),
7888            channel,
7889            0,
7890            corr,
7891            b"opaque".to_vec(),
7892        )
7893        .unwrap()
7894    }
7895
7896    fn route_ctx(
7897        connection_id: ConnectionId,
7898    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7899        let (tx, rx) = mpsc::channel(8);
7900        (
7901            RouteCtx {
7902                connection_id,
7903                egress: FrameSink::new(tx),
7904            },
7905            rx,
7906        )
7907    }
7908
7909    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7910        serde_json::from_slice(&frame.body).unwrap()
7911    }
7912
7913    /// Register a module over a connection that has a sink and return the
7914    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
7915    /// module's own sink rather than returning it as a reply, so the ack is
7916    /// taken off `rx` here and whatever the test reads next is what followed it.
7917    async fn hello_via_sink(
7918        handler: &ControlHandler,
7919        ctx: &RouteCtx,
7920        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7921        hello: Frame,
7922    ) -> Frame {
7923        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7924        assert!(
7925            replies.is_empty(),
7926            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7927        );
7928        let ack = rx
7929            .try_recv()
7930            .expect("HELLO_ACK is queued on the module sink")
7931            .frame;
7932        assert_eq!(ack.header.ty, FrameType::HelloAck);
7933        ack
7934    }
7935
7936    fn parse_error(frame: &Frame) -> Value {
7937        serde_json::from_slice(&frame.body).unwrap()
7938    }
7939
7940    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7941        serde_json::from_slice(&frame.body).unwrap()
7942    }
7943
7944    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7945        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7946            route_channel,
7947            route_epoch: 0,
7948            kind,
7949        })
7950        .unwrap();
7951        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7952    }
7953
7954    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7955        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7956            module_id: module_id.to_string(),
7957        })
7958        .unwrap();
7959        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7960    }
7961
7962    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7963        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7964    }
7965
7966    fn route_open_frame_with_consumer_capabilities(
7967        corr: u64,
7968        module_id: &str,
7969        project_root: ScratchDir,
7970        consumer_capabilities: Option<Vec<String>>,
7971    ) -> Frame {
7972        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7973            target: RouteTarget::ToolProvider {
7974                module_id: module_id.to_string(),
7975            },
7976            identity: BindIdentity::new(
7977                project_root.path().to_path_buf(),
7978                "unit".to_string(),
7979                "session".to_string(),
7980            ),
7981            consumer_identity: None,
7982            consumer_capabilities,
7983            role_versions: None,
7984            admission_facts: None,
7985            scope: None,
7986        })
7987        .unwrap();
7988        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7989    }
7990
7991    fn route_open_frame_with_role_versions(
7992        corr: u64,
7993        module_id: &str,
7994        project_root: ScratchDir,
7995        role_versions: Option<BTreeMap<String, String>>,
7996    ) -> Frame {
7997        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7998            target: RouteTarget::ToolProvider {
7999                module_id: module_id.to_string(),
8000            },
8001            identity: BindIdentity::new(
8002                project_root.path().to_path_buf(),
8003                "unit".to_string(),
8004                format!("session-{corr}"),
8005            ),
8006            consumer_identity: None,
8007            consumer_capabilities: None,
8008            role_versions,
8009            admission_facts: None,
8010            scope: None,
8011        })
8012        .unwrap();
8013        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8014    }
8015
8016    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8017        entries
8018            .iter()
8019            .map(|(role, version)| (role.to_string(), version.to_string()))
8020            .collect()
8021    }
8022
8023    fn route_open_frame_with_admission_facts(
8024        corr: u64,
8025        module_id: &str,
8026        project_root: ScratchDir,
8027        consumer_identity: Option<subc_control::ConsumerIdentity>,
8028        facts: Option<Value>,
8029    ) -> Frame {
8030        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8031            target: RouteTarget::ToolProvider {
8032                module_id: module_id.to_string(),
8033            },
8034            identity: BindIdentity::new(
8035                project_root.path().to_path_buf(),
8036                "unit".to_string(),
8037                format!("session-{corr}"),
8038            ),
8039            consumer_identity,
8040            consumer_capabilities: None,
8041            role_versions: None,
8042            admission_facts: facts,
8043            scope: None,
8044        })
8045        .unwrap();
8046        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8047    }
8048
8049    #[derive(Clone, Default)]
8050    struct EventCapture {
8051        events: Arc<Mutex<Vec<CapturedEvent>>>,
8052    }
8053
8054    #[derive(Clone, Debug)]
8055    struct CapturedEvent {
8056        target: String,
8057        level: tracing::Level,
8058        fields: BTreeMap<String, String>,
8059    }
8060
8061    impl EventCapture {
8062        fn events(&self) -> Vec<CapturedEvent> {
8063            self.events.lock().unwrap().clone()
8064        }
8065    }
8066
8067    impl<S> Layer<S> for EventCapture
8068    where
8069        S: Subscriber,
8070    {
8071        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8072            let mut visitor = EventFieldVisitor::default();
8073            event.record(&mut visitor);
8074            self.events.lock().unwrap().push(CapturedEvent {
8075                target: event.metadata().target().to_string(),
8076                level: *event.metadata().level(),
8077                fields: visitor.fields,
8078            });
8079        }
8080    }
8081
8082    #[derive(Default)]
8083    struct EventFieldVisitor {
8084        fields: BTreeMap<String, String>,
8085    }
8086
8087    impl Visit for EventFieldVisitor {
8088        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8089            self.fields
8090                .insert(field.name().to_string(), format!("{value:?}"));
8091        }
8092    }
8093
8094    fn health_response(corr: u64, status: HealthStatus) -> Frame {
8095        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8096            status,
8097            detail: Some("warming".to_string()),
8098            metrics: Some(json!({"queue_depth": 3})),
8099        })
8100        .unwrap();
8101        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8102    }
8103
8104    fn route_bind_ack(corr: u64) -> Frame {
8105        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8106        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8107    }
8108
8109    fn unique_project_root(label: &str) -> ScratchDir {
8110        ScratchDir::new(label)
8111    }
8112
8113    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8114        match parse_route_poll(frame) {
8115            ClientControlResponse::RoutePoll {
8116                status: None,
8117                live: Some(live),
8118                ..
8119            } => assert_eq!(live, expected_live),
8120            other => panic!("unexpected route.poll response: {other:?}"),
8121        }
8122    }
8123
8124    fn bind_liveness_route(
8125        registry: &Registry,
8126        forwarding: &ForwardingTable,
8127        module_id: &str,
8128    ) -> (RouteCtx, u16, u32) {
8129        let module_connection = ConnectionId::new(101);
8130        let client_connection = ConnectionId::new(202);
8131        let registration = registry
8132            .register_with_control_ops(
8133                manifest(module_id, PROTOCOL_VERSION),
8134                PROTOCOL_VERSION,
8135                module_connection,
8136                module_baseline_control_ops(),
8137            )
8138            .unwrap();
8139        let (module_tx, _module_rx) = mpsc::channel(8);
8140        let endpoint = forwarding
8141            .register_module_connection(
8142                module_connection,
8143                module_id.to_string(),
8144                PROTOCOL_VERSION,
8145                manifest_concurrency(&registration.manifest),
8146                FrameSink::new(module_tx),
8147            )
8148            .unwrap();
8149        let (client_ctx, _client_rx) = route_ctx(client_connection);
8150        let pending = forwarding
8151            .begin_route_bind_relay_for_test(
8152                client_connection,
8153                client_ctx.egress.clone(),
8154                1,
8155                module_id,
8156            )
8157            .unwrap();
8158        assert_eq!(pending.endpoint, endpoint);
8159        let route_channel = pending.client_channel;
8160        let route_epoch = pending.client_epoch;
8161        forwarding
8162            .complete_pending_relay(
8163                module_connection,
8164                pending.corr,
8165                RouteBindRelayOutcome::Accepted,
8166            )
8167            .unwrap();
8168        (client_ctx, route_channel, route_epoch)
8169    }
8170
8171    struct FakeProcessLiveness {
8172        live: Option<bool>,
8173    }
8174
8175    impl ModuleProcessLiveness for FakeProcessLiveness {
8176        fn process_live(&self, _module_id: &str) -> Option<bool> {
8177            self.live
8178        }
8179    }
8180
8181    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8182    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8183    {
8184        let registry = Arc::new(Registry::default());
8185        let supervisor_handle = SupervisorHandle::new();
8186        let supervisor = Supervisor::new_for_test(
8187            Arc::clone(&registry),
8188            RestartPolicy::new(1, Duration::from_millis(10)),
8189        )
8190        .with_handle(supervisor_handle.clone());
8191        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8192        let module = supervisor
8193            .spawn(ModuleSpec {
8194                module_id: "stderr-tail-wire".to_string(),
8195                program: fake_aft_stub_path(),
8196                args: Vec::new(),
8197                env: vec![
8198                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8199                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8200                ],
8201                reserved: false,
8202                reserved_prefixes: Vec::new(),
8203                protocol: ModuleProtocol::Subc,
8204                overlap: Default::default(),
8205            })
8206            .unwrap();
8207
8208        let deadline = Instant::now() + Duration::from_secs(5);
8209        loop {
8210            let tail = module.stderr_tail(None, None);
8211            if tail
8212                .entries
8213                .iter()
8214                .any(|entry| matches!(entry, TailEntry::ProcessStart))
8215                && tail.entries.iter().any(|entry| {
8216                    matches!(
8217                        entry,
8218                        TailEntry::Line {
8219                            truncated: true,
8220                            ..
8221                        }
8222                    )
8223                })
8224            {
8225                break;
8226            }
8227            assert!(
8228                Instant::now() < deadline,
8229                "module did not produce a truncated line and restart boundary: {tail:?}"
8230            );
8231            sleep(Duration::from_millis(10)).await;
8232        }
8233
8234        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8235        let request = ClientControlRequest::SupervisorStderrTail {
8236            module_id: "stderr-tail-wire".to_string(),
8237            max_lines: None,
8238            max_bytes: None,
8239        };
8240        let frame = Frame::build(
8241            FrameType::Request,
8242            control_flags(),
8243            0,
8244            0,
8245            1,
8246            serde_json::to_vec(&request).unwrap(),
8247        )
8248        .unwrap();
8249        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8250        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8251        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8252            serde_json::from_slice(&responses[0].body).unwrap()
8253        else {
8254            panic!("expected supervisor.stderr_tail response");
8255        };
8256
8257        assert!(
8258            tail.entries
8259                .iter()
8260                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8261            "the control response lost the restart boundary"
8262        );
8263        let Some(StderrTailEntry::Line {
8264            text,
8265            truncated,
8266            at_ms,
8267        }) = tail.entries.iter().find(|entry| {
8268            matches!(
8269                entry,
8270                StderrTailEntry::Line {
8271                    truncated: true,
8272                    ..
8273                }
8274            )
8275        })
8276        else {
8277            panic!("the control response lost the truncated line");
8278        };
8279        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8280        assert!(*truncated);
8281        assert!(
8282            at_ms.is_some(),
8283            "the control response lost the line's capture time"
8284        );
8285    }
8286
8287    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
8288    /// read done on the worker thread would stall every other task until it
8289    /// finished; the read must run off the worker so this test's own task keeps
8290    /// running while the read is paused.
8291    #[tokio::test(flavor = "current_thread")]
8292    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8293        let dir = ScratchDir::new("terminals-off-worker");
8294        let journal_path = dir.join("terminals.jsonl");
8295        let registry = Arc::new(Registry::default());
8296        let supervisor_handle = SupervisorHandle::new();
8297        let supervisor =
8298            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8299                .with_handle(supervisor_handle.clone())
8300                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8301        let module = supervisor
8302            .spawn(ModuleSpec {
8303                module_id: "terminal-off-worker".to_string(),
8304                program: fake_aft_stub_path(),
8305                args: Vec::new(),
8306                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8307                reserved: false,
8308                reserved_prefixes: Vec::new(),
8309                protocol: ModuleProtocol::Subc,
8310                overlap: Default::default(),
8311            })
8312            .unwrap();
8313        let deadline = Instant::now() + Duration::from_secs(5);
8314        while module.terminal_history().entries.len() != 2 {
8315            assert!(Instant::now() < deadline, "module did not record two exits");
8316            sleep(Duration::from_millis(10)).await;
8317        }
8318
8319        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8320        let handler =
8321            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
8322        let frame = Frame::build(
8323            FrameType::Request,
8324            control_flags(),
8325            0,
8326            0,
8327            1,
8328            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8329                module_id: "terminal-off-worker".to_string(),
8330            })
8331            .unwrap(),
8332        )
8333        .unwrap();
8334        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8335        let spawned_at = std::time::Instant::now();
8336        let read = tokio::spawn({
8337            let handler = Arc::clone(&handler);
8338            async move { handler.handle_control_frame(&ctx, frame).await }
8339        });
8340        // Waiting for the pause from a blocking thread keeps this task pending,
8341        // so the runtime's single worker is free to run the read task.
8342        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8343            .await
8344            .unwrap()
8345            .expect("the history read reached its pause");
8346        let elapsed = spawned_at.elapsed();
8347        assert!(
8348            elapsed < Duration::from_secs(2) && !read.is_finished(),
8349            "this task could not run while the history read was paused \
8350             (resumed after {elapsed:?}, read finished: {})",
8351            read.is_finished()
8352        );
8353
8354        drop(release);
8355        let responses = read.await.unwrap().unwrap();
8356        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8357        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8358            panic!("expected supervisor.terminals response");
8359        };
8360        assert_eq!(terminals.entries.len(), 2);
8361        assert_eq!(terminals.journal_skipped_lines, 0);
8362        assert_eq!(terminals.journal_read_errors, 0);
8363    }
8364
8365    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8366    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8367        let registry = Arc::new(Registry::default());
8368        let supervisor_handle = SupervisorHandle::new();
8369        let supervisor =
8370            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
8371                .with_handle(supervisor_handle.clone());
8372        let module = supervisor
8373            .spawn(ModuleSpec {
8374                module_id: "terminal-golden".to_string(),
8375                program: fake_aft_stub_path(),
8376                args: Vec::new(),
8377                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8378                reserved: false,
8379                reserved_prefixes: Vec::new(),
8380                protocol: ModuleProtocol::Subc,
8381                overlap: Default::default(),
8382            })
8383            .unwrap();
8384
8385        let deadline = Instant::now() + Duration::from_secs(5);
8386        while module.terminal_history().entries.len() != 2 {
8387            assert!(
8388                Instant::now() < deadline,
8389                "module did not retain two terminal exits: {:?}",
8390                module.terminal_history()
8391            );
8392            sleep(Duration::from_millis(10)).await;
8393        }
8394
8395        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8396        let request = ClientControlRequest::SupervisorTerminals {
8397            module_id: "terminal-golden".to_string(),
8398        };
8399        let frame = Frame::build(
8400            FrameType::Request,
8401            control_flags(),
8402            0,
8403            0,
8404            1,
8405            serde_json::to_vec(&request).unwrap(),
8406        )
8407        .unwrap();
8408        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8409        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8410        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8411        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8412            panic!("expected supervisor.terminals response");
8413        };
8414        assert_eq!(terminals.entries.len(), 2);
8415        assert_eq!(terminals.dropped, 0);
8416
8417        let mut rendered = serde_json::to_value(response).unwrap();
8418        // Wall-clock fields are the observation contract, but not stable fixture
8419        // bytes; normalize only them after the real handler has shaped the response.
8420        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8421        for (index, entry) in rendered["entries"]
8422            .as_array_mut()
8423            .expect("terminal response entries array")
8424            .iter_mut()
8425            .enumerate()
8426        {
8427            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8428        }
8429
8430        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8431            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8432        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8433        if std::env::var_os("UPDATE_GOLDEN").is_some() {
8434            std::fs::write(&golden_path, &serialized).unwrap();
8435        }
8436        let expected: Value =
8437            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8438        assert_eq!(rendered, expected);
8439    }
8440
8441    #[test]
8442    fn hello_registers_manifest_and_returns_ack() {
8443        let registry = Arc::new(Registry::default());
8444        let handler = ControlHandler::new(Arc::clone(&registry));
8445        let conn = ConnectionId::new(1);
8446
8447        let responses = handler
8448            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8449            .unwrap();
8450
8451        assert_eq!(responses.len(), 1);
8452        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8453        assert_eq!(responses[0].header.channel, 0);
8454        assert_eq!(responses[0].header.corr, 7);
8455        let ack = parse_ack(&responses[0]);
8456        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8457        assert!(ack
8458            .subc_capabilities
8459            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8460        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8461        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8462        assert!(ack
8463            .subc_ops
8464            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8465        assert!(ack
8466            .subc_ops
8467            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8468
8469        let registration = registry.get_module("aft").unwrap().unwrap();
8470        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8471        assert_eq!(registration.state, ChannelState::Active);
8472        assert_eq!(registration.connection_id, conn);
8473        assert_eq!(registration.control_ops, module_baseline_control_ops());
8474    }
8475
8476    #[test]
8477    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8478        let invalid_identifiers = [
8479            ("case_change", "credentials-Provider/v1"),
8480            ("leading_zero", "credentials-provider/v01"),
8481            ("trailing_hyphen", "credentials-provider-/v1"),
8482            ("consecutive_hyphens", "credentials--provider/v1"),
8483            ("uppercase", "Credentials-provider/v1"),
8484            ("missing_v", "credentials-provider/1"),
8485            ("whitespace", "credentials provider/v1"),
8486            ("zero_version", "credentials-provider/v0"),
8487            ("out_of_range_version", "credentials-provider/v4294967296"),
8488            (
8489                "overlength_name",
8490                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8491            ),
8492        ];
8493        let mut cases = invalid_identifiers
8494            .into_iter()
8495            .map(|(name, identifier)| {
8496                (
8497                    format!("identifier_{name}"),
8498                    "capabilities.provides[0]".to_string(),
8499                    identifier.to_string(),
8500                    json!({ "provides": [identifier] }),
8501                    None,
8502                )
8503            })
8504            .collect::<Vec<_>>();
8505        cases.extend([
8506            (
8507                "unknown_need".to_string(),
8508                "capabilities.requires[0].need".to_string(),
8509                "deferred".to_string(),
8510                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8511                None,
8512            ),
8513            (
8514                "duplicate_provides".to_string(),
8515                "capabilities.provides[1]".to_string(),
8516                "credentials-provider/v1".to_string(),
8517                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8518                None,
8519            ),
8520            (
8521                "duplicate_must_never_reach".to_string(),
8522                "capabilities.must_never_reach[1]".to_string(),
8523                "credentials-provider/v1".to_string(),
8524                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8525                None,
8526            ),
8527            (
8528                "duplicate_requires_same_need".to_string(),
8529                "capabilities.requires[1]".to_string(),
8530                "credentials-provider/v1".to_string(),
8531                json!({ "requires": [
8532                    { "capability": "credentials-provider/v1", "need": "required" },
8533                    { "capability": "credentials-provider/v1", "need": "required" }
8534                ] }),
8535                None,
8536            ),
8537            (
8538                "duplicate_requires_conflicting_need".to_string(),
8539                "capabilities.requires[1]".to_string(),
8540                "credentials-provider/v1".to_string(),
8541                json!({ "requires": [
8542                    { "capability": "credentials-provider/v1", "need": "required" },
8543                    { "capability": "credentials-provider/v1", "need": "optional" }
8544                ] }),
8545                None,
8546            ),
8547            (
8548                "capabilities_root_pointer".to_string(),
8549                "runtime_computed[0]".to_string(),
8550                "/capabilities".to_string(),
8551                json!({}),
8552                Some(json!(["/capabilities"])),
8553            ),
8554            (
8555                "capabilities_descendant_pointer".to_string(),
8556                "runtime_computed[0]".to_string(),
8557                "/capabilities/provides".to_string(),
8558                json!({}),
8559                Some(json!(["/capabilities/provides"])),
8560            ),
8561            (
8562                "malformed_pointer_without_leading_slash".to_string(),
8563                "runtime_computed[0]".to_string(),
8564                "capabilities".to_string(),
8565                json!({}),
8566                Some(json!(["capabilities"])),
8567            ),
8568            (
8569                "malformed_pointer_escape".to_string(),
8570                "runtime_computed[0]".to_string(),
8571                "/roles/~2/tools".to_string(),
8572                json!({}),
8573                Some(json!(["/roles/~2/tools"])),
8574            ),
8575            (
8576                "unknown_capabilities_field".to_string(),
8577                "capabilities.future".to_string(),
8578                "<array>".to_string(),
8579                json!({ "future": [] }),
8580                None,
8581            ),
8582        ]);
8583
8584        for (index, (name, field, value, capabilities, runtime_computed)) in
8585            cases.into_iter().enumerate()
8586        {
8587            let registry = Arc::new(Registry::default());
8588            let handler = ControlHandler::new(Arc::clone(&registry));
8589            let response = handler
8590                .handle_control(
8591                    ConnectionId::new((index + 1) as u64),
8592                    capability_grammar_hello_frame(
8593                        capabilities,
8594                        runtime_computed,
8595                        index as u64 + 1,
8596                    ),
8597                )
8598                .expect("invalid HELLO returns a refusal");
8599
8600            assert_eq!(response.len(), 1, "{name} must emit one refusal");
8601            let error = parse_error(&response[0]);
8602            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8603            let message = error["message"]
8604                .as_str()
8605                .expect("error message is a string");
8606            assert!(
8607                message.contains(&field),
8608                "{name}: field missing from {message}"
8609            );
8610            assert!(
8611                message.contains(&value),
8612                "{name}: value missing from {message}"
8613            );
8614            assert_eq!(
8615                registry
8616                    .active_registration_count()
8617                    .expect("registry reads"),
8618                0,
8619                "{name}: refused HELLO must not create a catalog entry"
8620            );
8621        }
8622    }
8623
8624    #[test]
8625    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8626        let registry = Arc::new(Registry::default());
8627        let handler = ControlHandler::new(Arc::clone(&registry));
8628        let capabilities = json!({
8629            "provides": ["credentials-provider/v1"],
8630            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8631            "must_never_reach": ["federation-transport/v1"]
8632        });
8633        let response = handler
8634            .handle_control(
8635                ConnectionId::new(99),
8636                capability_grammar_hello_frame(
8637                    capabilities.clone(),
8638                    Some(json!(["/roles/0/tools"])),
8639                    99,
8640                ),
8641            )
8642            .expect("valid HELLO registers");
8643        assert_eq!(response[0].header.ty, FrameType::HelloAck);
8644
8645        let request = Frame::build(
8646            FrameType::Request,
8647            control_flags(),
8648            0,
8649            0,
8650            100,
8651            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8652                .expect("catalog request serializes"),
8653        )
8654        .expect("catalog request frame builds");
8655        let response = handler
8656            .handle_catalog_list(request, None)
8657            .expect("catalog list succeeds");
8658        let ClientControlResponse::CatalogList { modules, .. } =
8659            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8660        else {
8661            panic!("catalog request must return catalog.list");
8662        };
8663        assert_eq!(modules.len(), 1);
8664        assert_eq!(
8665            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8666            capabilities
8667        );
8668    }
8669
8670    #[test]
8671    fn catalog_list_mirrors_management_operation_description() {
8672        let registry = Arc::new(Registry::default());
8673        let handler = ControlHandler::new(Arc::clone(&registry));
8674        let description = "List managed records and return their identifiers and metadata.";
8675        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8676        manifest.provides = vec![ProviderRole::ManagementSurface {
8677            operations: vec![ManagementOperation {
8678                name: "records.list".to_string(),
8679                kind: ManagementOperationKind::Query,
8680                description: Some(description.to_string()),
8681            }],
8682            config_schema: json!({"type": "object"}),
8683            observability: vec![ObservabilitySurface {
8684                name: "records.stats".to_string(),
8685                kind: ObservabilityKind::Snapshot,
8686            }],
8687            identity_scope: vec![IdentityScope::Project],
8688            concurrency: Concurrency::ModuleManaged,
8689        }];
8690        registry
8691            .register_with_control_ops(
8692                manifest,
8693                PROTOCOL_VERSION,
8694                ConnectionId::new(99),
8695                Vec::new(),
8696            )
8697            .expect("described management manifest registers");
8698
8699        let request = Frame::build(
8700            FrameType::Request,
8701            control_flags(),
8702            0,
8703            0,
8704            100,
8705            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8706                .expect("catalog request serializes"),
8707        )
8708        .expect("catalog request frame builds");
8709        let response = handler
8710            .handle_catalog_list(request, None)
8711            .expect("catalog list succeeds");
8712        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8713        assert_eq!(
8714            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8715            "catalog.list must preserve the declared operation description verbatim"
8716        );
8717    }
8718
8719    #[test]
8720    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8721        let registry = Arc::new(Registry::default());
8722        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
8723            [("vault".to_string(), true), ("squatter".to_string(), true)],
8724            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8725        );
8726        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8727        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8728            provides: vec!["credentials-provider/v1".to_string()],
8729            requires: Vec::new(),
8730            must_never_reach: Vec::new(),
8731        });
8732        let frame = Frame::build(
8733            FrameType::Hello,
8734            control_flags(),
8735            0,
8736            0,
8737            77,
8738            serde_json::to_vec(&ModuleHelloBody {
8739                manifest: squatter,
8740                protocol_ver: PROTOCOL_VERSION,
8741                control_ops: None,
8742                launch_nonce: None,
8743            })
8744            .expect("HELLO serializes"),
8745        )
8746        .expect("HELLO frame builds");
8747        let response = handler
8748            .handle_control(ConnectionId::new(77), frame)
8749            .expect("reserved claim receives a typed refusal");
8750        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8751        assert_eq!(
8752            registry
8753                .active_registration_count()
8754                .expect("registry reads"),
8755            0,
8756            "a reserved capability refusal must not leave a catalog entry"
8757        );
8758    }
8759
8760    #[test]
8761    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8762        for settlement in ["timeout", "inconclusive", "drop"] {
8763            let breakers = RouteBindBreakers::default();
8764            let RouteBindAdmission::Admitted {
8765                guard: mut old,
8766                probe: false,
8767            } = breakers.admit("prov")
8768            else {
8769                panic!("ordinary relay admitted")
8770            };
8771            let RouteBindAdmission::Admitted {
8772                guard: mut opener, ..
8773            } = breakers.admit("prov")
8774            else {
8775                panic!("second relay admitted")
8776            };
8777            assert!(
8778                !opener
8779                    .record_timeout(1, Duration::ZERO)
8780                    .unwrap()
8781                    .reopened_after_probe
8782            );
8783            let RouteBindAdmission::Admitted {
8784                guard: mut probe,
8785                probe: true,
8786            } = breakers.admit("prov")
8787            else {
8788                panic!("one cooldown probe admitted")
8789            };
8790            match settlement {
8791                "timeout" => assert!(
8792                    !old.record_timeout(1, Duration::ZERO)
8793                        .unwrap()
8794                        .reopened_after_probe
8795                ),
8796                "inconclusive" => old.record_inconclusive(),
8797                "drop" => drop(old),
8798                _ => unreachable!(),
8799            }
8800            assert!(
8801                matches!(
8802                    breakers.admit("prov"),
8803                    RouteBindAdmission::Refused {
8804                        probe_in_flight: true,
8805                        ..
8806                    }
8807                ),
8808                "{settlement} of a pre-open relay cannot release the real probe"
8809            );
8810            assert!(
8811                probe
8812                    .record_timeout(1, Duration::ZERO)
8813                    .unwrap()
8814                    .reopened_after_probe
8815            );
8816            assert!(matches!(
8817                breakers.admit("prov"),
8818                RouteBindAdmission::Admitted { probe: true, .. }
8819            ));
8820        }
8821        let breakers = RouteBindBreakers::default();
8822        let admit = || match breakers.admit("prov") {
8823            RouteBindAdmission::Admitted { guard, .. } => guard,
8824            _ => panic!("relay admitted"),
8825        };
8826        admit().record_timeout(1, Duration::ZERO);
8827        let mut old_probe = admit();
8828        breakers.reset_for_new_module_connection("prov");
8829        admit().record_timeout(1, Duration::ZERO);
8830        let _new_probe = admit();
8831        old_probe.record_inconclusive();
8832        assert!(matches!(
8833            breakers.admit("prov"),
8834            RouteBindAdmission::Refused {
8835                probe_in_flight: true,
8836                ..
8837            }
8838        ));
8839    }
8840
8841    #[tokio::test]
8842    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8843        for candidate in [false, true] {
8844            let registry = Arc::new(Registry::default());
8845            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
8846                [("vault".to_string(), true), ("squatter".to_string(), true)],
8847                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8848            );
8849            let conn = ConnectionId::new(77);
8850            let (ctx, mut rx) = route_ctx(conn);
8851            let initial = capability_manifest("squatter", &[], &[]);
8852            if candidate {
8853                registry
8854                    .register_candidate_with_control_ops(
8855                        initial.clone(),
8856                        PROTOCOL_VERSION,
8857                        conn,
8858                        module_baseline_control_ops(),
8859                    )
8860                    .unwrap();
8861                handler
8862                    .forwarding
8863                    .register_candidate_module_connection(
8864                        conn,
8865                        "squatter".to_string(),
8866                        PROTOCOL_VERSION,
8867                        manifest_concurrency(&initial),
8868                        ctx.egress.clone(),
8869                    )
8870                    .unwrap();
8871            } else {
8872                hello_via_sink(
8873                    &handler,
8874                    &ctx,
8875                    &mut rx,
8876                    hello_frame_with_manifest(initial.clone(), 1),
8877                )
8878                .await;
8879            }
8880            let response = handler
8881                .handle_control_frame(
8882                    &ctx,
8883                    catalog_update_with_capabilities_frame(
8884                        2,
8885                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
8886                            .capabilities
8887                            .unwrap(),
8888                    ),
8889                )
8890                .await
8891                .unwrap();
8892            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8893            assert_eq!(
8894                registry
8895                    .get_module_by_connection(conn)
8896                    .unwrap()
8897                    .unwrap()
8898                    .manifest,
8899                initial
8900            );
8901        }
8902    }
8903
8904    #[test]
8905    fn server_describe_surfaces_required_capability_verdict_fields() {
8906        let registry = Arc::new(Registry::default());
8907        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
8908            [
8909                ("consumer".to_string(), true),
8910                ("provider".to_string(), false),
8911            ],
8912            BTreeMap::new(),
8913        );
8914        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8915        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8916            provides: Vec::new(),
8917            requires: vec![subc_protocol::manifest::CapabilityRequirement {
8918                capability: "credentials-provider/v1".to_string(),
8919                need: subc_protocol::manifest::CapabilityNeed::Required,
8920            }],
8921            must_never_reach: Vec::new(),
8922        });
8923        let hello = Frame::build(
8924            FrameType::Hello,
8925            control_flags(),
8926            0,
8927            0,
8928            78,
8929            serde_json::to_vec(&ModuleHelloBody {
8930                manifest: consumer,
8931                protocol_ver: PROTOCOL_VERSION,
8932                control_ops: None,
8933                launch_nonce: None,
8934            })
8935            .expect("HELLO serializes"),
8936        )
8937        .expect("HELLO frame builds");
8938        handler
8939            .handle_control(ConnectionId::new(78), hello)
8940            .expect("consumer registers");
8941        let describe = Frame::build(
8942            FrameType::Request,
8943            control_flags(),
8944            0,
8945            0,
8946            79,
8947            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8948                .expect("request serializes"),
8949        )
8950        .expect("describe frame builds");
8951        let response = handler
8952            .handle_server_describe(describe)
8953            .expect("server.describe succeeds");
8954        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8955        let requirement = &rendered["capability_requirements"][0];
8956        assert_eq!(requirement["consumer"], "consumer");
8957        assert_eq!(requirement["verdict"], "never_provided");
8958        assert_eq!(requirement["episode_seq"], 1);
8959        assert_eq!(requirement["config_satisfiable"], false);
8960        assert_eq!(requirement["runtime_available"], false);
8961        assert!(requirement["detail"]
8962            .as_str()
8963            .expect("detail string")
8964            .contains("credentials-provider/v1"));
8965    }
8966
8967    #[test]
8968    fn catalog_list_omits_capabilities_for_legacy_manifest() {
8969        let registry = Arc::new(Registry::default());
8970        let handler = ControlHandler::new(Arc::clone(&registry));
8971        let hello = handler
8972            .handle_control(
8973                ConnectionId::new(101),
8974                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8975            )
8976            .expect("legacy HELLO registers");
8977        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8978
8979        let request = Frame::build(
8980            FrameType::Request,
8981            control_flags(),
8982            0,
8983            0,
8984            102,
8985            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8986                .expect("catalog request serializes"),
8987        )
8988        .expect("catalog request frame builds");
8989        let response = handler
8990            .handle_catalog_list(request, None)
8991            .expect("catalog list succeeds");
8992        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8993        assert!(
8994            body["modules"][0].get("capabilities").is_none(),
8995            "legacy manifest must retain an absent capabilities field on catalog.list"
8996        );
8997    }
8998
8999    #[test]
9000    fn hello_ack_omits_storage_when_no_storage_config() {
9001        let registry = Arc::new(Registry::default());
9002        let handler = ControlHandler::new(Arc::clone(&registry));
9003        let responses = handler
9004            .handle_control(
9005                ConnectionId::new(1),
9006                hello_frame("aft", PROTOCOL_VERSION, 7),
9007            )
9008            .unwrap();
9009        let ack = parse_ack(&responses[0]);
9010        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9011        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9012    }
9013
9014    #[tokio::test]
9015    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9016        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9017        let registry = Arc::new(Registry::default());
9018        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
9019        let responses = handler
9020            .handle_control(
9021                ConnectionId::new(1),
9022                hello_frame("aft", PROTOCOL_VERSION, 7),
9023            )
9024            .unwrap();
9025        let ack = parse_ack(&responses[0]);
9026        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9027
9028        let described = handler
9029            .handle_control_frame(
9030                &route_ctx(ConnectionId::new(2)).0,
9031                Frame::build(
9032                    FrameType::Request,
9033                    control_flags(),
9034                    0,
9035                    0,
9036                    9,
9037                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9038                )
9039                .unwrap(),
9040            )
9041            .await
9042            .unwrap();
9043        let ClientControlResponse::ServerDescribe { machine_id, .. } =
9044            serde_json::from_slice(&described[0].body).unwrap()
9045        else {
9046            panic!("server.describe answered with another shape");
9047        };
9048        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9049    }
9050
9051    #[test]
9052    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9053        // With a central sqlite storage policy, each registering module gets its
9054        // own resolved descriptor in HELLO_ACK, keyed by its module id.
9055        let registry = Arc::new(Registry::default());
9056        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
9057            crate::daemon_config::StorageConfig::Sqlite {
9058                data_home: std::path::PathBuf::from("/data"),
9059            },
9060        ));
9061
9062        let responses = handler
9063            .handle_control(
9064                ConnectionId::new(1),
9065                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9066            )
9067            .unwrap();
9068        let ack = parse_ack(&responses[0]);
9069        assert_eq!(
9070            ack.storage,
9071            Some(serde_json::json!({
9072                "module_id": "alfonso-routing",
9073                "storage_namespace": "default",
9074                "isolation": { "kind": "module" },
9075                "backend": {
9076                    "backend": "sqlite",
9077                    "path": "/data/cortexkit/alfonso-routing/store.db"
9078                }
9079            })),
9080            "the delivered descriptor is the module's own sqlite store path"
9081        );
9082    }
9083
9084    #[test]
9085    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9086        let registry = Arc::new(Registry::default());
9087        let handler = ControlHandler::new(Arc::clone(&registry));
9088        let conn = ConnectionId::new(1);
9089        let responses = handler
9090            .handle_control(
9091                conn,
9092                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9093            )
9094            .unwrap();
9095        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9096        let registration = registry.get_module("aft").unwrap().unwrap();
9097        assert_eq!(registration.control_ops, module_baseline_control_ops());
9098
9099        let frame =
9100            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9101        assert!(handler
9102            .guard_module_control_op(&frame, "aft", "route.bind")
9103            .unwrap()
9104            .is_none());
9105        let error = handler
9106            .guard_module_control_op(&frame, "aft", "test.synthetic")
9107            .unwrap()
9108            .expect("synthetic ungranted op should be rejected");
9109        assert_eq!(error.header.ty, FrameType::Error);
9110        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9111    }
9112
9113    #[test]
9114    fn hello_control_ops_some_adds_optional_grants() {
9115        let registry = Arc::new(Registry::default());
9116        let handler = ControlHandler::new(Arc::clone(&registry));
9117        handler
9118            .handle_control(
9119                ConnectionId::new(1),
9120                hello_frame_with_control_ops(
9121                    "aft",
9122                    PROTOCOL_VERSION,
9123                    7,
9124                    Some(vec![
9125                        "future.synthetic".to_string(),
9126                        "route.bind".to_string(),
9127                    ]),
9128                ),
9129            )
9130            .unwrap();
9131        let registration = registry.get_module("aft").unwrap().unwrap();
9132        assert_eq!(
9133            registration.control_ops,
9134            vec![
9135                "route.bind".to_string(),
9136                "route.status".to_string(),
9137                "future.synthetic".to_string(),
9138            ]
9139        );
9140        let frame =
9141            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9142        assert!(handler
9143            .guard_module_control_op(&frame, "aft", "future.synthetic")
9144            .unwrap()
9145            .is_none());
9146    }
9147
9148    #[tokio::test]
9149    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9150        let registry = Arc::new(Registry::default());
9151        let forwarding = Arc::new(ForwardingTable::default());
9152        let handler =
9153            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9154        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9155        hello_via_sink(
9156            &handler,
9157            &module_ctx,
9158            &mut module_rx,
9159            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9160        )
9161        .await;
9162
9163        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9164        let responses = handler
9165            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9166            .await
9167            .unwrap();
9168        assert_eq!(responses.len(), 1);
9169        assert_eq!(responses[0].header.ty, FrameType::Error);
9170        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9171        assert!(module_rx.try_recv().is_err());
9172    }
9173
9174    #[tokio::test]
9175    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9176        let registry = Arc::new(Registry::default());
9177        let forwarding = Arc::new(ForwardingTable::default());
9178        let handler =
9179            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9180        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9181        hello_via_sink(
9182            &handler,
9183            &module_ctx,
9184            &mut module_rx,
9185            hello_frame_with_control_ops(
9186                "aft",
9187                PROTOCOL_VERSION,
9188                7,
9189                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9190            ),
9191        )
9192        .await;
9193
9194        let project_root = unique_project_root("demux");
9195        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9196        let route_handler = handler.clone();
9197        let route_task = tokio::spawn(async move {
9198            route_handler
9199                .handle_control_frame(
9200                    &route_client_ctx,
9201                    route_open_frame(100, "aft", project_root),
9202                )
9203                .await
9204                .unwrap()
9205        });
9206        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9207            .await
9208            .unwrap()
9209            .unwrap();
9210        assert!(matches!(
9211            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9212            ModuleControlRequest::RouteBind { .. }
9213        ));
9214
9215        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9216        let health_handler = handler.clone();
9217        let health_task = tokio::spawn(async move {
9218            health_handler
9219                .handle_control_frame(
9220                    &health_client_ctx,
9221                    supervisor_health_probe_frame(101, "aft"),
9222                )
9223                .await
9224                .unwrap()
9225        });
9226        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9227            .await
9228            .unwrap()
9229            .unwrap();
9230        assert_eq!(
9231            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9232            ModuleControlRequest::HealthCheck {}
9233        );
9234
9235        handler
9236            .handle_control_frame(
9237                &module_ctx,
9238                health_response(health_frame.header.corr, HealthStatus::Degraded),
9239            )
9240            .await
9241            .unwrap();
9242        let health_response = health_task.await.unwrap();
9243        assert_eq!(health_response.len(), 1);
9244        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9245            ClientControlResponse::SupervisorHealthProbe {
9246                module_id,
9247                status,
9248                detail,
9249                metrics,
9250            } => {
9251                assert_eq!(module_id, "aft");
9252                assert_eq!(status, HealthStatus::Degraded);
9253                assert_eq!(detail.as_deref(), Some("warming"));
9254                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9255            }
9256            other => panic!("unexpected health response: {other:?}"),
9257        }
9258
9259        handler
9260            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9261            .await
9262            .unwrap();
9263        let route_response = route_task.await.unwrap();
9264        assert!(route_response.is_empty());
9265        let published = route_client_rx.recv().await.unwrap();
9266        assert!(matches!(
9267            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9268            ClientControlResponse::RouteOpen { .. }
9269        ));
9270    }
9271
9272    /// Start one `route.open` on `client_connection` and return its still-running
9273    /// handler task together with the `route.bind` the module received for it.
9274    /// The handler blocks until the module answers, so it has to run as a task
9275    /// while the test drives the module side.
9276    async fn relay_route_open(
9277        handler: &ControlHandler,
9278        client_connection: ConnectionId,
9279        client_egress: &FrameSink,
9280        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9281        corr: u64,
9282        module_id: &str,
9283        project_root_label: &str,
9284    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9285        let ctx = RouteCtx {
9286            connection_id: client_connection,
9287            egress: client_egress.clone(),
9288        };
9289        let handler = handler.clone();
9290        let project_root = unique_project_root(project_root_label);
9291        let module_id = module_id.to_string();
9292        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9293        let task = tokio::spawn(async move {
9294            let _guard = tracing::dispatcher::set_default(&dispatch);
9295            handler
9296                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9297                .await
9298                .unwrap()
9299        });
9300        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9301            .await
9302            .expect("module receives the relayed route.bind")
9303            .expect("module egress is open");
9304        (task, bind.frame)
9305    }
9306
9307    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9308        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9309            ModuleControlRequest::RouteBind {
9310                route_channel,
9311                epoch,
9312                ..
9313            } => (route_channel, epoch),
9314            other => panic!("expected a route.bind request, got {other:?}"),
9315        }
9316    }
9317
9318    fn published_route(frame: &Frame) -> (u16, u32) {
9319        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9320            ClientControlResponse::RouteOpen {
9321                route_channel,
9322                route_epoch,
9323            } => (route_channel, route_epoch),
9324            other => panic!("expected a route.open response, got {other:?}"),
9325        }
9326    }
9327
9328    /// Reproduction of a production outage. A client had `route.open`s in
9329    /// flight to a module and was already marked closing -- its egress had refused a
9330    /// module frame, so the daemon asked its connection to end -- while its sink
9331    /// was still open. When the module acked those binds, the daemon refused to
9332    /// commit a route for a closing client, and that refusal was returned from
9333    /// the MODULE connection's frame handler, where a router error that has no
9334    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
9335    /// the supervisor correctly did not respawn a clean exit, and every seat lost
9336    /// its tools for hours -- one client's teardown took down a connection
9337    /// carrying ~170 other routes.
9338    ///
9339    /// The window is opened here by calling the production path that opens it
9340    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
9341    /// state that matters is "in `closing_connections`, sink still open, relay
9342    /// still pending", and it lasts only from the close request until the
9343    /// connection loop reacts to it; a socket-level test can flood a client into
9344    /// that escalation but cannot pin the module's ack inside the window. Closing
9345    /// the socket instead takes the other path entirely -- connection teardown
9346    /// removes the pending relay under the same lock, so the ack finds nothing.
9347    #[tokio::test]
9348    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9349        let registry = Arc::new(Registry::default());
9350        let forwarding = Arc::new(ForwardingTable::default());
9351        let handler =
9352            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9353
9354        let module_connection = ConnectionId::new(30);
9355        let (module_ctx, mut module_rx) = route_ctx(module_connection);
9356        hello_via_sink(
9357            &handler,
9358            &module_ctx,
9359            &mut module_rx,
9360            hello_frame("aft", PROTOCOL_VERSION, 7),
9361        )
9362        .await;
9363
9364        let dying_client = ConnectionId::new(31);
9365        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9366
9367        // A published route on the dying client. The escalation below only marks
9368        // a connection closing for a route it has already published.
9369        let (first_task, first_bind) = relay_route_open(
9370            &handler,
9371            dying_client,
9372            &dying_ctx.egress,
9373            &mut module_rx,
9374            100,
9375            "aft",
9376            "closing-first",
9377        )
9378        .await;
9379        handler
9380            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9381            .await
9382            .unwrap();
9383        assert!(first_task.await.unwrap().is_empty());
9384        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9385
9386        // A second route.open from the same client, relayed and awaiting its ack.
9387        let (second_task, second_bind) = relay_route_open(
9388            &handler,
9389            dying_client,
9390            &dying_ctx.egress,
9391            &mut module_rx,
9392            101,
9393            "aft",
9394            "closing-second",
9395        )
9396        .await;
9397        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9398
9399        // The window: the client is closing, its sink is still open, and its
9400        // second bind is still pending.
9401        assert!(forwarding
9402            .escalate_client_delivery_failure(
9403                dying_client,
9404                first_channel,
9405                first_epoch,
9406                CloseReason::new(
9407                    "module_to_client_delivery_failed",
9408                    "client egress refused a module frame",
9409                ),
9410                crate::forwarding::UndeliveredFrame {
9411                    module_id: None,
9412                    sink: &dying_ctx.egress,
9413                },
9414            )
9415            .unwrap());
9416        assert!(!dying_ctx.egress.is_closed());
9417
9418        // The frame that used to end the module connection.
9419        let ack = handler
9420            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9421            .await;
9422        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9423        if module_loop_error.is_some() {
9424            // What the server's connection loop does with a router error that has
9425            // no ERROR-frame translation: end the connection, which releases the
9426            // module's registration and every route on it.
9427            handler.cleanup_connection(module_connection).unwrap();
9428        }
9429        // Read the module's next frame before opening the co-tenant's route, so
9430        // the GOODBYE assertion below is about THIS ack and not about later
9431        // traffic. `None` means the module was told nothing.
9432        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9433            .await
9434            .ok()
9435            .flatten();
9436
9437        // 1. The module connection is still registered.
9438        assert!(
9439            registry
9440                .get_module_by_connection(module_connection)
9441                .unwrap()
9442                .is_some(),
9443            "one client's closing connection ended the shared module connection: \
9444             {module_loop_error:?}"
9445        );
9446        // ...and still serving: another client can open and use a route on it.
9447        let cotenant = ConnectionId::new(32);
9448        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9449        let (cotenant_task, cotenant_bind) = relay_route_open(
9450            &handler,
9451            cotenant,
9452            &cotenant_ctx.egress,
9453            &mut module_rx,
9454            102,
9455            "aft",
9456            "closing-cotenant",
9457        )
9458        .await;
9459        handler
9460            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9461            .await
9462            .unwrap();
9463        assert!(cotenant_task.await.unwrap().is_empty());
9464        let (cotenant_channel, cotenant_epoch) =
9465            published_route(&cotenant_rx.recv().await.unwrap());
9466        assert!(matches!(
9467            forwarding
9468                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9469                .unwrap(),
9470            DataRoute::Client(DataRouteState::Bound(_))
9471        ));
9472
9473        // 2. The module was told to drop the binding it created for the route
9474        //    that will never be published.
9475        let goodbye = post_ack_module_frame
9476            .expect("module receives a GOODBYE for the abandoned route channel");
9477        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9478        assert_eq!(goodbye.header.channel, abandoned_channel);
9479        assert_eq!(goodbye.header.epoch, abandoned_epoch);
9480
9481        // 3. The dying client received nothing: no route was ever published to
9482        //    it. Its route.open is answered as unavailable, which the connection
9483        //    loop would write to a socket that is already going away.
9484        assert!(dying_rx.try_recv().is_err());
9485        let second_response = second_task.await.unwrap();
9486        assert_eq!(second_response.len(), 1);
9487        assert_eq!(
9488            parse_error(&second_response[0])["code"],
9489            "target_unavailable"
9490        );
9491    }
9492
9493    /// One `catalog.list` as a client sees it: the generation and the listed
9494    /// module ids with their readiness.
9495    async fn catalog_snapshot(
9496        handler: &ControlHandler,
9497        client_ctx: &RouteCtx,
9498        corr: u64,
9499    ) -> (u64, Vec<(String, bool)>) {
9500        let body =
9501            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9502        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9503        let reply = handler
9504            .handle_control_frame(client_ctx, request)
9505            .await
9506            .unwrap()
9507            .pop()
9508            .unwrap();
9509        let ClientControlResponse::CatalogList {
9510            generation,
9511            modules,
9512            ..
9513        } = serde_json::from_slice(&reply.body).unwrap()
9514        else {
9515            panic!("not a catalog.list reply");
9516        };
9517        (
9518            generation,
9519            modules
9520                .into_iter()
9521                .map(|entry| (entry.module_id, entry.ready))
9522                .collect(),
9523        )
9524    }
9525
9526    /// A handler whose HELLO for module "aft" on connection 40 is parked
9527    /// between reserving its registry entry and installing its forwarding
9528    /// endpoint. Returns the handler, the two-party barrier (wait once to know
9529    /// the HELLO is parked, once more to release it), the HELLO's thread, and
9530    /// the module's context and receiver.
9531    fn hello_held_before_forwarding() -> (
9532        ControlHandler,
9533        Arc<std::sync::Barrier>,
9534        std::thread::JoinHandle<Vec<Frame>>,
9535        RouteCtx,
9536        mpsc::Receiver<crate::router::OutboundFrame>,
9537    ) {
9538        let hold = Arc::new(std::sync::Barrier::new(2));
9539        let handler = ControlHandler::with_forwarding(
9540            Arc::new(Registry::default()),
9541            Arc::new(ForwardingTable::default()),
9542        )
9543        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9544        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9545        let hello = {
9546            let handler = handler.clone();
9547            let module_ctx = module_ctx.clone();
9548            std::thread::spawn(move || {
9549                handler
9550                    .handle_hello(
9551                        module_ctx.connection_id,
9552                        Some(module_ctx.egress.clone()),
9553                        hello_frame("aft", PROTOCOL_VERSION, 7),
9554                    )
9555                    .unwrap()
9556            })
9557        };
9558        (handler, hold, hello, module_ctx, module_rx)
9559    }
9560
9561    /// While a HELLO has reserved its registry entry but its forwarding
9562    /// endpoint is not installed yet, `route.open` to the module can only be
9563    /// refused. `catalog.list` must not list the module in that window: a
9564    /// client that opens a route as soon as it sees a module listed and ready
9565    /// would be refused for a module the daemon had just advertised. The
9566    /// refusal in the window stays the retryable `target_unavailable`, never
9567    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
9568    /// Once the HELLO completes, the module is listed and a route to it opens.
9569    #[tokio::test]
9570    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
9571        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
9572        let client = ConnectionId::new(41);
9573        let (client_ctx, mut client_rx) = route_ctx(client);
9574
9575        hold.wait();
9576        // Parked: the registry holds "aft", forwarding has no endpoint for it.
9577        assert!(handler.registry.get_module("aft").unwrap().is_some());
9578        assert!(!handler
9579            .forwarding
9580            .has_live_module_connection("aft")
9581            .unwrap());
9582        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
9583        assert!(
9584            listed.is_empty(),
9585            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
9586        );
9587        let refused = handler
9588            .handle_control_frame(
9589                &client_ctx,
9590                route_open_frame(301, "aft", unique_project_root("listed-window")),
9591            )
9592            .await
9593            .unwrap();
9594        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
9595        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
9596        let duplicate = handler
9597            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
9598            .await
9599            .unwrap();
9600        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9601
9602        hold.wait();
9603        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9604        let ack = module_rx.try_recv().unwrap().frame;
9605        assert_eq!(ack.header.ty, FrameType::HelloAck);
9606        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9607        assert_eq!(listed, vec![("aft".to_string(), true)]);
9608
9609        let (open_task, bind) = relay_route_open(
9610            &handler,
9611            client,
9612            &client_ctx.egress,
9613            &mut module_rx,
9614            303,
9615            "aft",
9616            "listed-after",
9617        )
9618        .await;
9619        assert!(handler
9620            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9621            .await
9622            .unwrap()
9623            .is_empty());
9624        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
9625        published_route(&client_rx.try_recv().unwrap().frame);
9626    }
9627
9628    /// A catalog watcher keys on the generation: when it changes, it re-reads
9629    /// the listing. So the generation must advance exactly when the module
9630    /// appears. If it advanced while the module was still hidden, a watcher
9631    /// would read the new generation without the module, and the module would
9632    /// later appear under that same generation with nothing telling the
9633    /// watcher to look again.
9634    #[tokio::test]
9635    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
9636        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
9637        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9638        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9639
9640        hold.wait();
9641        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
9642        assert!(listed.is_empty());
9643        assert_eq!(
9644            during, before,
9645            "the generation moved while the module was not listed"
9646        );
9647
9648        hold.wait();
9649        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9650        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9651        assert_eq!(listed, vec![("aft".to_string(), true)]);
9652        assert_eq!(
9653            after,
9654            before + 1,
9655            "the module appeared without exactly one generation bump"
9656        );
9657    }
9658
9659    /// A HELLO whose forwarding endpoint cannot be installed (here the
9660    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
9661    /// refused and leaves nothing behind: no registry entry, no catalog entry,
9662    /// and the catalog generation where it was.
9663    #[tokio::test]
9664    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
9665        let registry = Arc::new(Registry::default());
9666        let handler = ControlHandler::with_forwarding(
9667            Arc::clone(&registry),
9668            Arc::new(ForwardingTable::default()),
9669        );
9670        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9671        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9672        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9673        drop(module_rx);
9674
9675        let replies = handler
9676            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
9677            .await
9678            .unwrap();
9679        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
9680        assert!(registry.get_module("aft").unwrap().is_none());
9681        assert!(registry
9682            .get_module_by_connection(module_ctx.connection_id)
9683            .unwrap()
9684            .is_none());
9685        assert_eq!(
9686            catalog_snapshot(&handler, &client_ctx, 301).await,
9687            (before, Vec::new())
9688        );
9689    }
9690
9691    /// A handler whose `route.open` parks after queueing the route.bind and
9692    /// before marking it enqueued, with module "aft" registered on connection
9693    /// 40. Returns the two-party barrier that parks and releases it.
9694    async fn route_open_held_after_bind_send() -> (
9695        ControlHandler,
9696        Arc<tokio::sync::Barrier>,
9697        RouteCtx,
9698        mpsc::Receiver<crate::router::OutboundFrame>,
9699    ) {
9700        let hold = Arc::new(tokio::sync::Barrier::new(2));
9701        let handler = ControlHandler::with_forwarding(
9702            Arc::new(Registry::default()),
9703            Arc::new(ForwardingTable::default()),
9704        )
9705        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
9706        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9707        hello_via_sink(
9708            &handler,
9709            &module_ctx,
9710            &mut module_rx,
9711            hello_frame("aft", PROTOCOL_VERSION, 7),
9712        )
9713        .await;
9714        (handler, hold, module_ctx, module_rx)
9715    }
9716
9717    /// A fast module acks the route.bind, and the reader commits the route,
9718    /// before `route.open` has marked the relay enqueued. The daemon used to
9719    /// read the missing relay as an abandoned bind and send the module a
9720    /// GOODBYE for the route it had just committed: the client was told its
9721    /// route was open, the module unbound the channel, and the client's first
9722    /// request was dropped on the module side as unbound.
9723    #[tokio::test]
9724    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
9725        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9726        let client = ConnectionId::new(41);
9727        let (client_ctx, mut client_rx) = route_ctx(client);
9728
9729        let (open_task, bind) = relay_route_open(
9730            &handler,
9731            client,
9732            &client_ctx.egress,
9733            &mut module_rx,
9734            200,
9735            "aft",
9736            "acked-before-mark",
9737        )
9738        .await;
9739        let (module_channel, module_epoch) = route_bind_channel(&bind);
9740        // route.open is now parked between queueing the bind and marking it.
9741        hold.wait().await;
9742
9743        // The module answers inside that window, and the route commits.
9744        assert!(handler
9745            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9746            .await
9747            .unwrap()
9748            .is_empty());
9749        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
9750
9751        hold.wait().await;
9752        assert!(
9753            open_task.await.unwrap().is_empty(),
9754            "route.open answers accepted"
9755        );
9756
9757        // Every frame route.open could have sent is already queued, because it
9758        // has returned; the module must have been sent nothing.
9759        match module_rx.try_recv() {
9760            Err(mpsc::error::TryRecvError::Empty) => {}
9761            Ok(outbound) => panic!(
9762                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
9763                outbound.frame.header.ty,
9764                outbound.frame.header.channel,
9765                outbound.frame.header.epoch
9766            ),
9767            Err(err) => panic!("module egress closed: {err}"),
9768        }
9769
9770        // The client's first request on the route is the next thing the module
9771        // reads, on the binding it just created.
9772        let router = Router::with_control_handler(Arc::new(handler.clone()));
9773        let request = Frame::build(
9774            FrameType::Request,
9775            Flags::new(true, Priority::Interactive, false),
9776            client_channel,
9777            client_epoch,
9778            201,
9779            b"first request".to_vec(),
9780        )
9781        .unwrap();
9782        router
9783            .route_for_connection(&client_ctx, request)
9784            .await
9785            .unwrap();
9786        let delivered = module_rx.try_recv().unwrap().frame;
9787        assert_eq!(delivered.header.ty, FrameType::Request);
9788        assert_eq!(
9789            (delivered.header.channel, delivered.header.epoch),
9790            (module_channel, module_epoch)
9791        );
9792        assert_eq!(delivered.body, b"first request");
9793    }
9794
9795    /// The client goes away while `route.open` is still queueing the bind, so
9796    /// the relay is dropped with no module reply. The teardown cannot send the
9797    /// GOODBYE itself (it could reach the module ahead of the bind), so
9798    /// `route.open` must send it once the bind is queued; otherwise the module
9799    /// keeps a binding the daemon will never commit.
9800    #[tokio::test]
9801    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
9802        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9803        let client = ConnectionId::new(42);
9804        let (client_ctx, _client_rx) = route_ctx(client);
9805
9806        let (open_task, bind) = relay_route_open(
9807            &handler,
9808            client,
9809            &client_ctx.egress,
9810            &mut module_rx,
9811            300,
9812            "aft",
9813            "released-before-mark",
9814        )
9815        .await;
9816        let (module_channel, module_epoch) = route_bind_channel(&bind);
9817        hold.wait().await;
9818
9819        handler.cleanup_connection(client).unwrap();
9820        assert!(
9821            module_rx.try_recv().is_err(),
9822            "client teardown must not GOODBYE a bind route.open is still queueing"
9823        );
9824
9825        hold.wait().await;
9826        let response = open_task.await.unwrap();
9827        assert_eq!(response.len(), 1);
9828        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9829
9830        let goodbye = module_rx
9831            .try_recv()
9832            .expect("module is told to drop the binding the daemon abandoned")
9833            .frame;
9834        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9835        assert_eq!(
9836            (goodbye.header.channel, goodbye.header.epoch),
9837            (module_channel, module_epoch)
9838        );
9839        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9840    }
9841
9842    /// The module acks inside the same window, but the client's egress closed
9843    /// first, so the reader refuses to commit. The ack proves the module holds
9844    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
9845    /// send a second one.
9846    #[tokio::test]
9847    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
9848        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9849        let client = ConnectionId::new(43);
9850        let (client_ctx, client_rx) = route_ctx(client);
9851
9852        let (open_task, bind) = relay_route_open(
9853            &handler,
9854            client,
9855            &client_ctx.egress,
9856            &mut module_rx,
9857            400,
9858            "aft",
9859            "gone-client-before-mark",
9860        )
9861        .await;
9862        let (module_channel, module_epoch) = route_bind_channel(&bind);
9863        hold.wait().await;
9864
9865        drop(client_rx);
9866        assert!(client_ctx.egress.is_closed());
9867        handler
9868            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9869            .await
9870            .unwrap();
9871
9872        hold.wait().await;
9873        let response = open_task.await.unwrap();
9874        assert_eq!(response.len(), 1);
9875        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9876
9877        let goodbye = module_rx
9878            .try_recv()
9879            .expect("module is told to drop the binding it acked")
9880            .frame;
9881        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9882        assert_eq!(
9883            (goodbye.header.channel, goodbye.header.epoch),
9884            (module_channel, module_epoch)
9885        );
9886        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9887        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
9888    }
9889
9890    /// The fence at the module-loop boundary, stated as its own contract: which
9891    /// forwarding failures are allowed to end the module connection that is being
9892    /// served. A `ConnectionClosing` naming some client is about that client, and
9893    /// a module connection is shared; the same error naming the module's own
9894    /// connection is about this connection and must stay fatal, as must failures
9895    /// that are about the forwarding table itself.
9896    #[test]
9897    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9898        let handler = ControlHandler::default();
9899        let module_connection = ConnectionId::new(30);
9900        let client_connection = ConnectionId::new(31);
9901
9902        handler
9903            .refuse_to_end_module_connection_for_a_client(
9904                module_connection,
9905                77,
9906                ForwardingError::ConnectionClosing {
9907                    connection_id: client_connection,
9908                },
9909            )
9910            .expect("a closing client must never end the module connection");
9911
9912        assert!(matches!(
9913            handler.refuse_to_end_module_connection_for_a_client(
9914                module_connection,
9915                78,
9916                ForwardingError::ConnectionClosing {
9917                    connection_id: module_connection,
9918                },
9919            ),
9920            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9921                connection_id
9922            })) if connection_id == module_connection
9923        ));
9924        assert!(matches!(
9925            handler.refuse_to_end_module_connection_for_a_client(
9926                module_connection,
9927                79,
9928                ForwardingError::Poisoned,
9929            ),
9930            Err(RouterError::Forwarding(ForwardingError::Poisoned))
9931        ));
9932        assert!(matches!(
9933            handler.refuse_to_end_module_connection_for_a_client(
9934                module_connection,
9935                80,
9936                ForwardingError::StaleModuleEndpoint,
9937            ),
9938            Err(RouterError::Forwarding(
9939                ForwardingError::StaleModuleEndpoint
9940            ))
9941        ));
9942    }
9943
9944    /// The spawn-attestation guard is what stops a connected module from claiming
9945    /// another module's identity and being stamped `Reserved` for it. Every other
9946    /// test that supplies a consumer_identity supplies a CORRECT one, because a
9947    /// correct one is what the rest of the flow needs -- so the guard's rejection
9948    /// branch was never the subject of an assertion, only its acceptance branch.
9949    ///
9950    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
9951    /// whole subc-core library suite green; only the forwarding integration tests
9952    /// notice, and they notice for unrelated reasons. This test exists so the
9953    /// refusal itself is asserted where the guard lives: it fails if the identity
9954    /// check stops refusing, which is the direction that matters, since a guard
9955    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
9956    #[tokio::test]
9957    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9958        let registry = Arc::new(Registry::default());
9959        let forwarding = Arc::new(ForwardingTable::default());
9960        let supervisor = SupervisorHandle::new();
9961        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9962        let handler =
9963            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9964                .with_supervisor(supervisor);
9965
9966        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9967        hello_via_sink(
9968            &handler,
9969            &target_ctx,
9970            &mut target_rx,
9971            hello_frame("target", PROTOCOL_VERSION, 1),
9972        )
9973        .await;
9974
9975        // A real supervised module id presenting the wrong nonce. This is the
9976        // impersonation case: the attacker knows a privileged module_id, which is
9977        // public, and guesses at the nonce, which is not.
9978        let wrong_nonce = handler
9979            .handle_control_frame(
9980                &route_ctx(ConnectionId::new(91)).0,
9981                route_open_frame_with_admission_facts(
9982                    20,
9983                    "target",
9984                    unique_project_root("admission-facts"),
9985                    Some(subc_control::ConsumerIdentity {
9986                        module_id: "fed".to_string(),
9987                        launch_nonce: "not-the-real-nonce".to_string(),
9988                    }),
9989                    None,
9990                ),
9991            )
9992            .await
9993            .unwrap();
9994        assert_eq!(
9995            parse_error(&wrong_nonce[0])["code"],
9996            "bad_consumer_identity",
9997            "a mismatched launch nonce must be refused, not stamped Reserved"
9998        );
9999
10000        // A module id the supervisor never spawned at all, so no nonce exists to
10001        // compare against. An implementation that treats "no record" as "nothing
10002        // to check" fails open here while passing the case above.
10003        let never_spawned = handler
10004            .handle_control_frame(
10005                &route_ctx(ConnectionId::new(92)).0,
10006                route_open_frame_with_admission_facts(
10007                    21,
10008                    "target",
10009                    unique_project_root("admission-facts"),
10010                    Some(subc_control::ConsumerIdentity {
10011                        module_id: "never-spawned".to_string(),
10012                        launch_nonce: "any-nonce".to_string(),
10013                    }),
10014                    None,
10015                ),
10016            )
10017            .await
10018            .unwrap();
10019        assert_eq!(
10020            parse_error(&never_spawned[0])["code"],
10021            "bad_consumer_identity",
10022            "an unspawned module_id must be refused rather than accepted for lack of a record"
10023        );
10024    }
10025
10026    /// The refusal test above proves the guard says NO. Nothing proved it can say
10027    /// YES, and the difference is not academic: replacing the whole authorization
10028    /// with `false` -- admitting no consumer identity at all, revoking Reserved
10029    /// standing for every supervised module in the fleet -- leaves 110 of the 111
10030    /// library tests GREEN. The one that notices does so by HANGING, because it
10031    /// waits for a bind that can no longer happen.
10032    ///
10033    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
10034    /// or flaky test, invites a RETRY rather than an investigation, and the retry
10035    /// hangs too and gets blamed on the runner. So a total revocation of the
10036    /// daemon's trust grant would have shipped behind a symptom nobody attributes
10037    /// to code.
10038    ///
10039    /// The bias is structural rather than accidental. A REFUSAL looks like a
10040    /// failure someone writes a test for; a GRANT looks like the happy path. Every
10041    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
10042    /// suite for that reason, and this one is the purest case in the daemon.
10043    ///
10044    /// This test asserts the EFFECT rather than the absence of an error: the module
10045    /// receives a RouteBind and it carries `Reserved` naming the attested module.
10046    /// A guard that admitted nobody would produce no bind at all; one that admitted
10047    /// everybody would stamp the wrong principal, which the refusal test catches.
10048    #[tokio::test]
10049    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10050        let registry = Arc::new(Registry::default());
10051        let forwarding = Arc::new(ForwardingTable::default());
10052        let supervisor = SupervisorHandle::new();
10053        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10054        let handler =
10055            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10056                .with_supervisor(supervisor);
10057
10058        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10059        hello_via_sink(
10060            &handler,
10061            &target_ctx,
10062            &mut target_rx,
10063            hello_frame("target", PROTOCOL_VERSION, 1),
10064        )
10065        .await;
10066
10067        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10068        let route_handler = handler.clone();
10069        let route_task = tokio::spawn(async move {
10070            route_handler
10071                .handle_control_frame(
10072                    &client_ctx,
10073                    route_open_frame_with_admission_facts(
10074                        30,
10075                        "target",
10076                        unique_project_root("admission-facts"),
10077                        Some(subc_control::ConsumerIdentity {
10078                            module_id: "fed".to_string(),
10079                            launch_nonce: "fed-nonce".to_string(),
10080                        }),
10081                        None,
10082                    ),
10083                )
10084                .await
10085                .unwrap()
10086        });
10087
10088        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
10089        // the very mutation it exists to catch -- a guard that admits nobody -- no
10090        // bind is ever sent, so it HUNG rather than failing. That reproduces the
10091        // exact defect being fixed: a total revocation detected only as a stalled
10092        // suite, which reads as flakiness and invites a retry. An acceptance test
10093        // that waits for an effect must bound the wait, or a red becomes a hang.
10094        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10095            .await
10096            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10097            .expect("module control channel closed before route.bind");
10098        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10099        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10100            panic!("expected route.bind")
10101        };
10102        assert_eq!(
10103            principal,
10104            Some(Principal::Reserved {
10105                module_id: "fed".to_string()
10106            }),
10107            "a correctly attested consumer must be stamped Reserved for its own id"
10108        );
10109
10110        handler
10111            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10112            .await
10113            .unwrap();
10114        assert!(route_task.await.unwrap().is_empty());
10115        assert!(
10116            matches!(
10117                serde_json::from_slice::<ClientControlResponse>(
10118                    &client_rx.recv().await.unwrap().body
10119                )
10120                .unwrap(),
10121                ClientControlResponse::RouteOpen { .. }
10122            ),
10123            "the route must actually open, not merely avoid an error"
10124        );
10125    }
10126
10127    #[tokio::test(start_paused = true)]
10128    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10129        let registry = Arc::new(Registry::default());
10130        let forwarding = Arc::new(ForwardingTable::default());
10131        let supervisor = SupervisorHandle::new();
10132        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10133        let handler =
10134            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10135                .with_supervisor(supervisor);
10136
10137        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10138        hello_via_sink(
10139            &handler,
10140            &target_ctx,
10141            &mut target_rx,
10142            hello_frame("target", PROTOCOL_VERSION, 1),
10143        )
10144        .await;
10145
10146        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10147        let direct_handler = handler.clone();
10148        let direct_open = tokio::spawn(async move {
10149            direct_handler
10150                .handle_control_frame(
10151                    &direct_ctx,
10152                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
10153                )
10154                .await
10155                .unwrap()
10156        });
10157        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10158            .await
10159            .expect("no direct route.bind within 5s")
10160            .expect("target control channel closed before direct route.bind");
10161        handler
10162            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10163            .await
10164            .unwrap();
10165        assert!(direct_open.await.unwrap().is_empty());
10166        let _ = direct_rx.recv().await.unwrap();
10167
10168        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10169        let reserved_handler = handler.clone();
10170        let reserved_open = tokio::spawn(async move {
10171            reserved_handler
10172                .handle_control_frame(
10173                    &reserved_ctx,
10174                    route_open_frame_with_admission_facts(
10175                        3,
10176                        "target",
10177                        unique_project_root("admission-facts"),
10178                        Some(ConsumerIdentity {
10179                            module_id: "fed".to_string(),
10180                            launch_nonce: "fed-nonce".to_string(),
10181                        }),
10182                        None,
10183                    ),
10184                )
10185                .await
10186                .unwrap()
10187        });
10188        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10189            .await
10190            .expect("no reserved route.bind within 5s")
10191            .expect("target control channel closed before reserved route.bind");
10192        handler
10193            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10194            .await
10195            .unwrap();
10196        assert!(reserved_open.await.unwrap().is_empty());
10197        let _ = reserved_rx.recv().await.unwrap();
10198
10199        forwarding
10200            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10201            .unwrap();
10202        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10203        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10204            module_id: Some("target".to_string()),
10205        })
10206        .unwrap();
10207        let census_frame =
10208            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10209        let response = handler
10210            .handle_control_frame(&census_ctx, census_frame)
10211            .await
10212            .unwrap()
10213            .pop()
10214            .unwrap();
10215        let actual: Value = serde_json::from_slice(&response.body).unwrap();
10216        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10217        assert!(matches!(
10218            decoded,
10219            ClientControlResponse::SupervisorRoutes { .. }
10220        ));
10221        let routes = actual["modules"][0]["routes"].as_array().unwrap();
10222        assert_eq!(routes.len(), 2);
10223        assert!(routes.iter().all(|route| route["draining"] == true));
10224        // The census carries WHY: the reason the drain was begun with, in the
10225        // route.closing vocabulary, on every draining route this drain marked.
10226        assert!(
10227            routes.iter().all(|route| route["drain_reason"] == "reload"),
10228            "draining routes must name the drain's reason: {routes:?}"
10229        );
10230        assert!(routes.iter().any(|route| {
10231            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10232        }));
10233        assert!(routes.iter().any(|route| {
10234            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10235        }));
10236
10237        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10238            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10239        if std::env::var_os("UPDATE_GOLDEN").is_some() {
10240            std::fs::write(
10241                &golden_path,
10242                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10243            )
10244            .unwrap();
10245        }
10246        let expected: Value =
10247            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10248        assert_eq!(actual, expected);
10249    }
10250
10251    async fn query_live_roots(
10252        handler: &ControlHandler,
10253        module_ctx: &RouteCtx,
10254    ) -> ModuleControlResponseToModule {
10255        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10256        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10257        let response = handler
10258            .handle_control_frame(module_ctx, frame)
10259            .await
10260            .unwrap()
10261            .pop()
10262            .unwrap();
10263        serde_json::from_slice(&response.body).unwrap()
10264    }
10265
10266    #[tokio::test(start_paused = true)]
10267    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10268        let registry = Arc::new(Registry::default());
10269        let forwarding = Arc::new(ForwardingTable::default());
10270        let handler = ControlHandler::with_forwarding(registry, forwarding);
10271        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10272        hello_via_sink(
10273            &handler,
10274            &target_ctx,
10275            &mut target_rx,
10276            hello_frame("target", PROTOCOL_VERSION, 1),
10277        )
10278        .await;
10279        let root = unique_project_root("live-roots-known");
10280        let path = ProjectRootId::from_path_allowing_missing(root.path())
10281            .unwrap()
10282            .as_path()
10283            .to_path_buf();
10284        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10285        let open_handler = handler.clone();
10286        let opened = tokio::spawn(async move {
10287            open_handler
10288                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10289                .await
10290                .unwrap()
10291        });
10292        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10293            .await
10294            .unwrap()
10295            .unwrap();
10296        handler
10297            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10298            .await
10299            .unwrap();
10300        assert!(opened.await.unwrap().is_empty());
10301        let _ = client_rx.recv().await.unwrap();
10302
10303        let root = unique_project_root("live-roots-pending");
10304        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10305            .unwrap()
10306            .as_path()
10307            .to_path_buf();
10308        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10309        let open_handler = handler.clone();
10310        let pending = tokio::spawn(async move {
10311            open_handler
10312                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10313                .await
10314                .unwrap()
10315        });
10316        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10317            .await
10318            .unwrap()
10319            .unwrap();
10320        let actual = query_live_roots(&handler, &target_ctx).await;
10321        let ModuleControlResponseToModule::LiveRoots {
10322            roots,
10323            unknown_root_bindings,
10324            total_bindings,
10325        } = actual
10326        else {
10327            panic!("expected live roots")
10328        };
10329        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10330        assert_eq!(unknown_root_bindings, 0);
10331        assert_eq!(
10332            roots.len(),
10333            2,
10334            "root-known arm must retain each canonical root"
10335        );
10336        assert_eq!(
10337            total_bindings,
10338            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10339        );
10340        let counts = roots
10341            .iter()
10342            .map(|root| (root.project_root.clone(), root.bound, root.pending))
10343            .collect::<Vec<_>>();
10344        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10345        expected.sort_by(|a, b| a.0.cmp(&b.0));
10346        assert_eq!(
10347            counts, expected,
10348            "roots must sort by path and count pending separately"
10349        );
10350        handler
10351            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10352            .await
10353            .unwrap();
10354        assert!(pending.await.unwrap().is_empty());
10355    }
10356
10357    #[tokio::test(start_paused = true)]
10358    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10359        let forwarding = Arc::new(ForwardingTable::default());
10360        let handler =
10361            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10362        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10363        hello_via_sink(
10364            &handler,
10365            &target_ctx,
10366            &mut target_rx,
10367            hello_frame("target", PROTOCOL_VERSION, 1),
10368        )
10369        .await;
10370        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10371        let pending = forwarding
10372            .begin_route_bind_relay_for_test(
10373                client_ctx.connection_id,
10374                client_ctx.egress.clone(),
10375                2,
10376                "target",
10377            )
10378            .unwrap();
10379        forwarding
10380            .complete_pending_relay(
10381                target_ctx.connection_id,
10382                pending.corr,
10383                RouteBindRelayOutcome::Accepted,
10384            )
10385            .unwrap();
10386        let actual = query_live_roots(&handler, &target_ctx).await;
10387        let ModuleControlResponseToModule::LiveRoots {
10388            roots,
10389            unknown_root_bindings,
10390            total_bindings,
10391        } = actual
10392        else {
10393            panic!("expected live roots")
10394        };
10395        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10396        assert_eq!(
10397            unknown_root_bindings, 1,
10398            "unknown-root arm must not read as no bindings"
10399        );
10400        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10401        assert_eq!(
10402            total_bindings,
10403            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10404        );
10405    }
10406
10407    /// A module reads HELLO_ACK as its first frame and exits on anything else,
10408    /// so the ack has to be on its outbound queue before the module is
10409    /// routable. The connection loop writes a handler's replies only after the
10410    /// handler returns; this test stops in exactly that gap, runs a real
10411    /// route.open from another connection, and only then writes whatever the
10412    /// HELLO handler returned, the way the loop would. If the ack were still a
10413    /// reply, the route.bind request would reach the module first.
10414    #[tokio::test(start_paused = true)]
10415    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10416        let forwarding = Arc::new(ForwardingTable::default());
10417        let handler =
10418            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10419        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10420        let replies = handler
10421            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10422            .await
10423            .unwrap();
10424        let queued_by_hello = module_rx.len();
10425
10426        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10427        let open_handler = handler.clone();
10428        let open = tokio::spawn(async move {
10429            open_handler
10430                .handle_control_frame(
10431                    &client_ctx,
10432                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10433                )
10434                .await
10435                .unwrap()
10436        });
10437        // Let the route.open run until its route.bind is on the module's queue.
10438        let mut spins = 0;
10439        while module_rx.len() == queued_by_hello {
10440            spins += 1;
10441            assert!(spins < 10_000, "route.open never queued a route.bind");
10442            tokio::task::yield_now().await;
10443        }
10444
10445        // Now the connection loop's half: write the HELLO handler's replies.
10446        for reply in replies {
10447            module_ctx.egress.send(reply).await.unwrap();
10448        }
10449
10450        let first = module_rx.recv().await.unwrap().frame;
10451        assert_eq!(
10452            first.header.ty,
10453            FrameType::HelloAck,
10454            "the first frame a registering module reads must be its HELLO_ACK"
10455        );
10456        assert_eq!(first.header.corr, 7);
10457        let second = module_rx.recv().await.unwrap().frame;
10458        assert_eq!(second.header.ty, FrameType::Request);
10459        assert!(
10460            matches!(
10461                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10462                ModuleControlRequest::RouteBind { .. }
10463            ),
10464            "the route.bind follows the ack"
10465        );
10466        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10467
10468        handler
10469            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10470            .await
10471            .unwrap();
10472        assert!(open.await.unwrap().is_empty());
10473        let _ = client_rx.recv().await.unwrap();
10474    }
10475
10476    #[tokio::test(start_paused = true)]
10477    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10478        let handler = ControlHandler::with_forwarding(
10479            Arc::new(Registry::default()),
10480            Arc::new(ForwardingTable::default()),
10481        );
10482        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10483        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10484        hello_via_sink(
10485            &handler,
10486            &first_ctx,
10487            &mut first_rx,
10488            hello_frame("first", PROTOCOL_VERSION, 1),
10489        )
10490        .await;
10491        hello_via_sink(
10492            &handler,
10493            &second_ctx,
10494            &mut second_rx,
10495            hello_frame("second", PROTOCOL_VERSION, 2),
10496        )
10497        .await;
10498        let root = unique_project_root("second-only");
10499        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10500        let cloned = handler.clone();
10501        let open = tokio::spawn(async move {
10502            cloned
10503                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10504                .await
10505                .unwrap()
10506        });
10507        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10508            .await
10509            .unwrap()
10510            .unwrap();
10511        let first = query_live_roots(&handler, &first_ctx).await;
10512        let second = query_live_roots(&handler, &second_ctx).await;
10513        assert!(
10514            matches!(
10515                first,
10516                ModuleControlResponseToModule::LiveRoots {
10517                    total_bindings: 0,
10518                    ..
10519                }
10520            ),
10521            "cross-module scope must not expose another module's roots"
10522        );
10523        assert!(
10524            matches!(
10525                second,
10526                ModuleControlResponseToModule::LiveRoots {
10527                    total_bindings: 1,
10528                    ..
10529                }
10530            ),
10531            "second module must see its pending route"
10532        );
10533        handler
10534            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10535            .await
10536            .unwrap();
10537        assert!(open.await.unwrap().is_empty());
10538    }
10539
10540    #[tokio::test(start_paused = true)]
10541    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10542        let handler = ControlHandler::with_forwarding(
10543            Arc::new(Registry::default()),
10544            Arc::new(ForwardingTable::default()),
10545        );
10546        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10547        hello_via_sink(
10548            &handler,
10549            &target_ctx,
10550            &mut target_rx,
10551            hello_frame("target", PROTOCOL_VERSION, 1),
10552        )
10553        .await;
10554        let actual = query_live_roots(&handler, &target_ctx).await;
10555        let ModuleControlResponseToModule::LiveRoots {
10556            roots,
10557            unknown_root_bindings,
10558            total_bindings,
10559        } = actual
10560        else {
10561            panic!("expected live roots")
10562        };
10563        assert!(roots.is_empty());
10564        assert_eq!(unknown_root_bindings, 0);
10565        assert_eq!(total_bindings, 0);
10566        assert_eq!(
10567            total_bindings,
10568            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10569        );
10570    }
10571
10572    /// Read the vendored fed corpus rather than hand-building a package.
10573    ///
10574    /// A hand-built object encodes what the test author believed the carrier
10575    /// emits. These vectors are what it actually emits, and one of them exists
10576    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
10577    /// ignores additive unknown fields at the traversal emit terminus."
10578    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
10579        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10580            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
10581        let text = std::fs::read_to_string(&path)
10582            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
10583        let vectors: Vec<(String, Value)> = text
10584            .lines()
10585            .filter(|line| !line.trim().is_empty())
10586            .map(|line| {
10587                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
10588                let id = entry["corpus_id"]
10589                    .as_str()
10590                    .expect("every vector carries a corpus_id")
10591                    .to_string();
10592                (id, entry["package"].clone())
10593            })
10594            .collect();
10595        // Pin the count: a corpus that silently shrinks would take its coverage
10596        // with it, and a suite reading N-1 vectors reports the same clean pass
10597        // as one reading N.
10598        assert_eq!(
10599            vectors.len(),
10600            3,
10601            "vendored fed corpus changed size; re-sync from subc-federation"
10602        );
10603
10604        // Pin what makes the corpus DISCRIMINATING, not just present.
10605        //
10606        // The relay test below takes its expected value from the corpus, so the
10607        // corpus supplies the test's power to detect a lossy relay rather than
10608        // its correctness. A relay that dropped unrecognised fields would still
10609        // be caught -- but only by a package carrying fields it does not know.
10610        // Shrink every package to the handful of keys any implementation would
10611        // recognise and the test keeps passing over an input that can no longer
10612        // fail, which is the same clean green as a corpus that shrank away.
10613        //
10614        // So assert the precondition rather than duplicating the packages here:
10615        // at least one vector must carry a field beyond the small common set.
10616        // That is one claim to maintain instead of nine, and it fails loudly if
10617        // a re-sync ever flattens the corpus.
10618        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10619        let richest = vectors
10620            .iter()
10621            .filter_map(|(_, package)| package.as_object())
10622            .map(|object| {
10623                object
10624                    .keys()
10625                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10626                    .count()
10627            })
10628            .max()
10629            .unwrap_or(0);
10630        assert!(
10631            richest >= 2,
10632            "vendored corpus no longer carries a package with unmodelled fields, \
10633             so the relay test can no longer distinguish a verbatim relay from a lossy one"
10634        );
10635
10636        vectors
10637    }
10638
10639    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
10640    ///
10641    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
10642    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
10643    /// three-key object, so a relay that quietly dropped fields it did not
10644    /// recognise would satisfy it. These vectors carry nine keys including ones
10645    /// this crate has no type for, so a typed relay fails here and only here.
10646    #[tokio::test]
10647    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10648        for (corpus_id, package) in fed_admission_facts_vectors() {
10649            let registry = Arc::new(Registry::default());
10650            let forwarding = Arc::new(ForwardingTable::default());
10651            let supervisor = SupervisorHandle::new();
10652            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10653            let handler =
10654                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10655                    .with_supervisor(supervisor)
10656                    .with_admission_facts_config(
10657                        Some("fed".to_string()),
10658                        Some(vec!["target".to_string()]),
10659                    );
10660
10661            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10662            hello_via_sink(
10663                &handler,
10664                &target_ctx,
10665                &mut target_rx,
10666                hello_frame("target", PROTOCOL_VERSION, 1),
10667            )
10668            .await;
10669
10670            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
10671            let route_handler = handler.clone();
10672            let expected = package.clone();
10673            let route_task = tokio::spawn(async move {
10674                route_handler
10675                    .handle_control_frame(
10676                        &client_ctx,
10677                        route_open_frame_with_admission_facts(
10678                            20,
10679                            "target",
10680                            unique_project_root("admission-facts"),
10681                            Some(subc_control::ConsumerIdentity {
10682                                module_id: "fed".to_string(),
10683                                launch_nonce: "fed-nonce".to_string(),
10684                            }),
10685                            Some(package),
10686                        ),
10687                    )
10688                    .await
10689                    .unwrap()
10690            });
10691
10692            let bind_frame = target_rx.recv().await.unwrap();
10693            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10694            let ModuleControlRequest::RouteBind {
10695                admission_facts, ..
10696            } = bind
10697            else {
10698                panic!("{corpus_id}: expected route.bind")
10699            };
10700            assert_eq!(
10701                admission_facts,
10702                Some(expected),
10703                "{corpus_id}: relay must not add, drop or reshape any field"
10704            );
10705
10706            handler
10707                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10708                .await
10709                .unwrap();
10710            route_task.await.unwrap();
10711        }
10712    }
10713
10714    #[tokio::test]
10715    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
10716        let registry = Arc::new(Registry::default());
10717        let forwarding = Arc::new(ForwardingTable::default());
10718        let supervisor = SupervisorHandle::new();
10719        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10720        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
10721        let handler =
10722            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10723                .with_supervisor(supervisor)
10724                .with_admission_facts_config(
10725                    Some("fed".to_string()),
10726                    Some(vec!["target".to_string()]),
10727                );
10728
10729        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
10730        hello_via_sink(
10731            &handler,
10732            &target_ctx,
10733            &mut target_rx,
10734            hello_frame("target", PROTOCOL_VERSION, 1),
10735        )
10736        .await;
10737        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
10738        hello_via_sink(
10739            &handler,
10740            &other_ctx,
10741            &mut other_rx,
10742            hello_frame("other", PROTOCOL_VERSION, 2),
10743        )
10744        .await;
10745
10746        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
10747        let expected_facts = facts.clone();
10748        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
10749        let route_handler = handler.clone();
10750        let route_task = tokio::spawn(async move {
10751            route_handler
10752                .handle_control_frame(
10753                    &client_ctx,
10754                    route_open_frame_with_admission_facts(
10755                        10,
10756                        "target",
10757                        unique_project_root("admission-facts"),
10758                        Some(subc_control::ConsumerIdentity {
10759                            module_id: "fed".to_string(),
10760                            launch_nonce: "fed-nonce".to_string(),
10761                        }),
10762                        Some(facts.clone()),
10763                    ),
10764                )
10765                .await
10766                .unwrap()
10767        });
10768        let bind_frame = target_rx.recv().await.unwrap();
10769        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10770        let ModuleControlRequest::RouteBind {
10771            admission_facts, ..
10772        } = bind
10773        else {
10774            panic!("expected route.bind")
10775        };
10776        assert_eq!(admission_facts, Some(expected_facts));
10777        handler
10778            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10779            .await
10780            .unwrap();
10781        assert!(route_task.await.unwrap().is_empty());
10782        assert!(matches!(
10783            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
10784                .unwrap(),
10785            ClientControlResponse::RouteOpen { .. }
10786        ));
10787
10788        let direct = handler
10789            .handle_control_frame(
10790                &route_ctx(ConnectionId::new(73)).0,
10791                route_open_frame_with_admission_facts(
10792                    11,
10793                    "target",
10794                    unique_project_root("admission-facts"),
10795                    None,
10796                    Some(json!({"x": 1})),
10797                ),
10798            )
10799            .await
10800            .unwrap();
10801        assert_eq!(
10802            parse_error(&direct[0])["code"],
10803            "admission_facts_not_permitted"
10804        );
10805
10806        let different_reserved = handler
10807            .handle_control_frame(
10808                &route_ctx(ConnectionId::new(77)).0,
10809                route_open_frame_with_admission_facts(
10810                    15,
10811                    "target",
10812                    unique_project_root("admission-facts"),
10813                    Some(subc_control::ConsumerIdentity {
10814                        module_id: "other".to_string(),
10815                        launch_nonce: "other-nonce".to_string(),
10816                    }),
10817                    Some(json!({"x": 1})),
10818                ),
10819            )
10820            .await
10821            .unwrap();
10822        assert_eq!(
10823            parse_error(&different_reserved[0])["code"],
10824            "admission_facts_not_permitted"
10825        );
10826
10827        let other_target = handler
10828            .handle_control_frame(
10829                &route_ctx(ConnectionId::new(74)).0,
10830                route_open_frame_with_admission_facts(
10831                    12,
10832                    "other",
10833                    unique_project_root("admission-facts"),
10834                    Some(subc_control::ConsumerIdentity {
10835                        module_id: "fed".to_string(),
10836                        launch_nonce: "fed-nonce".to_string(),
10837                    }),
10838                    Some(json!({"x": 1})),
10839                ),
10840            )
10841            .await
10842            .unwrap();
10843        assert_eq!(
10844            parse_error(&other_target[0])["code"],
10845            "admission_facts_target_not_allowed"
10846        );
10847
10848        let nonexistent = handler
10849            .handle_control_frame(
10850                &route_ctx(ConnectionId::new(75)).0,
10851                route_open_frame_with_admission_facts(
10852                    13,
10853                    "missing",
10854                    unique_project_root("admission-facts"),
10855                    None,
10856                    Some(json!({"x": 1})),
10857                ),
10858            )
10859            .await
10860            .unwrap();
10861        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
10862
10863        let described = handler
10864            .handle_control_frame(
10865                &route_ctx(ConnectionId::new(76)).0,
10866                Frame::build(
10867                    FrameType::Request,
10868                    control_flags(),
10869                    0,
10870                    0,
10871                    14,
10872                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10873                )
10874                .unwrap(),
10875            )
10876            .await
10877            .unwrap();
10878        let ClientControlResponse::ServerDescribe { capabilities, .. } =
10879            serde_json::from_slice(&described[0].body).unwrap()
10880        else {
10881            panic!("expected server.describe response")
10882        };
10883        assert!(capabilities
10884            .iter()
10885            .any(|cap| cap == "admission_facts_relay_v1"));
10886    }
10887
10888    #[tokio::test]
10889    async fn admission_facts_without_configured_carrier_are_rejected() {
10890        let registry = Arc::new(Registry::default());
10891        let forwarding = Arc::new(ForwardingTable::default());
10892        let handler = ControlHandler::with_forwarding(registry, forwarding);
10893        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10894        hello_via_sink(
10895            &handler,
10896            &target_ctx,
10897            &mut target_rx,
10898            hello_frame("target", PROTOCOL_VERSION, 1),
10899        )
10900        .await;
10901
10902        let responses = handler
10903            .handle_control_frame(
10904                &route_ctx(ConnectionId::new(79)).0,
10905                route_open_frame_with_admission_facts(
10906                    16,
10907                    "target",
10908                    unique_project_root("admission-facts"),
10909                    None,
10910                    Some(json!({"x": 1})),
10911                ),
10912            )
10913            .await
10914            .unwrap();
10915        assert_eq!(
10916            parse_error(&responses[0])["code"],
10917            "admission_facts_not_permitted"
10918        );
10919    }
10920
10921    #[tokio::test]
10922    async fn route_open_relays_consumer_capabilities_verbatim() {
10923        let registry = Arc::new(Registry::default());
10924        let forwarding = Arc::new(ForwardingTable::default());
10925        let handler =
10926            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10927        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10928        hello_via_sink(
10929            &handler,
10930            &module_ctx,
10931            &mut module_rx,
10932            hello_frame("aft", PROTOCOL_VERSION, 7),
10933        )
10934        .await;
10935
10936        let expected = vec!["elicitation".to_string(), "roots".to_string()];
10937        let expected_for_request = expected.clone();
10938        let project_root = unique_project_root("consumer-capabilities-present");
10939        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10940        let route_handler = handler.clone();
10941        let route_task = tokio::spawn(async move {
10942            route_handler
10943                .handle_control_frame(
10944                    &client_ctx,
10945                    route_open_frame_with_consumer_capabilities(
10946                        401,
10947                        "aft",
10948                        project_root,
10949                        Some(expected_for_request),
10950                    ),
10951                )
10952                .await
10953                .unwrap()
10954        });
10955        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10956            .await
10957            .unwrap()
10958            .unwrap();
10959        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10960        let ModuleControlRequest::RouteBind {
10961            consumer_capabilities,
10962            ..
10963        } = bind
10964        else {
10965            panic!("expected route.bind request, got {bind:?}");
10966        };
10967        assert_eq!(consumer_capabilities, Some(expected.clone()));
10968
10969        handler
10970            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10971            .await
10972            .unwrap();
10973        let route_response = route_task.await.unwrap();
10974        assert!(route_response.is_empty());
10975        let published = client_rx.recv().await.unwrap();
10976        assert!(matches!(
10977            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10978            ClientControlResponse::RouteOpen { .. }
10979        ));
10980    }
10981
10982    #[tokio::test]
10983    async fn route_open_without_consumer_capabilities_relays_none() {
10984        let registry = Arc::new(Registry::default());
10985        let forwarding = Arc::new(ForwardingTable::default());
10986        let handler =
10987            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10988        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
10989        hello_via_sink(
10990            &handler,
10991            &module_ctx,
10992            &mut module_rx,
10993            hello_frame("aft", PROTOCOL_VERSION, 7),
10994        )
10995        .await;
10996
10997        let project_root = unique_project_root("consumer-capabilities-absent");
10998        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
10999        let route_handler = handler.clone();
11000        let route_task = tokio::spawn(async move {
11001            route_handler
11002                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11003                .await
11004                .unwrap()
11005        });
11006        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11007            .await
11008            .unwrap()
11009            .unwrap();
11010        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11011        let ModuleControlRequest::RouteBind {
11012            consumer_capabilities,
11013            ..
11014        } = bind
11015        else {
11016            panic!("expected route.bind request, got {bind:?}");
11017        };
11018        assert_eq!(consumer_capabilities, None);
11019
11020        handler
11021            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11022            .await
11023            .unwrap();
11024        let route_response = route_task.await.unwrap();
11025        assert!(route_response.is_empty());
11026        let published = client_rx.recv().await.unwrap();
11027        assert!(matches!(
11028            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11029            ClientControlResponse::RouteOpen { .. }
11030        ));
11031    }
11032
11033    /// Opens a route to a freshly registered `aft` with `sent` as the
11034    /// route.open's role_versions, acks the bind, and returns the role_versions
11035    /// the module's bind carried.
11036    async fn bind_role_versions_for(
11037        sent: Option<BTreeMap<String, String>>,
11038        connection: u64,
11039    ) -> Option<BTreeMap<String, String>> {
11040        let registry = Arc::new(Registry::default());
11041        let forwarding = Arc::new(ForwardingTable::default());
11042        let handler =
11043            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11044        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11045        hello_via_sink(
11046            &handler,
11047            &module_ctx,
11048            &mut module_rx,
11049            hello_frame("aft", PROTOCOL_VERSION, 7),
11050        )
11051        .await;
11052        let project_root = unique_project_root("role-versions");
11053        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11054        let route_handler = handler.clone();
11055        let route_task = tokio::spawn(async move {
11056            route_handler
11057                .handle_control_frame(
11058                    &client_ctx,
11059                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
11060                )
11061                .await
11062                .unwrap()
11063        });
11064        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11065            .await
11066            .expect("a well-formed route.open reaches the module as a bind")
11067            .unwrap();
11068        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11069        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11070            panic!("expected route.bind request, got {bind:?}");
11071        };
11072        handler
11073            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11074            .await
11075            .unwrap();
11076        assert!(route_task.await.unwrap().is_empty());
11077        let published = client_rx.recv().await.unwrap();
11078        assert!(matches!(
11079            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11080            ClientControlResponse::RouteOpen { .. }
11081        ));
11082        role_versions
11083    }
11084
11085    #[tokio::test]
11086    async fn route_open_relays_role_versions_verbatim() {
11087        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11088        assert_eq!(
11089            bind_role_versions_for(Some(sent.clone()), 141).await,
11090            Some(sent)
11091        );
11092    }
11093
11094    /// An empty map declares nothing, so the provider sees no field rather
11095    /// than an empty object it would have to treat as a second "none".
11096    #[tokio::test]
11097    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11098        assert_eq!(bind_role_versions_for(None, 143).await, None);
11099        assert_eq!(
11100            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11101            None
11102        );
11103    }
11104
11105    #[tokio::test]
11106    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11107        let registry = Arc::new(Registry::default());
11108        let forwarding = Arc::new(ForwardingTable::default());
11109        let handler =
11110            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11111        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11112        hello_via_sink(
11113            &handler,
11114            &module_ctx,
11115            &mut module_rx,
11116            hello_frame("aft", PROTOCOL_VERSION, 7),
11117        )
11118        .await;
11119
11120        let nine: BTreeMap<String, String> = (0..9)
11121            .map(|index| (format!("role-{index}"), "v1".to_string()))
11122            .collect();
11123        for (label, malformed) in [
11124            (
11125                "invalid role name",
11126                role_versions(&[("Tool_Provider", "v1")]),
11127            ),
11128            ("invalid version", role_versions(&[("tool-provider", "v0")])),
11129            ("nine entries", nine),
11130        ] {
11131            // A refused open answers at once; one that reached the module
11132            // would wait for its bind ack and trip this timeout.
11133            let responses = tokio::time::timeout(
11134                Duration::from_secs(1),
11135                handler.handle_control_frame(
11136                    &route_ctx(ConnectionId::new(148)).0,
11137                    route_open_frame_with_role_versions(
11138                        404,
11139                        "aft",
11140                        unique_project_root("role-versions-malformed"),
11141                        Some(malformed),
11142                    ),
11143                ),
11144            )
11145            .await
11146            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11147            .unwrap();
11148            assert_eq!(responses.len(), 1, "{label}");
11149            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11150            let error = parse_error(&responses[0]);
11151            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11152            assert_eq!(
11153                error["detail"]["field"], "role_versions",
11154                "{label}: {error}"
11155            );
11156            assert!(
11157                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11158                "{label}: a malformed declaration is terminal"
11159            );
11160            assert!(
11161                module_rx.try_recv().is_err(),
11162                "{label}: the module must never see a bind"
11163            );
11164        }
11165    }
11166
11167    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
11168    /// consumer can tell this daemon forwards the field from one that would
11169    /// drop it.
11170    #[tokio::test]
11171    async fn route_role_versions_capability_is_advertised() {
11172        let handler = ControlHandler::new(Arc::new(Registry::default()));
11173        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11174        let ack = hello_via_sink(
11175            &handler,
11176            &ctx,
11177            &mut rx,
11178            hello_frame("m", PROTOCOL_VERSION, 1),
11179        )
11180        .await;
11181        let ack = parse_ack(&ack);
11182        assert!(
11183            ack.subc_capabilities
11184                .iter()
11185                .any(|c| c == "route-role-versions/v1"),
11186            "{:?}",
11187            ack.subc_capabilities
11188        );
11189
11190        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11191        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11192        let reply = handler
11193            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11194            .await
11195            .unwrap()
11196            .pop()
11197            .unwrap();
11198        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11199            serde_json::from_slice(&reply.body).unwrap()
11200        else {
11201            panic!("not a server.describe reply");
11202        };
11203        assert!(
11204            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11205            "{capabilities:?}"
11206        );
11207    }
11208
11209    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
11210    /// catalog entry without `events` (the module declares none) from a daemon
11211    /// that never reports declared events.
11212    #[tokio::test]
11213    async fn catalog_events_capability_is_advertised() {
11214        let handler = ControlHandler::new(Arc::new(Registry::default()));
11215        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11216        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11217        let reply = handler
11218            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11219            .await
11220            .unwrap()
11221            .pop()
11222            .unwrap();
11223        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11224            serde_json::from_slice(&reply.body).unwrap()
11225        else {
11226            panic!("not a server.describe reply");
11227        };
11228        assert!(
11229            capabilities
11230                .iter()
11231                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11232            "{capabilities:?}"
11233        );
11234    }
11235
11236    #[tokio::test]
11237    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
11238        let handler = ControlHandler::new(Arc::new(Registry::default()));
11239        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
11240        let ack = hello_via_sink(
11241            &handler,
11242            &module,
11243            &mut module_rx,
11244            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
11245        )
11246        .await;
11247        let ack = parse_ack(&ack);
11248        assert!(
11249            ack.subc_capabilities
11250                .iter()
11251                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11252            "{:?}",
11253            ack.subc_capabilities
11254        );
11255        assert!(ack
11256            .subc_capabilities
11257            .iter()
11258            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11259
11260        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
11261        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11262        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11263        let reply = handler
11264            .handle_control_frame(&client, frame)
11265            .await
11266            .unwrap()
11267            .pop()
11268            .unwrap();
11269        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11270            serde_json::from_slice(&reply.body).unwrap()
11271        else {
11272            panic!("not a server.describe reply");
11273        };
11274        assert!(
11275            capabilities
11276                .iter()
11277                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11278            "{capabilities:?}"
11279        );
11280        assert!(capabilities
11281            .iter()
11282            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11283    }
11284
11285    #[tokio::test]
11286    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
11287        let handler = ControlHandler::new(Arc::new(Registry::default()));
11288        let (module, mut rx) = route_ctx(ConnectionId::new(154));
11289        let ack = hello_via_sink(
11290            &handler,
11291            &module,
11292            &mut rx,
11293            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
11294        )
11295        .await;
11296        assert!(parse_ack(&ack)
11297            .subc_capabilities
11298            .iter()
11299            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11300        let (client, _rx) = route_ctx(ConnectionId::new(155));
11301        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11302        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11303        let reply = handler
11304            .handle_control_frame(&client, frame)
11305            .await
11306            .unwrap()
11307            .pop()
11308            .unwrap();
11309        let ClientControlResponse::ServerDescribe { capabilities, .. } =
11310            serde_json::from_slice(&reply.body).unwrap()
11311        else {
11312            panic!("not a server.describe reply")
11313        };
11314        assert!(capabilities
11315            .iter()
11316            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11317    }
11318
11319    #[tokio::test]
11320    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11321        let registry = Arc::new(Registry::default());
11322        let forwarding = Arc::new(ForwardingTable::default());
11323        let handler =
11324            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11325                .with_health_probe_timeout(Duration::from_secs(5));
11326        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11327        hello_via_sink(
11328            &handler,
11329            &module_ctx,
11330            &mut module_rx,
11331            non_routable_hello_frame_with_control_ops(
11332                "mcp",
11333                300,
11334                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11335            ),
11336        )
11337        .await;
11338        assert!(registry
11339            .get_module("mcp")
11340            .unwrap()
11341            .unwrap()
11342            .manifest
11343            .provides
11344            .is_empty());
11345
11346        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11347        let route_response = handler
11348            .handle_control_frame(
11349                &route_client_ctx,
11350                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11351            )
11352            .await
11353            .unwrap();
11354        assert_eq!(route_response[0].header.ty, FrameType::Error);
11355        assert_eq!(
11356            parse_error(&route_response[0])["code"],
11357            "target_unavailable"
11358        );
11359        assert!(parse_error(&route_response[0])["message"]
11360            .as_str()
11361            .unwrap()
11362            .contains("does not provide the requested target"));
11363        assert!(module_rx.try_recv().is_err());
11364
11365        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11366        let health_handler = handler.clone();
11367        let health_task = tokio::spawn(async move {
11368            health_handler
11369                .handle_control_frame(
11370                    &health_client_ctx,
11371                    supervisor_health_probe_frame(302, "mcp"),
11372                )
11373                .await
11374                .unwrap()
11375        });
11376        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11377            .await
11378            .unwrap()
11379            .unwrap();
11380        assert_eq!(
11381            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11382            ModuleControlRequest::HealthCheck {}
11383        );
11384        handler
11385            .handle_control_frame(
11386                &module_ctx,
11387                health_response(health_frame.header.corr, HealthStatus::Ok),
11388            )
11389            .await
11390            .unwrap();
11391        let health_response = health_task.await.unwrap();
11392        assert_eq!(health_response[0].header.ty, FrameType::Response);
11393        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11394            ClientControlResponse::SupervisorHealthProbe {
11395                module_id, status, ..
11396            } => {
11397                assert_eq!(module_id, "mcp");
11398                assert_eq!(status, HealthStatus::Ok);
11399            }
11400            other => panic!("unexpected health response: {other:?}"),
11401        }
11402
11403        // Exercise the forwarding cleanup path directly while leaving the registry
11404        // advertisement in place. If cleanup leaves a stale control sink behind,
11405        // the next probe will enqueue onto it and wait for the long probe timeout
11406        // instead of returning an immediate no-connection error.
11407        forwarding
11408            .cleanup_connection(module_ctx.connection_id)
11409            .unwrap();
11410        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11411        let cleanup_response = tokio::time::timeout(
11412            Duration::from_millis(200),
11413            handler.handle_control_frame(
11414                &cleanup_probe_ctx,
11415                supervisor_health_probe_frame(303, "mcp"),
11416            ),
11417        )
11418        .await
11419        .expect("probe should fail immediately when the control lane is gone")
11420        .unwrap();
11421        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11422        assert_eq!(
11423            parse_error(&cleanup_response[0])["code"],
11424            "target_unavailable"
11425        );
11426        assert!(parse_error(&cleanup_response[0])["message"]
11427            .as_str()
11428            .unwrap()
11429            .contains("no module connection"));
11430
11431        handler
11432            .cleanup_connection(module_ctx.connection_id)
11433            .unwrap();
11434    }
11435
11436    #[tokio::test]
11437    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11438        let registry = Arc::new(Registry::default());
11439        let supervisor_handle = SupervisorHandle::new();
11440        let supervisor =
11441            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
11442                .with_handle(supervisor_handle.clone())
11443                .with_connection_file_path(
11444                    std::env::temp_dir()
11445                        .join(format!("subc-route-open-warming-{}", std::process::id())),
11446                );
11447        let module = supervisor
11448            .supervise_configured(
11449                ModuleSpec {
11450                    module_id: "warming".to_string(),
11451                    program: fake_aft_stub_path(),
11452                    args: Vec::new(),
11453                    env: Vec::new(),
11454                    reserved: false,
11455                    reserved_prefixes: Vec::new(),
11456                    protocol: ModuleProtocol::Subc,
11457                    overlap: Default::default(),
11458                },
11459                true,
11460            )
11461            .unwrap();
11462        assert_eq!(module.state().unwrap(), ModuleState::Running);
11463
11464        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11465        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11466        let response = handler
11467            .handle_control_frame(
11468                &ctx,
11469                route_open_frame(304, "warming", unique_project_root("warming")),
11470            )
11471            .await
11472            .unwrap();
11473        module.stop().await.unwrap();
11474
11475        assert_eq!(response[0].header.ty, FrameType::Error);
11476        let error = parse_error(&response[0]);
11477        assert_eq!(error["code"], "module_warming");
11478        assert!(error["message"]
11479            .as_str()
11480            .unwrap()
11481            .contains("state=running, enabled=true, live=false"));
11482    }
11483
11484    #[test]
11485    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11486        let handler = ControlHandler::new(Arc::new(Registry::default()));
11487        let capture = EventCapture::default();
11488        let _subscriber =
11489            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11490        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11491        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11492        let pending = (0..limit).collect::<Vec<_>>();
11493        let response = handler
11494            .route_open_capacity_refusal(
11495                &ctx,
11496                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11497                "busy",
11498                pending.len(),
11499                limit,
11500            )
11501            .unwrap();
11502        assert_eq!(parse_error(&response)["code"], "target_unavailable");
11503        let event = capture
11504            .events()
11505            .into_iter()
11506            .find(|event| {
11507                event.target == "control"
11508                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11509            })
11510            .expect("connection admission refusal event");
11511        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11512        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11513    }
11514
11515    #[test]
11516    fn route_open_target_cap_logs_admission_reason_and_capacity() {
11517        let handler = ControlHandler::new(Arc::new(Registry::default()));
11518        let capture = EventCapture::default();
11519        let _subscriber =
11520            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11521        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
11522        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
11523        let guards = (0..limit)
11524            .map(|_| {
11525                handler
11526                    .route_bind_concurrency
11527                    .try_admit("busy", limit)
11528                    .unwrap()
11529            })
11530            .collect::<Vec<_>>();
11531        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
11532            Err(in_flight) => in_flight,
11533            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
11534        };
11535        let response = handler
11536            .route_open_target_capacity_refusal(
11537                &ctx,
11538                &route_open_frame(397, "busy", unique_project_root("target-cap")),
11539                "busy",
11540                in_flight,
11541            )
11542            .unwrap();
11543        assert_eq!(parse_error(&response)["code"], "target_unavailable");
11544        let event = capture
11545            .events()
11546            .into_iter()
11547            .find(|event| {
11548                event.target == "control"
11549                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
11550            })
11551            .expect("target admission refusal event");
11552        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11553        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11554        drop(guards);
11555    }
11556
11557    #[tokio::test(flavor = "current_thread")]
11558    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
11559        let handler = ControlHandler::new(Arc::new(Registry::default()));
11560        let capture = EventCapture::default();
11561        let _subscriber =
11562            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11563        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
11564        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11565
11566        let _ = handler
11567            .handle_client_control_request(
11568                &ctx,
11569                frame,
11570                ClientControlRequest::SupervisorSetEnabled {
11571                    module_id: "broca".to_string(),
11572                    enabled: false,
11573                },
11574                &mut None,
11575            )
11576            .await
11577            .unwrap();
11578
11579        let events = capture
11580            .events()
11581            .into_iter()
11582            .filter(|event| {
11583                event.target == "control"
11584                    && event.fields.get("message").map(String::as_str)
11585                        == Some("supervisor request received")
11586            })
11587            .collect::<Vec<_>>();
11588        assert_eq!(events.len(), 1, "one supervisor request log line");
11589        let event = &events[0];
11590        assert_eq!(event.level, tracing::Level::INFO);
11591        assert_eq!(
11592            event.fields.get("op"),
11593            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
11594        );
11595        assert_eq!(
11596            event.fields.get("module_id"),
11597            Some(&"\"broca\"".to_string())
11598        );
11599        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
11600        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
11601        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
11602    }
11603
11604    #[tokio::test(flavor = "current_thread")]
11605    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
11606        let registry = Arc::new(Registry::default());
11607        let module_connection = ConnectionId::new(303);
11608        registry
11609            .register_with_control_ops(
11610                manifest("aft", PROTOCOL_VERSION),
11611                PROTOCOL_VERSION,
11612                module_connection,
11613                module_baseline_control_ops(),
11614            )
11615            .unwrap();
11616        let handler = ControlHandler::new(registry);
11617        let capture = EventCapture::default();
11618        let _subscriber =
11619            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11620        let (ctx, _rx) = route_ctx(module_connection);
11621        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11622
11623        let _ = handler
11624            .handle_client_control_request(
11625                &ctx,
11626                frame,
11627                ClientControlRequest::SupervisorSetEnabled {
11628                    module_id: "broca".to_string(),
11629                    enabled: false,
11630                },
11631                &mut None,
11632            )
11633            .await
11634            .unwrap();
11635
11636        let callers = capture
11637            .events()
11638            .into_iter()
11639            .filter(|event| {
11640                event.fields.get("message").map(String::as_str)
11641                    == Some("supervisor request received")
11642            })
11643            .map(|event| event.fields.get("caller").cloned())
11644            .collect::<Vec<_>>();
11645        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
11646    }
11647
11648    #[tokio::test(flavor = "current_thread")]
11649    async fn supervisor_list_does_not_log_request_received() {
11650        let handler = ControlHandler::new(Arc::new(Registry::default()));
11651        let capture = EventCapture::default();
11652        let _subscriber =
11653            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11654        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
11655        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
11656
11657        handler
11658            .handle_client_control_request(
11659                &ctx,
11660                frame,
11661                ClientControlRequest::SupervisorList {},
11662                &mut None,
11663            )
11664            .await
11665            .unwrap();
11666
11667        assert!(capture.events().into_iter().all(|event| {
11668            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11669        }));
11670    }
11671
11672    #[tokio::test(flavor = "current_thread")]
11673    async fn supervisor_rescan_preview_does_not_log_request_received() {
11674        let handler = ControlHandler::new(Arc::new(Registry::default()));
11675        let capture = EventCapture::default();
11676        let _subscriber =
11677            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11678        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
11679        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
11680
11681        handler
11682            .handle_client_control_request(
11683                &ctx,
11684                frame,
11685                ClientControlRequest::SupervisorRescan { preview: true },
11686                &mut None,
11687            )
11688            .await
11689            .unwrap();
11690
11691        assert!(capture.events().into_iter().all(|event| {
11692            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11693        }));
11694    }
11695
11696    struct ApproveOperator;
11697    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
11698        fn prompt(
11699            &self,
11700            _: &str,
11701            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
11702        ) -> crate::operator_confirm::ProviderResult {
11703            crate::operator_confirm::ProviderResult::Approved
11704        }
11705    }
11706
11707    #[tokio::test]
11708    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
11709        assert_eq!(
11710            ControlHandler::default().privileged_operator_prompt,
11711            PrivilegedOperatorPrompt::Log
11712        );
11713        for (initial, changed, running) in [
11714            ("log", "enforce", PrivilegedOperatorPrompt::Log),
11715            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
11716        ] {
11717            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
11718            let path = dir.join("subc.jsonc");
11719            std::fs::write(
11720                &path,
11721                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
11722            )
11723            .unwrap();
11724            let config = crate::daemon_config::load(&path).unwrap().unwrap();
11725            let registry = Arc::new(Registry::default());
11726            let handle = SupervisorHandle::new();
11727            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
11728                .with_handle(handle.clone());
11729            let handler = ControlHandler::new(registry)
11730                .with_supervisor(handle)
11731                .with_privileged_operator_prompt(config.privileged_operator_prompt)
11732                .with_supervisor_rescan(supervisor, &path, config.port);
11733            handler.forwarding.operator_confirms().configure(
11734                Arc::new(ApproveOperator),
11735                crate::operator_confirm::OperatorLimits::default(),
11736            );
11737            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
11738
11739            for (disk_mode, needs_restart) in [
11740                (Some(initial), false),
11741                (Some(changed), true),
11742                (Some(changed), true),
11743                (None, running == PrivilegedOperatorPrompt::Enforce),
11744                (Some(initial), false),
11745            ] {
11746                let mut doc = json!({"version":1});
11747                if let Some(mode) = disk_mode {
11748                    doc["privileged_operator_prompt"] = json!(mode);
11749                }
11750                std::fs::write(&path, doc.to_string()).unwrap();
11751                for preview in [true, false] {
11752                    let request = ClientControlRequest::SupervisorRescan { preview };
11753                    let frame = Frame::build(
11754                        FrameType::Request,
11755                        control_flags(),
11756                        0,
11757                        0,
11758                        1,
11759                        serde_json::to_vec(&request).unwrap(),
11760                    )
11761                    .unwrap();
11762                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11763                    assert_eq!(frames.len(), 1);
11764                    let ClientControlResponse::SupervisorRescan { result } =
11765                        serde_json::from_slice(&frames[0].body).unwrap()
11766                    else {
11767                        panic!("expected a rescan response");
11768                    };
11769                    let expected = if needs_restart {
11770                        vec!["privileged_operator_prompt"]
11771                    } else {
11772                        Vec::new()
11773                    };
11774                    assert_eq!(result.restart_required, expected);
11775                    assert_eq!(handler.privileged_operator_prompt, running);
11776                    assert_eq!(
11777                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
11778                        running
11779                    );
11780                }
11781            }
11782        }
11783    }
11784
11785    #[tokio::test]
11786    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
11787        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
11788        let path = dir.join("subc.jsonc");
11789        let registry = Arc::new(Registry::default());
11790        let handle = SupervisorHandle::new();
11791        let supervisor =
11792            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
11793        let handler = ControlHandler::new(registry)
11794            .with_supervisor(handle)
11795            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
11796            .with_supervisor_rescan(supervisor, &path, None);
11797        handler.forwarding.operator_confirms().configure(
11798            Arc::new(ApproveOperator),
11799            crate::operator_confirm::OperatorLimits::default(),
11800        );
11801        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
11802        for preview in [true, false] {
11803            std::fs::write(
11804                &path,
11805                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
11806            )
11807            .unwrap();
11808            let request = ClientControlRequest::SupervisorRescan { preview };
11809            let frame = Frame::build(
11810                FrameType::Request,
11811                control_flags(),
11812                0,
11813                0,
11814                1,
11815                serde_json::to_vec(&request).unwrap(),
11816            )
11817            .unwrap();
11818            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11819            assert_eq!(frames.len(), 1);
11820            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
11821            assert_eq!(
11822                handler.privileged_operator_prompt,
11823                PrivilegedOperatorPrompt::Enforce
11824            );
11825        }
11826    }
11827
11828    /// One wire code has several senders, so the refusal line names the check
11829    /// that refused. This drives the shared refusal path for ordinary refusals
11830    /// with an unregistered
11831    /// target and requires the branch label on the event.
11832    #[tokio::test]
11833    async fn route_open_refusal_names_the_check_that_refused() {
11834        let handler = ControlHandler::new(Arc::new(Registry::default()));
11835        let capture = EventCapture::default();
11836        let _subscriber =
11837            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11838        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11839        let response = handler
11840            .handle_control_frame(
11841                &ctx,
11842                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
11843            )
11844            .await
11845            .unwrap();
11846
11847        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11848        let event = capture
11849            .events()
11850            .into_iter()
11851            .find(|event| {
11852                event.target == "control"
11853                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11854            })
11855            .expect("route.open refusal event");
11856        assert_eq!(
11857            event.fields.get("reason"),
11858            Some(&"\"not_registered\"".to_string())
11859        );
11860    }
11861
11862    #[tokio::test]
11863    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
11864        let registry = Arc::new(Registry::default());
11865        let supervisor_handle = SupervisorHandle::new();
11866        let supervisor =
11867            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
11868                .with_handle(supervisor_handle.clone())
11869                .with_connection_file_path(std::env::temp_dir().join(format!(
11870                    "subc-route-open-refusal-info-{}",
11871                    std::process::id()
11872                )));
11873        let module = supervisor
11874            .supervise_configured(
11875                ModuleSpec {
11876                    module_id: "warming".to_string(),
11877                    program: fake_aft_stub_path(),
11878                    args: Vec::new(),
11879                    env: Vec::new(),
11880                    reserved: false,
11881                    reserved_prefixes: Vec::new(),
11882                    protocol: ModuleProtocol::Subc,
11883                    overlap: Default::default(),
11884                },
11885                true,
11886            )
11887            .unwrap();
11888        assert_eq!(module.state().unwrap(), ModuleState::Running);
11889
11890        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11891        assert!(handler
11892            .counters()
11893            .snapshot()
11894            .get("route_open_refused_by_code")
11895            .is_none());
11896        let capture = EventCapture::default();
11897        let _subscriber =
11898            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11899        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
11900        let response = handler
11901            .handle_control_frame(
11902                &ctx,
11903                route_open_frame(394, "warming", unique_project_root("refusal-info")),
11904            )
11905            .await
11906            .unwrap();
11907        module.stop().await.unwrap();
11908
11909        assert_eq!(parse_error(&response[0])["code"], "module_warming");
11910        let event = capture
11911            .events()
11912            .into_iter()
11913            .find(|event| {
11914                event.target == "control"
11915                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
11916            })
11917            .expect("route.open refusal event");
11918        assert_eq!(
11919            event.fields.get("module_id"),
11920            Some(&"\"warming\"".to_string())
11921        );
11922        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
11923        assert_eq!(
11924            event.fields.get("reason"),
11925            Some(&"\"supervised_not_registered\"".to_string())
11926        );
11927        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
11928        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
11929        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
11930        assert_eq!(
11931            handler.counters().snapshot()["route_open_refused_by_code"],
11932            json!({ "module_warming": 1 })
11933        );
11934    }
11935
11936    const OUTAGE_START: &str = "route.open refusing module: not serving";
11937    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
11938
11939    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
11940        capture
11941            .events()
11942            .into_iter()
11943            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
11944            .collect()
11945    }
11946
11947    fn supervise_stub(
11948        registry: &Arc<Registry>,
11949        module_id: &str,
11950        enabled: bool,
11951    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
11952        let supervisor_handle = SupervisorHandle::new();
11953        let supervisor =
11954            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
11955                .with_handle(supervisor_handle.clone())
11956                .with_connection_file_path(std::env::temp_dir().join(format!(
11957                    "subc-route-outage-{module_id}-{}",
11958                    std::process::id()
11959                )));
11960        let module = supervisor
11961            .supervise_configured(
11962                ModuleSpec {
11963                    module_id: module_id.to_string(),
11964                    program: fake_aft_stub_path(),
11965                    args: Vec::new(),
11966                    env: Vec::new(),
11967                    reserved: false,
11968                    reserved_prefixes: Vec::new(),
11969                    protocol: ModuleProtocol::Subc,
11970                    overlap: Default::default(),
11971                },
11972                enabled,
11973            )
11974            .unwrap();
11975        (supervisor_handle, module)
11976    }
11977
11978    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
11979        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
11980            module_id: module_id.to_string(),
11981            drain_timeout_ms: Some(50),
11982        })
11983        .unwrap();
11984        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11985    }
11986
11987    /// Two handlers built over one forwarding table must share one outage
11988    /// tracker; separate trackers would each log their own opening line for
11989    /// the same outage.
11990    #[test]
11991    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
11992        let registry = Arc::new(Registry::default());
11993        let forwarding = Arc::new(ForwardingTable::default());
11994        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11995        let second = ControlHandler::with_forwarding(registry, forwarding);
11996        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
11997    }
11998
11999    /// A client can name any module id it likes. Refusing an unknown one,
12000    /// however often, must not create outage state or outage lines, or the
12001    /// tracker would be a memory sink any client could fill.
12002    #[tokio::test(flavor = "current_thread")]
12003    async fn route_open_unknown_module_refusals_add_no_outage_state() {
12004        let handler = ControlHandler::new(Arc::new(Registry::default()));
12005        let capture = EventCapture::default();
12006        let _subscriber =
12007            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12008        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12009        for corr in 0..8 {
12010            let response = handler
12011                .handle_control_frame(
12012                    &ctx,
12013                    route_open_frame(
12014                        380 + corr,
12015                        &format!("nobody-{corr}"),
12016                        unique_project_root("outage-unknown"),
12017                    ),
12018                )
12019                .await
12020                .unwrap();
12021            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12022        }
12023
12024        assert_eq!(handler.route_outages.tracked_module_count(), 0);
12025        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12026        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12027    }
12028
12029    /// Drives the refusal path end to end: a supervised module that served
12030    /// before and stopped being registered with no instruction to stop is a
12031    /// WARN, and the same module refused after an operator `supervisor.restart`
12032    /// is an INFO.
12033    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12034    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12035        let registry = Arc::new(Registry::default());
12036        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
12037        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12038        let capture = EventCapture::default();
12039        let _subscriber =
12040            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12041        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12042        // The stub never registers, so pretend it served once: otherwise every
12043        // refusal would fall in its startup window.
12044        handler.route_outages.record_accepted("outage-restart");
12045
12046        let response = handler
12047            .handle_control_frame(
12048                &ctx,
12049                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12050            )
12051            .await
12052            .unwrap();
12053        assert_eq!(response[0].header.ty, FrameType::Error);
12054        let starts = outage_lines(&capture, OUTAGE_START);
12055        assert_eq!(starts.len(), 1, "{starts:?}");
12056        assert_eq!(starts[0].level, tracing::Level::WARN);
12057        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12058        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12059        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12060        handler.route_outages.record_accepted("outage-restart");
12061        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12062
12063        let restart = handler
12064            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12065            .await
12066            .unwrap();
12067        assert_eq!(
12068            restart[0].header.ty,
12069            FrameType::Response,
12070            "{:?}",
12071            parse_error(&restart[0])
12072        );
12073        handler
12074            .handle_control_frame(
12075                &ctx,
12076                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
12077            )
12078            .await
12079            .unwrap();
12080        module.stop().await.unwrap();
12081
12082        let starts = outage_lines(&capture, OUTAGE_START);
12083        assert_eq!(starts.len(), 2, "{starts:?}");
12084        assert_eq!(starts[1].level, tracing::Level::INFO);
12085        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
12086    }
12087
12088    /// A restart refused before it touched the module (here: the module is
12089    /// disabled) must clear its operator mark, so the next real outage is
12090    /// still reported as a warning.
12091    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12092    async fn failed_operator_restart_leaves_no_operator_mark() {
12093        let registry = Arc::new(Registry::default());
12094        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
12095        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12096        let capture = EventCapture::default();
12097        let _subscriber =
12098            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12099        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
12100        handler.route_outages.record_accepted("outage-disabled");
12101
12102        let restart = handler
12103            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
12104            .await
12105            .unwrap();
12106        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
12107        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
12108
12109        handler
12110            .handle_control_frame(
12111                &ctx,
12112                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
12113            )
12114            .await
12115            .unwrap();
12116        let starts = outage_lines(&capture, OUTAGE_START);
12117        assert_eq!(starts.len(), 1, "{starts:?}");
12118        assert_eq!(starts[0].level, tracing::Level::WARN);
12119    }
12120
12121    #[tokio::test(flavor = "current_thread")]
12122    async fn route_open_unknown_module_escapes_target_module_id() {
12123        let handler = ControlHandler::new(Arc::new(Registry::default()));
12124        let capture = EventCapture::default();
12125        let _subscriber =
12126            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12127        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
12128        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12129        let response = handler
12130            .handle_control_frame(
12131                &ctx,
12132                route_open_frame(
12133                    395,
12134                    hostile_module_id,
12135                    unique_project_root("hostile-target-module-id"),
12136                ),
12137            )
12138            .await
12139            .unwrap();
12140
12141        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12142        let event = capture
12143            .events()
12144            .into_iter()
12145            .find(|event| {
12146                event.target == "control"
12147                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12148            })
12149            .expect("route.open unknown-module refusal event");
12150        let logged = event.fields.get("module_id").expect("module_id field");
12151        assert!(!logged.bytes().any(|byte| byte < 0x20));
12152        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12153    }
12154
12155    #[tokio::test(flavor = "current_thread")]
12156    async fn route_open_module_rejection_uses_daemon_counter_key() {
12157        let registry = Arc::new(Registry::default());
12158        let forwarding = Arc::new(ForwardingTable::default());
12159        let handler =
12160            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12161        let module_connection = ConnectionId::new(95);
12162        let (module_ctx, mut module_rx) = route_ctx(module_connection);
12163        hello_via_sink(
12164            &handler,
12165            &module_ctx,
12166            &mut module_rx,
12167            hello_frame("aft", PROTOCOL_VERSION, 395),
12168        )
12169        .await;
12170
12171        let client_connection = ConnectionId::new(96);
12172        let (client_ctx, _client_rx) = route_ctx(client_connection);
12173        let capture = EventCapture::default();
12174        let _subscriber =
12175            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12176        let (route_task, bind) = relay_route_open(
12177            &handler,
12178            client_connection,
12179            &client_ctx.egress,
12180            &mut module_rx,
12181            396,
12182            "aft",
12183            "hostile-module-code",
12184        )
12185        .await;
12186        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
12187        let rejection = Frame::build(
12188            FrameType::Error,
12189            control_flags(),
12190            0,
12191            0,
12192            bind.header.corr,
12193            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
12194        )
12195        .unwrap();
12196        handler
12197            .handle_control_frame(&module_ctx, rejection)
12198            .await
12199            .unwrap();
12200
12201        let response = route_task.await.unwrap();
12202        assert_eq!(parse_error(&response[0])["code"], hostile_code);
12203        let counters = handler.counters().snapshot();
12204        assert_eq!(
12205            counters["route_open_refused_by_code"],
12206            json!({ "module_rejected": 1 })
12207        );
12208        assert!(counters["route_open_refused_by_code"]
12209            .get(hostile_code)
12210            .is_none());
12211
12212        let event = capture
12213            .events()
12214            .into_iter()
12215            .find(|event| {
12216                event.target == "control"
12217                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
12218            })
12219            .expect("route.open module-rejection refusal event");
12220        let logged = event.fields.get("module_code").expect("module_code field");
12221        assert!(!logged.bytes().any(|byte| byte < 0x20));
12222        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12223    }
12224
12225    #[tokio::test]
12226    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12227        let registry = Arc::new(Registry::default());
12228        let supervisor_handle = SupervisorHandle::new();
12229        let missing_program = std::env::temp_dir().join(format!(
12230            "subc-route-open-missing-program-{}",
12231            std::process::id()
12232        ));
12233        let supervisor =
12234            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
12235                .with_handle(supervisor_handle.clone());
12236        let module = supervisor
12237            .supervise_configured(
12238                ModuleSpec {
12239                    module_id: "failed".to_string(),
12240                    program: missing_program,
12241                    args: Vec::new(),
12242                    env: Vec::new(),
12243                    reserved: false,
12244                    reserved_prefixes: Vec::new(),
12245                    protocol: ModuleProtocol::Subc,
12246                    overlap: Default::default(),
12247                },
12248                true,
12249            )
12250            .unwrap();
12251        assert_eq!(module.state().unwrap(), ModuleState::Failed);
12252
12253        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12254        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12255        let response = handler
12256            .handle_control_frame(
12257                &ctx,
12258                route_open_frame(305, "failed", unique_project_root("failed")),
12259            )
12260            .await
12261            .unwrap();
12262
12263        assert_eq!(response[0].header.ty, FrameType::Error);
12264        let error = parse_error(&response[0]);
12265        assert_eq!(error["code"], "target_unavailable");
12266        assert!(error["message"]
12267            .as_str()
12268            .unwrap()
12269            .contains("state=failed, enabled=true, live=false"));
12270    }
12271
12272    #[tokio::test]
12273    async fn route_open_role_mismatch_remains_target_unavailable() {
12274        let registry = Arc::new(Registry::default());
12275        let handler = ControlHandler::new(Arc::clone(&registry));
12276        handler
12277            .handle_control(
12278                ConnectionId::new(41),
12279                non_routable_hello_frame_with_control_ops("health-only", 306, None),
12280            )
12281            .unwrap();
12282
12283        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12284        let response = handler
12285            .handle_control_frame(
12286                &ctx,
12287                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12288            )
12289            .await
12290            .unwrap();
12291
12292        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12293        assert!(parse_error(&response[0])["message"]
12294            .as_str()
12295            .unwrap()
12296            .contains("does not provide the requested target"));
12297    }
12298
12299    #[tokio::test]
12300    async fn route_open_inactive_registration_remains_target_unavailable() {
12301        let registry = Arc::new(Registry::default());
12302        let handler = ControlHandler::new(Arc::clone(&registry));
12303        handler
12304            .handle_control(
12305                ConnectionId::new(43),
12306                hello_frame("inactive", PROTOCOL_VERSION, 308),
12307            )
12308            .unwrap();
12309        assert!(registry
12310            .set_module_state_for_test("inactive", ChannelState::Closed)
12311            .unwrap());
12312
12313        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12314        let response = handler
12315            .handle_control_frame(
12316                &ctx,
12317                route_open_frame(309, "inactive", unique_project_root("inactive")),
12318            )
12319            .await
12320            .unwrap();
12321
12322        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12323        assert!(parse_error(&response[0])["message"]
12324            .as_str()
12325            .unwrap()
12326            .contains("is not active"));
12327    }
12328
12329    #[tokio::test]
12330    async fn late_health_reply_is_recorded_through_the_module_response_path() {
12331        let registry = Arc::new(Registry::default());
12332        let forwarding = Arc::new(ForwardingTable::default());
12333        let supervisor_handle = SupervisorHandle::new();
12334        let supervisor =
12335            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
12336                .with_forwarding(Arc::clone(&forwarding))
12337                .with_handle(supervisor_handle.clone());
12338        let module = supervisor
12339            .supervise_configured(
12340                crate::ModuleSpec {
12341                    module_id: "late-health-response".to_string(),
12342                    program: PathBuf::from("disabled-module"),
12343                    args: Vec::new(),
12344                    env: Vec::new(),
12345                    reserved: false,
12346                    reserved_prefixes: Vec::new(),
12347                    protocol: ModuleProtocol::Subc,
12348                    overlap: Default::default(),
12349                },
12350                false,
12351            )
12352            .unwrap();
12353        let handler =
12354            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12355                .with_supervisor(supervisor_handle);
12356        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12357        handler
12358            .handle_control_frame(
12359                &module_ctx,
12360                hello_frame_with_control_ops(
12361                    "late-health-response",
12362                    PROTOCOL_VERSION,
12363                    7,
12364                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12365                ),
12366            )
12367            .await
12368            .unwrap();
12369        let probe_started_at = Instant::now() - Duration::from_millis(80);
12370        let pending = forwarding
12371            .begin_health_probe_rpc_for(
12372                "late-health-response",
12373                MODULE_CONTROL_OP_HEALTH_CHECK,
12374                probe_started_at,
12375                Instant::now() - Duration::from_millis(1),
12376            )
12377            .unwrap();
12378        assert!(forwarding
12379            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12380            .unwrap());
12381
12382        let responses = handler
12383            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12384            .await
12385            .unwrap();
12386
12387        assert!(responses.is_empty());
12388        let health = module.status().unwrap().health;
12389        assert_eq!(health.late_answer_count, 1);
12390        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12391    }
12392
12393    #[tokio::test]
12394    async fn health_probe_timeout_and_module_death_are_typed() {
12395        let registry = Arc::new(Registry::default());
12396        let forwarding = Arc::new(ForwardingTable::default());
12397        let handler =
12398            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12399                .with_health_probe_timeout(Duration::from_millis(50));
12400        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12401        hello_via_sink(
12402            &handler,
12403            &module_ctx,
12404            &mut module_rx,
12405            hello_frame_with_control_ops(
12406                "aft",
12407                PROTOCOL_VERSION,
12408                7,
12409                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12410            ),
12411        )
12412        .await;
12413
12414        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12415        let responses = handler
12416            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12417            .await
12418            .unwrap();
12419        assert_eq!(responses[0].header.ty, FrameType::Error);
12420        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12421        let _ = module_rx.try_recv();
12422
12423        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12424        let health_handler = handler.clone();
12425        let death_task = tokio::spawn(async move {
12426            health_handler
12427                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12428                .await
12429                .unwrap()
12430        });
12431        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12432            .await
12433            .unwrap()
12434            .unwrap();
12435        handler
12436            .cleanup_connection(module_ctx.connection_id)
12437            .unwrap();
12438        let responses = death_task.await.unwrap();
12439        assert_eq!(responses[0].header.ty, FrameType::Error);
12440        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12441    }
12442
12443    #[test]
12444    fn hello_requires_exact_protocol_version() {
12445        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12446            let registry = Arc::new(Registry::default());
12447            let handler = ControlHandler::new(Arc::clone(&registry));
12448            let responses = handler
12449                .handle_control(
12450                    ConnectionId::new(connection),
12451                    hello_frame("aft", offered, 9),
12452                )
12453                .unwrap();
12454
12455            assert_eq!(responses.len(), 1);
12456            assert_eq!(responses[0].header.ty, FrameType::Error);
12457            let error = parse_error(&responses[0]);
12458            assert_eq!(error["code"], "version_unsupported");
12459            assert!(registry.get_module("aft").unwrap().is_none());
12460            assert_eq!(registry.active_registration_count().unwrap(), 0);
12461        }
12462    }
12463
12464    #[test]
12465    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12466        let registry = Arc::new(Registry::default());
12467        let forwarding = Arc::new(ForwardingTable::default());
12468        let handler =
12469            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12470        let module_connection = ConnectionId::new(301);
12471        let registration = registry
12472            .register_with_control_ops(
12473                manifest("aft-push", PROTOCOL_VERSION),
12474                PROTOCOL_VERSION,
12475                module_connection,
12476                module_baseline_control_ops(),
12477            )
12478            .unwrap();
12479        let (module_tx, _module_rx) = mpsc::channel(8);
12480        let endpoint = forwarding
12481            .register_module_connection(
12482                module_connection,
12483                "aft-push".to_string(),
12484                PROTOCOL_VERSION,
12485                manifest_concurrency(&registration.manifest),
12486                FrameSink::new(module_tx),
12487            )
12488            .unwrap();
12489
12490        // A push op this version does not know is ignored (forward-compat), not errored.
12491        let unknown = Frame::build(
12492            FrameType::Push,
12493            control_flags(),
12494            0,
12495            0,
12496            5,
12497            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12498        )
12499        .unwrap();
12500        let out = handler.handle_status_update(endpoint, unknown).unwrap();
12501        assert!(
12502            out.is_empty(),
12503            "unknown push op must be ignored, got {out:?}"
12504        );
12505
12506        // A malformed body for a KNOWN op is a real error worth surfacing.
12507        let malformed = Frame::build(
12508            FrameType::Push,
12509            control_flags(),
12510            0,
12511            0,
12512            6,
12513            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
12514        )
12515        .unwrap();
12516        let out = handler.handle_status_update(endpoint, malformed).unwrap();
12517        assert_eq!(out.len(), 1);
12518        assert_eq!(out[0].header.ty, FrameType::Error);
12519        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
12520    }
12521
12522    #[test]
12523    fn hello_rejected_when_connection_already_owns_client_routes() {
12524        let registry = Arc::new(Registry::default());
12525        let forwarding = Arc::new(ForwardingTable::default());
12526        let handler =
12527            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12528        // Commits a client route on connection 202 (bound to a module on conn 101).
12529        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
12530        let client_connection = ConnectionId::new(202);
12531
12532        // That same connection now tries to register as a module: rejected, so one
12533        // connection never holds both client-route and module-endpoint state.
12534        let responses = handler
12535            .handle_control(
12536                client_connection,
12537                hello_frame("aft-second", PROTOCOL_VERSION, 9),
12538            )
12539            .unwrap();
12540        assert_eq!(responses[0].header.ty, FrameType::Error);
12541        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
12542        assert!(registry.get_module("aft-second").unwrap().is_none());
12543    }
12544
12545    #[tokio::test]
12546    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
12547        let registry = Arc::new(Registry::default());
12548        let forwarding = Arc::new(ForwardingTable::default());
12549        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
12550        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
12551        hello_via_sink(
12552            &handler,
12553            &module_ctx,
12554            &mut module_rx,
12555            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
12556        )
12557        .await;
12558        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
12559        let pending = forwarding
12560            .begin_route_bind_relay_for_test(
12561                client_ctx.connection_id,
12562                client_ctx.egress.clone(),
12563                2,
12564                "alpha",
12565            )
12566            .unwrap();
12567        forwarding
12568            .complete_pending_relay(
12569                module_ctx.connection_id,
12570                pending.corr,
12571                RouteBindRelayOutcome::Accepted,
12572            )
12573            .unwrap();
12574        client_rx.try_recv().unwrap();
12575        for module_id in ["beta", "alpha"] {
12576            let replies = handler
12577                .handle_control_frame(
12578                    &module_ctx,
12579                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
12580                )
12581                .await
12582                .unwrap();
12583            assert_eq!(replies.len(), 1, "second HELLO must be refused");
12584            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
12585        }
12586        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
12587        assert!(registry.get_module("beta").unwrap().is_none());
12588        assert!(matches!(
12589            forwarding
12590                .lookup_data_route(
12591                    client_ctx.connection_id,
12592                    pending.client_channel,
12593                    pending.client_epoch,
12594                )
12595                .unwrap(),
12596            DataRoute::Client(DataRouteState::Bound(_))
12597        ));
12598        assert!(handler
12599            .hello_launch_nonces
12600            .lock()
12601            .unwrap()
12602            .presented(module_ctx.connection_id, Some("alpha-nonce")));
12603        assert!(module_rx.try_recv().is_err());
12604    }
12605
12606    #[test]
12607    fn reserved_module_hello_requires_matching_launch_nonce() {
12608        let registry = Arc::new(Registry::default());
12609        let supervisor = SupervisorHandle::new();
12610        // The supervisor recorded the nonce it injected when it spawned the reserved
12611        // module; the HELLO verifier checks against the same shared handle.
12612        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12613        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
12614
12615        // A HELLO with NO nonce is rejected.
12616        let no_nonce = handler
12617            .handle_control(
12618                ConnectionId::new(1),
12619                hello_frame("vault", PROTOCOL_VERSION, 1),
12620            )
12621            .unwrap();
12622        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
12623        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
12624        assert!(registry.get_module("vault").unwrap().is_none());
12625
12626        // A HELLO with the WRONG nonce is rejected.
12627        let wrong = handler
12628            .handle_control(
12629                ConnectionId::new(2),
12630                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
12631            )
12632            .unwrap();
12633        assert_eq!(wrong[0].header.ty, FrameType::Error);
12634        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
12635        assert!(registry.get_module("vault").unwrap().is_none());
12636
12637        // A HELLO with the CORRECT nonce registers.
12638        let ok = handler
12639            .handle_control(
12640                ConnectionId::new(3),
12641                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
12642            )
12643            .unwrap();
12644        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
12645        assert!(registry.get_module("vault").unwrap().is_some());
12646    }
12647
12648    #[test]
12649    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
12650        let registry = Arc::new(Registry::default());
12651        let supervisor = SupervisorHandle::new();
12652        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12653        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12654        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
12655
12656        let squat = handler
12657            .handle_control(
12658                ConnectionId::new(1),
12659                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
12660            )
12661            .unwrap();
12662        assert_eq!(squat[0].header.ty, FrameType::Error);
12663        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
12664        assert!(parse_error(&squat[0])["message"]
12665            .as_str()
12666            .unwrap()
12667            .contains("fed:"));
12668
12669        let accepted_peer = handler
12670            .handle_control(
12671                ConnectionId::new(2),
12672                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
12673            )
12674            .unwrap();
12675        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
12676
12677        let accepted_short = handler
12678            .handle_control(
12679                ConnectionId::new(3),
12680                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
12681            )
12682            .unwrap();
12683        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
12684
12685        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
12686            let response = handler
12687                .handle_control(
12688                    ConnectionId::new(conn),
12689                    hello_frame(module_id, PROTOCOL_VERSION, conn),
12690                )
12691                .unwrap();
12692            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
12693        }
12694    }
12695
12696    #[test]
12697    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
12698        let registry = Arc::new(Registry::default());
12699        let supervisor = SupervisorHandle::new();
12700        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12701        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12702        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
12703        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
12704
12705        let owner_nonce = handler
12706            .handle_control(
12707                ConnectionId::new(1),
12708                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
12709            )
12710            .unwrap();
12711        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
12712        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
12713        assert!(registry.get_module("fed:special").unwrap().is_none());
12714
12715        let exact_nonce = handler
12716            .handle_control(
12717                ConnectionId::new(2),
12718                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
12719            )
12720            .unwrap();
12721        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
12722        assert!(registry.get_module("fed:special").unwrap().is_some());
12723    }
12724
12725    #[test]
12726    fn non_reserved_module_ignores_launch_nonce() {
12727        let registry = Arc::new(Registry::default());
12728        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
12729        // registration succeeds whether a spawned process echoes a nonce or not.
12730        let handler = ControlHandler::new(Arc::clone(&registry));
12731        let no_nonce = handler
12732            .handle_control(
12733                ConnectionId::new(1),
12734                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
12735            )
12736            .unwrap();
12737        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
12738        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
12739
12740        let echoed_nonce = handler
12741            .handle_control(
12742                ConnectionId::new(2),
12743                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
12744            )
12745            .unwrap();
12746        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
12747        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
12748    }
12749
12750    #[test]
12751    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
12752        let handler = ControlHandler::default();
12753        let conn = ConnectionId::new(1);
12754        let malformed = Frame::build(
12755            FrameType::Hello,
12756            control_flags(),
12757            0,
12758            0,
12759            3,
12760            b"{not json".to_vec(),
12761        )
12762        .unwrap();
12763
12764        let error = handler.handle_control(conn, malformed).unwrap();
12765        assert_eq!(error[0].header.ty, FrameType::Error);
12766        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
12767
12768        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12769        let pong = handler.handle_control(conn, ping).unwrap();
12770        assert_eq!(pong[0].header.ty, FrameType::Pong);
12771        assert_eq!(pong[0].header.corr, 4);
12772    }
12773
12774    #[test]
12775    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
12776        let registry = Arc::new(Registry::default());
12777        let handler = ControlHandler::new(Arc::clone(&registry));
12778
12779        handler
12780            .handle_control(
12781                ConnectionId::new(1),
12782                hello_frame("aft", PROTOCOL_VERSION, 1),
12783            )
12784            .unwrap();
12785        let duplicate = handler
12786            .handle_control(
12787                ConnectionId::new(2),
12788                hello_frame("aft", PROTOCOL_VERSION, 2),
12789            )
12790            .unwrap();
12791
12792        assert_eq!(duplicate[0].header.ty, FrameType::Error);
12793        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12794        let registration = registry.get_module("aft").unwrap().unwrap();
12795        assert_eq!(registration.connection_id, ConnectionId::new(1));
12796    }
12797
12798    #[test]
12799    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
12800        let registry = Arc::new(Registry::default());
12801        let forwarding = Arc::new(ForwardingTable::default());
12802        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
12803        let handler =
12804            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12805                .with_process_liveness(process_liveness);
12806        let (ctx, route_channel, route_epoch) =
12807            bind_liveness_route(&registry, &forwarding, "aft-dead");
12808        let responses = handler
12809            .handle_route_poll(
12810                &ctx,
12811                route_poll_frame(41, PollKind::Liveness, route_channel),
12812                route_channel,
12813                route_epoch,
12814                PollKind::Liveness,
12815            )
12816            .unwrap();
12817
12818        assert_eq!(responses.len(), 1);
12819        assert_eq!(responses[0].header.ty, FrameType::Response);
12820        assert_route_poll_liveness(&responses[0], false);
12821    }
12822
12823    #[test]
12824    fn liveness_poll_without_process_source_uses_bound_route() {
12825        let registry = Arc::new(Registry::default());
12826        let forwarding = Arc::new(ForwardingTable::default());
12827        let handler =
12828            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12829        let (ctx, route_channel, route_epoch) =
12830            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
12831        let responses = handler
12832            .handle_route_poll(
12833                &ctx,
12834                route_poll_frame(42, PollKind::Liveness, route_channel),
12835                route_channel,
12836                route_epoch,
12837                PollKind::Liveness,
12838            )
12839            .unwrap();
12840
12841        assert_route_poll_liveness(&responses[0], true);
12842    }
12843
12844    #[test]
12845    fn liveness_poll_untracked_process_source_uses_bound_route() {
12846        let registry = Arc::new(Registry::default());
12847        let forwarding = Arc::new(ForwardingTable::default());
12848        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
12849        let handler =
12850            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12851                .with_process_liveness(process_liveness);
12852        let (ctx, route_channel, route_epoch) =
12853            bind_liveness_route(&registry, &forwarding, "aft-untracked");
12854        let responses = handler
12855            .handle_route_poll(
12856                &ctx,
12857                route_poll_frame(43, PollKind::Liveness, route_channel),
12858                route_channel,
12859                route_epoch,
12860                PollKind::Liveness,
12861            )
12862            .unwrap();
12863
12864        assert_route_poll_liveness(&responses[0], true);
12865    }
12866
12867    #[tokio::test]
12868    async fn unknown_op_returns_unknown_control_op() {
12869        let handler = ControlHandler::default();
12870        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
12871        let request = Frame::build(
12872            FrameType::Request,
12873            control_flags(),
12874            0,
12875            0,
12876            55,
12877            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
12878        )
12879        .unwrap();
12880
12881        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12882
12883        assert_eq!(response.len(), 1);
12884        assert_eq!(response[0].header.ty, FrameType::Error);
12885        assert_eq!(response[0].header.corr, 55);
12886        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
12887    }
12888
12889    #[tokio::test]
12890    async fn supervisor_provenance_rejects_unknown_exact_module() {
12891        let handler = ControlHandler::default();
12892        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
12893        let request = Frame::build(
12894            FrameType::Request,
12895            control_flags(),
12896            0,
12897            0,
12898            57,
12899            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
12900        )
12901        .unwrap();
12902
12903        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12904
12905        assert_eq!(response.len(), 1);
12906        assert_eq!(response[0].header.ty, FrameType::Error);
12907        assert_eq!(response[0].header.corr, 57);
12908        let error = parse_error(&response[0]);
12909        assert_eq!(error["code"], "unknown_module");
12910        assert_eq!(error["message"], "module_id 'missing' is not supervised");
12911    }
12912
12913    #[test]
12914    fn provenance_probe_override_keeps_handler_tests_deterministic() {
12915        let expected = subc_control::RunningImageAgreement::Unavailable {
12916            reason: subc_control::RunningImageUnavailableReason::HashFailed,
12917        };
12918        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
12919        assert_eq!(handler.provenance_probe_override, Some(expected));
12920    }
12921
12922    #[test]
12923    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
12924        let verdict = reload_verdict(
12925            std::path::Path::new("/bin/new"),
12926            Some(std::path::Path::new("/bin/old")),
12927            subc_control::RunningImageAgreement::Unavailable {
12928                reason: subc_control::RunningImageUnavailableReason::HashFailed,
12929            },
12930        );
12931        assert!(matches!(
12932            verdict.path,
12933            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
12934                if configured == std::path::Path::new("/bin/new")
12935                    && spawned_from == std::path::Path::new("/bin/old")
12936        ));
12937    }
12938
12939    #[cfg(windows)]
12940    #[test]
12941    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
12942        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
12943        let configured = dir.join("module.exe");
12944        std::fs::write(&configured, b"executable path fixture").unwrap();
12945        let spawned = std::fs::canonicalize(&configured).unwrap();
12946        assert_ne!(
12947            configured, spawned,
12948            "fixture must exercise different path spellings"
12949        );
12950        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
12951        let lower_case = configured.to_string_lossy().to_lowercase();
12952        for spelling in [
12953            &configured,
12954            Path::new(&forward_slashes),
12955            Path::new(&lower_case),
12956        ] {
12957            let verdict = reload_verdict(
12958                spelling,
12959                Some(&spawned),
12960                subc_control::RunningImageAgreement::Unavailable {
12961                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12962                },
12963            );
12964            assert_eq!(
12965                verdict.path,
12966                ReloadPathAgreement::Match,
12967                "configured {spelling:?}, spawned {spawned:?}"
12968            );
12969        }
12970    }
12971
12972    #[cfg(windows)]
12973    #[test]
12974    fn windows_reload_path_keeps_different_existing_program_pending() {
12975        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
12976        let old = dir.join("old.exe");
12977        let new = dir.join("new.exe");
12978        std::fs::write(&old, b"same executable content").unwrap();
12979        std::fs::copy(&old, &new).unwrap();
12980        let spawned = std::fs::canonicalize(&old).unwrap();
12981        let image = subc_control::RunningImageAgreement::Unavailable {
12982            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12983        };
12984        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
12985        assert!(
12986            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
12987            if configured == new && spawned_from == spawned)
12988        );
12989        let missing = dir.join("missing.exe");
12990        assert!(matches!(
12991            reload_verdict(&missing, Some(&spawned), image).path,
12992            ReloadPathAgreement::Mismatch { .. }
12993        ));
12994    }
12995
12996    #[test]
12997    fn reload_verdict_detects_replaced_image_at_same_path() {
12998        let image = subc_control::RunningImageAgreement::Mismatch {
12999            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13000                digest: "old".into(),
13001            },
13002            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13003                digest: "new".into(),
13004            },
13005        };
13006        let verdict = reload_verdict(
13007            std::path::Path::new("/bin/same"),
13008            Some(std::path::Path::new("/bin/same")),
13009            image.clone(),
13010        );
13011        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13012        assert_eq!(verdict.image, image);
13013    }
13014
13015    #[test]
13016    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13017        let image = subc_control::RunningImageAgreement::Unavailable {
13018            reason: subc_control::RunningImageUnavailableReason::NotRunning,
13019        };
13020        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13021        assert_eq!(
13022            verdict.path,
13023            subc_control::ReloadPathAgreement::Unavailable {
13024                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13025            }
13026        );
13027        assert_eq!(verdict.image, image);
13028
13029        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13030            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13031        };
13032        let verdict = reload_verdict(
13033            std::path::Path::new("/bin/same"),
13034            Some(std::path::Path::new("/bin/same")),
13035            unconfirmed.clone(),
13036        );
13037        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13038        assert_eq!(verdict.image, unconfirmed);
13039    }
13040
13041    #[test]
13042    fn reload_verdict_preserves_each_image_unavailability_reason() {
13043        use subc_control::RunningImageUnavailableReason as Reason;
13044
13045        for reason in [
13046            Reason::NotRunning,
13047            Reason::UnsupportedPlatform,
13048            Reason::RunningExecutableUnreadable,
13049            Reason::SpawnedPathUnreadable,
13050            Reason::HashFailed,
13051            Reason::ProcessIdentityUnconfirmed,
13052            Reason::Unknown("future_probe_reason".to_string()),
13053        ] {
13054            let image = subc_control::RunningImageAgreement::Unavailable {
13055                reason: reason.clone(),
13056            };
13057            let verdict = reload_verdict(
13058                std::path::Path::new("/bin/same"),
13059                Some(std::path::Path::new("/bin/same")),
13060                image.clone(),
13061            );
13062            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13063            assert_eq!(verdict.image, image, "{reason:?}");
13064        }
13065    }
13066
13067    #[tokio::test]
13068    async fn malformed_control_bodies_return_invalid_control_body() {
13069        let handler = ControlHandler::default();
13070        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
13071
13072        for (corr, body) in [
13073            (56, br#"{"route_channel":1}"#.as_slice()),
13074            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
13075            (
13076                58,
13077                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
13078            ),
13079        ] {
13080            let request = Frame::build(
13081                FrameType::Request,
13082                control_flags(),
13083                0,
13084                0,
13085                corr,
13086                body.to_vec(),
13087            )
13088            .unwrap();
13089            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13090
13091            assert_eq!(response.len(), 1);
13092            assert_eq!(response[0].header.ty, FrameType::Error);
13093            assert_eq!(response[0].header.corr, corr);
13094            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
13095        }
13096    }
13097
13098    #[tokio::test]
13099    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
13100        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13101        let registry = Arc::new(Registry::default());
13102        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
13103        let router = Router::with_control_handler(Arc::clone(&control));
13104        let connection = router.begin_connection();
13105        let (ctx, mut rx) = route_ctx(connection.id());
13106
13107        router
13108            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
13109            .await
13110            .unwrap();
13111        let response = rx.recv().await.unwrap();
13112        let ack = parse_ack(&response);
13113        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13114        let channel = 1;
13115
13116        let goodbye =
13117            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
13118        router.route_for_connection(&ctx, goodbye).await.unwrap();
13119        assert!(rx.try_recv().is_err());
13120        assert!(registry.get_module("aft").unwrap().is_none());
13121
13122        router
13123            .route_for_connection(&ctx, channel_request(channel, 13))
13124            .await
13125            .unwrap();
13126        let error_frame = rx.recv().await.unwrap();
13127        assert_eq!(error_frame.header.ty, FrameType::Error);
13128        assert_eq!(error_frame.header.channel, channel);
13129        let captured = crate::router::test_log::captured_logs(&logs);
13130        assert_eq!(
13131            captured
13132                .lines()
13133                .filter(|line| {
13134                    line.contains("module_id=aft")
13135                        && line.contains("reason=explicit_goodbye")
13136                        && line.contains("module registration ended")
13137                })
13138                .count(),
13139            1,
13140            "unexpected GOODBYE registry log: {captured}"
13141        );
13142    }
13143
13144    #[tokio::test]
13145    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
13146        let registry = Arc::new(Registry::default());
13147        let handler = ControlHandler::new(registry).with_capability_config(
13148            [("prov".to_string(), true), ("cons".to_string(), true)],
13149            BTreeMap::new(),
13150        );
13151        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
13152        register_capability_manifest(
13153            &handler,
13154            &provider_ctx,
13155            &mut provider_rx,
13156            capability_manifest("prov", &["thing/v1"], &[]),
13157            1,
13158        )
13159        .await;
13160        let mut consumer = capability_manifest("cons", &[], &[]);
13161        consumer.capabilities.as_mut().unwrap().requires.push(
13162            subc_protocol::manifest::CapabilityRequirement {
13163                capability: "thing/v1".to_string(),
13164                need: subc_protocol::manifest::CapabilityNeed::Required,
13165            },
13166        );
13167        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
13168        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
13169        assert_eq!(
13170            handler.capability_evaluator.verdict("cons", "thing/v1"),
13171            Some(CapabilityVerdict::Provided)
13172        );
13173        let (mut client_rx, _) = open_route_for_capability_test(
13174            &handler,
13175            &provider_ctx,
13176            &mut provider_rx,
13177            703,
13178            3,
13179            "prov",
13180            None,
13181        )
13182        .await;
13183        let goodbye =
13184            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13185        handler
13186            .handle_control_frame(&provider_ctx, goodbye)
13187            .await
13188            .unwrap();
13189        assert_eq!(
13190            handler.capability_evaluator.verdict("cons", "thing/v1"),
13191            Some(CapabilityVerdict::NeverProvided)
13192        );
13193        let closed = client_rx
13194            .try_recv()
13195            .expect("GOODBYE pushes route.closed before route GOODBYE");
13196        assert!(
13197            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
13198            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
13199        );
13200        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
13201        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
13202    }
13203
13204    struct RouteClosedFixture {
13205        handler: ControlHandler,
13206        #[cfg(unix)]
13207        supervisor: Supervisor,
13208        module_ctx: RouteCtx,
13209        #[cfg(unix)]
13210        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13211        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13212        route: Arc<crate::forwarding::RouteBinding>,
13213    }
13214
13215    async fn route_closed_fixture() -> RouteClosedFixture {
13216        let registry = Arc::new(Registry::default());
13217        let handler = ControlHandler::new(Arc::clone(&registry));
13218        #[cfg(unix)]
13219        let supervisor = Supervisor::new(registry, RestartPolicy::default())
13220            .with_forwarding(Arc::clone(&handler.forwarding));
13221        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13222        register_capability_manifest(
13223            &handler,
13224            &module_ctx,
13225            &mut module_rx,
13226            capability_manifest("prov", &[], &[]),
13227            1,
13228        )
13229        .await;
13230        let (client_rx, bind) = open_route_for_capability_test(
13231            &handler,
13232            &module_ctx,
13233            &mut module_rx,
13234            702,
13235            2,
13236            "prov",
13237            None,
13238        )
13239        .await;
13240        let ModuleControlRequest::RouteBind {
13241            route_channel: channel,
13242            epoch: route_epoch,
13243            ..
13244        } = bind
13245        else {
13246            panic!("expected route.bind");
13247        };
13248        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13249            handler
13250                .forwarding
13251                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13252                .unwrap()
13253        else {
13254            panic!("the acknowledged route must be bound");
13255        };
13256        RouteClosedFixture {
13257            handler,
13258            #[cfg(unix)]
13259            supervisor,
13260            module_ctx,
13261            #[cfg(unix)]
13262            module_rx,
13263            client_rx,
13264            route,
13265        }
13266    }
13267
13268    impl RouteClosedFixture {
13269        #[cfg(unix)]
13270        async fn cut(&mut self) {
13271            let notices = async {
13272                let mut draining = self.module_rx.recv().await.unwrap();
13273                assert!(matches!(
13274                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13275                        &draining.body
13276                    )
13277                    .unwrap(),
13278                    subc_protocol::session::ModuleControlCommand::Draining {
13279                        reason: RouteCloseReason::Restart,
13280                        ..
13281                    }
13282                ));
13283                draining.flushed.take().unwrap().send(()).unwrap();
13284                let mut closing = self.client_rx.recv().await.unwrap();
13285                assert!(matches!(
13286                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13287                    ClientControlPush::RouteClosing {
13288                        reason: RouteCloseReason::Restart,
13289                        ..
13290                    }
13291                ));
13292                closing.flushed.take().unwrap().send(()).unwrap();
13293            };
13294            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13295            result.unwrap();
13296        }
13297
13298        fn disconnect(&mut self) -> ClientControlPush {
13299            self.handler
13300                .cleanup_connection(self.module_ctx.connection_id)
13301                .unwrap();
13302            let closed = self.client_rx.try_recv().unwrap();
13303            assert_eq!(closed.header.ty, FrameType::Push);
13304            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13305            assert!(matches!(
13306                &push,
13307                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13308                    if channels == &[self.route.client_channel]
13309            ));
13310            assert_eq!(
13311                self.client_rx.try_recv().unwrap().header.ty,
13312                FrameType::Goodbye
13313            );
13314            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13315            push
13316        }
13317    }
13318
13319    #[cfg(unix)]
13320    #[tokio::test(start_paused = true)]
13321    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13322        let mut fixture = route_closed_fixture().await;
13323        fixture.cut().await;
13324        assert!(matches!(
13325            fixture.disconnect(),
13326            ClientControlPush::RouteClosed {
13327                reason: RouteCloseReason::Restart,
13328                drained: true,
13329                ..
13330            }
13331        ));
13332    }
13333
13334    #[cfg(unix)]
13335    #[tokio::test(start_paused = true)]
13336    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13337        let mut fixture = route_closed_fixture().await;
13338        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13339        let started = tokio::time::Instant::now();
13340        fixture.cut().await;
13341        assert!(started.elapsed() >= Duration::from_secs(2));
13342        assert_eq!(fixture.route.flow.in_flight(), 1);
13343        assert!(matches!(
13344            fixture.disconnect(),
13345            ClientControlPush::RouteClosed {
13346                reason: RouteCloseReason::Restart,
13347                drained: false,
13348                ..
13349            }
13350        ));
13351    }
13352
13353    #[tokio::test]
13354    async fn module_connection_crash_reports_crash_and_not_drained() {
13355        let mut fixture = route_closed_fixture().await;
13356        // The server calls cleanup_connection on module EOF. Without a planned
13357        // daemon shutdown, that disconnect must remain a crash with no prior
13358        // route.closing notice, even when the route has no in-flight calls.
13359        assert!(matches!(
13360            fixture.disconnect(),
13361            ClientControlPush::RouteClosed {
13362                reason: RouteCloseReason::Crash,
13363                drained: false,
13364                ..
13365            }
13366        ));
13367    }
13368
13369    #[tokio::test]
13370    async fn dropping_router_connection_releases_registration() {
13371        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13372        let registry = Arc::new(Registry::default());
13373        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
13374        let router = Router::with_control_handler(Arc::clone(&control));
13375        let connection = router.begin_connection();
13376        let connection_id = connection.id();
13377        let (ctx, mut rx) = route_ctx(connection_id);
13378
13379        router
13380            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13381            .await
13382            .unwrap();
13383        let response = rx.recv().await.unwrap();
13384        let ack = parse_ack(&response);
13385        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13386        assert!(registry.get_module("aft").unwrap().is_some());
13387
13388        drop(connection);
13389
13390        assert!(registry.get_module("aft").unwrap().is_none());
13391        assert_eq!(registry.active_registration_count().unwrap(), 0);
13392
13393        control
13394            .cleanup_connection(ConnectionId::new(u64::MAX))
13395            .unwrap();
13396        let captured = crate::router::test_log::captured_logs(&logs);
13397        println!("captured registry lifecycle logs:\n{captured}");
13398        let events: Vec<_> = captured
13399            .lines()
13400            .filter(|line| {
13401                line.contains("module registered module_id=aft ")
13402                    || line.contains("module registration ended")
13403            })
13404            .collect();
13405        assert_eq!(
13406            events.len(),
13407            2,
13408            "unexpected registry lifecycle logs: {captured}"
13409        );
13410        assert!(events[0].contains("module registered module_id=aft "));
13411        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13412        assert!(events[1].contains(&format!(
13413            "module_id=aft connection_id={}",
13414            connection_id.get()
13415        )));
13416        assert!(events[1].contains("reason=connection_closed"));
13417        assert!(events[1].contains("module registration ended"));
13418        assert_eq!(
13419            captured
13420                .lines()
13421                .filter(|line| line.contains("module registered module_id=aft "))
13422                .count(),
13423            1,
13424            "legacy registration admission line must appear once: {captured}"
13425        );
13426    }
13427
13428    #[tokio::test]
13429    async fn hello_registration_keeps_legacy_module_registered_line_once() {
13430        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13431        let registry = Arc::new(Registry::default());
13432        let control = ControlHandler::new(Arc::clone(&registry));
13433        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13434        hello_via_sink(
13435            &control,
13436            &ctx,
13437            &mut rx,
13438            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13439        )
13440        .await;
13441
13442        let captured = crate::router::test_log::captured_logs(&logs);
13443        let admissions: Vec<_> = captured
13444            .lines()
13445            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13446            .collect();
13447        assert_eq!(
13448            admissions.len(),
13449            1,
13450            "legacy admission line must remain exactly once: {captured}"
13451        );
13452        assert!(admissions[0].contains("routable_provider=true"));
13453        assert!(admissions[0].contains("connection_id=777"));
13454    }
13455
13456    fn capability_manifest(
13457        module_id: &str,
13458        provides: &[&str],
13459        must_never_reach: &[&str],
13460    ) -> ModuleManifest {
13461        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13462        manifest.capabilities = Some(CapabilityDeclarations {
13463            provides: provides
13464                .iter()
13465                .map(|capability| (*capability).to_string())
13466                .collect(),
13467            requires: Vec::new(),
13468            must_never_reach: must_never_reach
13469                .iter()
13470                .map(|capability| (*capability).to_string())
13471                .collect(),
13472        });
13473        manifest
13474    }
13475
13476    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
13477        Frame::build(
13478            FrameType::Hello,
13479            control_flags(),
13480            0,
13481            0,
13482            corr,
13483            serde_json::to_vec(&ModuleHelloBody {
13484                protocol_ver: manifest.protocol_ver,
13485                manifest,
13486                control_ops: None,
13487                launch_nonce: None,
13488            })
13489            .expect("capability test HELLO serializes"),
13490        )
13491        .expect("capability test HELLO frame builds")
13492    }
13493
13494    fn catalog_update_with_capabilities_frame(
13495        corr: u64,
13496        capabilities: CapabilityDeclarations,
13497    ) -> Frame {
13498        Frame::build(
13499            FrameType::Request,
13500            control_flags(),
13501            0,
13502            0,
13503            corr,
13504            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13505                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
13506                capabilities: Some(capabilities),
13507                ready: None,
13508            })
13509            .expect("capability catalog.update serializes"),
13510        )
13511        .expect("capability catalog.update frame builds")
13512    }
13513
13514    async fn register_capability_manifest(
13515        handler: &ControlHandler,
13516        ctx: &RouteCtx,
13517        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13518        manifest: ModuleManifest,
13519        corr: u64,
13520    ) {
13521        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
13522    }
13523
13524    async fn open_route_for_capability_test(
13525        handler: &ControlHandler,
13526        target_ctx: &RouteCtx,
13527        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13528        client_connection_id: u64,
13529        corr: u64,
13530        target_module_id: &str,
13531        consumer_identity: Option<ConsumerIdentity>,
13532    ) -> (
13533        mpsc::Receiver<crate::router::OutboundFrame>,
13534        ModuleControlRequest,
13535    ) {
13536        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
13537        let route_handler = handler.clone();
13538        let target_module_id = target_module_id.to_string();
13539        let route_task = tokio::spawn(async move {
13540            route_handler
13541                .handle_control_frame(
13542                    &client_ctx,
13543                    route_open_frame_with_admission_facts(
13544                        corr,
13545                        &target_module_id,
13546                        unique_project_root("admission-facts"),
13547                        consumer_identity,
13548                        None,
13549                    ),
13550                )
13551                .await
13552                .expect("capability test route.open succeeds")
13553        });
13554        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
13555            .await
13556            .expect("capability test route.open must reach route.bind")
13557            .expect("target control receiver stays open");
13558        let bind_request: ModuleControlRequest =
13559            serde_json::from_slice(&bind.body).expect("route.bind decodes");
13560        handler
13561            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
13562            .await
13563            .expect("capability test route.bind ACK succeeds");
13564        assert!(route_task.await.expect("route.open task joins").is_empty());
13565        let opened = client_rx
13566            .recv()
13567            .await
13568            .expect("successful route.open publishes a response");
13569        assert!(matches!(
13570            serde_json::from_slice::<ClientControlResponse>(&opened.body),
13571            Ok(ClientControlResponse::RouteOpen { .. })
13572        ));
13573        (client_rx, bind_request)
13574    }
13575
13576    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
13577        assert_eq!(frame.header.ty, FrameType::Push);
13578        assert_eq!(frame.header.channel, 0);
13579        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
13580            .expect("route.closed control push decodes");
13581        let ClientControlPush::RouteClosed { channels, .. } = &push else {
13582            panic!("expected route.closed");
13583        };
13584        assert_eq!(channels.len(), 1, "exactly one violating route closed");
13585        let channels = channels.clone();
13586        assert_eq!(
13587            push,
13588            ClientControlPush::RouteClosed {
13589                module_id: target_module_id.to_string(),
13590                channels,
13591                reason: RouteCloseReason::CapabilityDenied,
13592                drained: false,
13593                abandoned: 0,
13594                excluded_subscriptions: 0,
13595                terminal: Some(false),
13596            }
13597        );
13598    }
13599
13600    #[tokio::test]
13601    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
13602        let registry = Arc::new(Registry::default());
13603        let forwarding = Arc::new(ForwardingTable::default());
13604        let supervisor = SupervisorHandle::new();
13605        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13606        let handler =
13607            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13608                .with_supervisor(supervisor);
13609        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
13610        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
13611        register_capability_manifest(
13612            &handler,
13613            &target_ctx,
13614            &mut target_rx,
13615            capability_manifest("target", &["credentials-provider/v1"], &[]),
13616            1,
13617        )
13618        .await;
13619        register_capability_manifest(
13620            &handler,
13621            &opener_ctx,
13622            &mut opener_rx,
13623            capability_manifest("opener", &[], &["credentials-provider/v1"]),
13624            2,
13625        )
13626        .await;
13627
13628        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
13629        let replies = handler
13630            .handle_control_frame(
13631                &client_ctx,
13632                route_open_frame_with_admission_facts(
13633                    3,
13634                    "target",
13635                    unique_project_root("admission-facts"),
13636                    Some(ConsumerIdentity {
13637                        module_id: "opener".to_string(),
13638                        launch_nonce: "opener-nonce".to_string(),
13639                    }),
13640                    None,
13641                ),
13642            )
13643            .await
13644            .expect("denied route.open returns a typed frame");
13645        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13646        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13647        assert!(
13648            target_rx.try_recv().is_err(),
13649            "forbidden route.open must not relay route.bind"
13650        );
13651    }
13652
13653    #[tokio::test]
13654    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
13655        let registry = Arc::new(Registry::default());
13656        let forwarding = Arc::new(ForwardingTable::default());
13657        let supervisor = SupervisorHandle::new();
13658        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13659        let handler =
13660            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13661                .with_supervisor(supervisor);
13662        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
13663        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
13664        register_capability_manifest(
13665            &handler,
13666            &target_ctx,
13667            &mut target_rx,
13668            capability_manifest("target", &["credentials-provider/v1"], &[]),
13669            1,
13670        )
13671        .await;
13672        register_capability_manifest(
13673            &handler,
13674            &old_opener_ctx,
13675            &mut old_opener_rx,
13676            capability_manifest("opener", &[], &[]),
13677            2,
13678        )
13679        .await;
13680        let (mut client_rx, _) = open_route_for_capability_test(
13681            &handler,
13682            &target_ctx,
13683            &mut target_rx,
13684            712,
13685            3,
13686            "target",
13687            Some(ConsumerIdentity {
13688                module_id: "opener".to_string(),
13689                launch_nonce: "opener-nonce".to_string(),
13690            }),
13691        )
13692        .await;
13693        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13694
13695        handler
13696            .cleanup_connection(old_opener_ctx.connection_id)
13697            .expect("old opener registration cleans up");
13698        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
13699        register_capability_manifest(
13700            &handler,
13701            &new_opener_ctx,
13702            &mut new_opener_rx,
13703            capability_manifest("opener", &[], &["credentials-provider/v1"]),
13704            4,
13705        )
13706        .await;
13707
13708        assert_capability_denied_push(
13709            client_rx
13710                .try_recv()
13711                .expect("HELLO deny addition must emit route.closed")
13712                .frame,
13713            "target",
13714        );
13715        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13716        assert!(matches!(
13717            target_rx.try_recv(),
13718            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13719        ));
13720    }
13721
13722    #[tokio::test]
13723    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
13724        let registry = Arc::new(Registry::default());
13725        let forwarding = Arc::new(ForwardingTable::default());
13726        let supervisor = SupervisorHandle::new();
13727        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13728        let handler =
13729            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13730                .with_supervisor(supervisor);
13731        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
13732        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
13733        register_capability_manifest(
13734            &handler,
13735            &target_ctx,
13736            &mut target_rx,
13737            capability_manifest("target", &[], &[]),
13738            1,
13739        )
13740        .await;
13741        register_capability_manifest(
13742            &handler,
13743            &opener_ctx,
13744            &mut opener_rx,
13745            capability_manifest("opener", &[], &["credentials-provider/v1"]),
13746            2,
13747        )
13748        .await;
13749        let (mut client_rx, _) = open_route_for_capability_test(
13750            &handler,
13751            &target_ctx,
13752            &mut target_rx,
13753            722,
13754            3,
13755            "target",
13756            Some(ConsumerIdentity {
13757                module_id: "opener".to_string(),
13758                launch_nonce: "opener-nonce".to_string(),
13759            }),
13760        )
13761        .await;
13762        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13763
13764        let replies = handler
13765            .handle_control_frame(
13766                &target_ctx,
13767                catalog_update_with_capabilities_frame(
13768                    4,
13769                    CapabilityDeclarations {
13770                        provides: vec!["credentials-provider/v1".to_string()],
13771                        requires: Vec::new(),
13772                        must_never_reach: Vec::new(),
13773                    },
13774                ),
13775            )
13776            .await
13777            .expect("claim catalog.update succeeds");
13778        assert!(matches!(
13779            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
13780            Ok(ModuleControlResponseToModule::CatalogUpdate {})
13781        ));
13782        assert_capability_denied_push(
13783            client_rx
13784                .try_recv()
13785                .expect("claim addition must emit route.closed")
13786                .frame,
13787            "target",
13788        );
13789        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13790        assert!(matches!(
13791            target_rx.try_recv(),
13792            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13793        ));
13794    }
13795
13796    #[tokio::test]
13797    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
13798        let registry = Arc::new(Registry::default());
13799        let forwarding = Arc::new(ForwardingTable::default());
13800        let supervisor = SupervisorHandle::new();
13801        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13802        let handler =
13803            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13804                .with_supervisor(supervisor);
13805        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
13806        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
13807        register_capability_manifest(
13808            &handler,
13809            &target_ctx,
13810            &mut target_rx,
13811            capability_manifest("target", &["credentials-provider/v1"], &[]),
13812            1,
13813        )
13814        .await;
13815        register_capability_manifest(
13816            &handler,
13817            &opener_ctx,
13818            &mut opener_rx,
13819            capability_manifest("opener", &[], &[]),
13820            2,
13821        )
13822        .await;
13823        let (mut client_rx, _) = open_route_for_capability_test(
13824            &handler,
13825            &target_ctx,
13826            &mut target_rx,
13827            732,
13828            3,
13829            "target",
13830            Some(ConsumerIdentity {
13831                module_id: "opener".to_string(),
13832                launch_nonce: "opener-nonce".to_string(),
13833            }),
13834        )
13835        .await;
13836        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13837
13838        handler
13839            .handle_control_frame(
13840                &target_ctx,
13841                catalog_update_with_capabilities_frame(
13842                    4,
13843                    CapabilityDeclarations {
13844                        provides: Vec::new(),
13845                        requires: Vec::new(),
13846                        must_never_reach: Vec::new(),
13847                    },
13848                ),
13849            )
13850            .await
13851            .expect("claim removal catalog.update succeeds");
13852        assert_eq!(
13853            forwarding.active_binding_count().unwrap(),
13854            1,
13855            "removing an attested target claim must leave the route census unchanged"
13856        );
13857        assert!(
13858            client_rx.try_recv().is_err(),
13859            "claim removal must not emit route.closed capability_denied"
13860        );
13861        assert!(
13862            target_rx.try_recv().is_err(),
13863            "claim removal must not send the target a route GOODBYE"
13864        );
13865    }
13866
13867    /// A direct client may open a route to a denied capability provider; this
13868    /// policy applies only to attested supervised module origins, not to direct clients.
13869    #[tokio::test]
13870    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
13871        let registry = Arc::new(Registry::default());
13872        let forwarding = Arc::new(ForwardingTable::default());
13873        let supervisor = SupervisorHandle::new();
13874        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13875        let handler =
13876            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13877                .with_supervisor(supervisor);
13878        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
13879        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
13880        register_capability_manifest(
13881            &handler,
13882            &target_ctx,
13883            &mut target_rx,
13884            capability_manifest("target", &["credentials-provider/v1"], &[]),
13885            1,
13886        )
13887        .await;
13888        register_capability_manifest(
13889            &handler,
13890            &opener_ctx,
13891            &mut opener_rx,
13892            capability_manifest("opener", &[], &["credentials-provider/v1"]),
13893            2,
13894        )
13895        .await;
13896
13897        let (_client_rx, bind) = open_route_for_capability_test(
13898            &handler,
13899            &target_ctx,
13900            &mut target_rx,
13901            742,
13902            3,
13903            "target",
13904            None,
13905        )
13906        .await;
13907        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
13908            panic!("direct scope-honesty route must bind");
13909        };
13910        assert_eq!(principal, Some(Principal::Direct));
13911        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13912    }
13913
13914    /// A module that denies a capability receives no self-route exemption when it
13915    /// also attestedly provides that capability.
13916    #[tokio::test]
13917    async fn must_never_reach_self_route_is_capability_forbidden() {
13918        let registry = Arc::new(Registry::default());
13919        let forwarding = Arc::new(ForwardingTable::default());
13920        let supervisor = SupervisorHandle::new();
13921        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
13922        let handler =
13923            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13924                .with_supervisor(supervisor);
13925        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
13926        register_capability_manifest(
13927            &handler,
13928            &self_ctx,
13929            &mut self_rx,
13930            capability_manifest(
13931                "self-provider",
13932                &["credentials-provider/v1"],
13933                &["credentials-provider/v1"],
13934            ),
13935            1,
13936        )
13937        .await;
13938
13939        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
13940        let replies = handler
13941            .handle_control_frame(
13942                &client_ctx,
13943                route_open_frame_with_admission_facts(
13944                    2,
13945                    "self-provider",
13946                    unique_project_root("admission-facts"),
13947                    Some(ConsumerIdentity {
13948                        module_id: "self-provider".to_string(),
13949                        launch_nonce: "self-nonce".to_string(),
13950                    }),
13951                    None,
13952                ),
13953            )
13954            .await
13955            .expect("self-route refusal returns a typed frame");
13956        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13957        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13958        assert!(
13959            self_rx.try_recv().is_err(),
13960            "self denial must not relay route.bind"
13961        );
13962    }
13963
13964    #[test]
13965    fn unsupported_channel_zero_frame_returns_error() {
13966        let handler = ControlHandler::default();
13967        let request = Frame::build(
13968            FrameType::Request,
13969            control_flags(),
13970            0,
13971            0,
13972            21,
13973            b"opaque".to_vec(),
13974        )
13975        .unwrap();
13976
13977        let response = handler
13978            .handle_control(ConnectionId::new(1), request)
13979            .unwrap();
13980
13981        assert_eq!(response[0].header.ty, FrameType::Error);
13982        assert_eq!(
13983            parse_error(&response[0])["code"],
13984            "unsupported_control_frame"
13985        );
13986    }
13987
13988    /// Blue/green swap at the control-plane boundary. The supervisor that opens
13989    /// a swap is not wired yet, so the candidate is registered here directly
13990    /// into the registry and forwarding candidate slots, the way the swap's
13991    /// HELLO admission will.
13992    mod swap {
13993        use super::*;
13994
13995        const INCUMBENT: ConnectionId = ConnectionId::new(30);
13996        const CANDIDATE: ConnectionId = ConnectionId::new(40);
13997
13998        struct Swap {
13999            registry: Arc<Registry>,
14000            forwarding: Arc<ForwardingTable>,
14001            handler: ControlHandler,
14002            incumbent_ctx: RouteCtx,
14003            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14004            candidate_ctx: RouteCtx,
14005            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14006        }
14007
14008        async fn swap_with_incumbent() -> Swap {
14009            let registry = Arc::new(Registry::default());
14010            let forwarding = Arc::new(ForwardingTable::default());
14011            let handler =
14012                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
14013            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14014            hello_via_sink(
14015                &handler,
14016                &incumbent_ctx,
14017                &mut incumbent_rx,
14018                hello_frame("aft", PROTOCOL_VERSION, 7),
14019            )
14020            .await;
14021            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14022            Swap {
14023                registry,
14024                forwarding,
14025                handler,
14026                incumbent_ctx,
14027                incumbent_rx,
14028                candidate_ctx,
14029                candidate_rx,
14030            }
14031        }
14032
14033        fn register_candidate(swap: &Swap, ready: Option<bool>) {
14034            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14035            candidate_manifest.ready = ready;
14036            let registration = swap
14037                .registry
14038                .register_candidate_with_control_ops(
14039                    candidate_manifest,
14040                    PROTOCOL_VERSION,
14041                    CANDIDATE,
14042                    module_baseline_control_ops(),
14043                )
14044                .unwrap();
14045            swap.forwarding
14046                .register_candidate_module_connection(
14047                    CANDIDATE,
14048                    "aft".to_string(),
14049                    PROTOCOL_VERSION,
14050                    manifest_concurrency(&registration.manifest),
14051                    swap.candidate_ctx.egress.clone(),
14052                )
14053                .unwrap();
14054        }
14055
14056        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
14057            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
14058            swap.registry.promote_candidate("aft").unwrap().unwrap();
14059            cutover.incumbent.unwrap()
14060        }
14061
14062        /// A swap candidate's HELLO reserves its candidate registry entry
14063        /// before installing its candidate forwarding endpoint. The swap
14064        /// supervisor promotes a ready candidate as soon as it reads it as
14065        /// registered, and promotion cuts forwarding over to the candidate
14066        /// endpoint, so the lookup the supervisor reads
14067        /// (`Registry::published_registration`) must not return the candidate
14068        /// until that endpoint exists. Otherwise the cutover finds no candidate
14069        /// endpoint and the swap fails as if the candidate had disconnected.
14070        #[tokio::test]
14071        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
14072            let mut swap = swap_with_incumbent().await;
14073            let hold = Arc::new(std::sync::Barrier::new(2));
14074            let handler = swap
14075                .handler
14076                .clone()
14077                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14078            handler
14079                .supervisor
14080                .open_swap("aft", "candidate-nonce".to_string());
14081            let hello = {
14082                let handler = handler.clone();
14083                let candidate_ctx = swap.candidate_ctx.clone();
14084                std::thread::spawn(move || {
14085                    handler
14086                        .handle_hello(
14087                            candidate_ctx.connection_id,
14088                            Some(candidate_ctx.egress.clone()),
14089                            hello_frame_with_nonce(
14090                                "aft",
14091                                PROTOCOL_VERSION,
14092                                8,
14093                                Some("candidate-nonce"),
14094                            ),
14095                        )
14096                        .unwrap()
14097                })
14098            };
14099
14100            hold.wait();
14101            // Parked: the candidate slot is claimed but not published, and
14102            // forwarding has no candidate endpoint.
14103            assert!(swap
14104                .registry
14105                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
14106                .unwrap()
14107                .is_some());
14108            assert!(swap
14109                .registry
14110                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14111                .unwrap()
14112                .is_none());
14113            assert!(swap
14114                .forwarding
14115                .module_endpoint_for_connection(CANDIDATE)
14116                .unwrap()
14117                .is_none());
14118
14119            hold.wait();
14120            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
14121            assert_eq!(
14122                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
14123                FrameType::HelloAck
14124            );
14125            let published = swap
14126                .registry
14127                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14128                .unwrap()
14129                .expect("candidate is published once its HELLO completes");
14130            assert_eq!(published.connection_id, CANDIDATE);
14131            assert!(swap
14132                .forwarding
14133                .module_endpoint_for_connection(CANDIDATE)
14134                .unwrap()
14135                .is_some());
14136        }
14137
14138        fn keyed_total(counters: &Value, key: &str) -> u64 {
14139            counters[key]
14140                .as_object()
14141                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
14142                .unwrap_or(0)
14143        }
14144
14145        #[tokio::test]
14146        async fn replacement_logs_old_end_and_new_admission_once() {
14147            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14148            let mut swap = swap_with_incumbent().await;
14149            swap.handler
14150                .supervisor
14151                .open_swap("aft", "candidate-nonce".to_string());
14152            hello_via_sink(
14153                &swap.handler,
14154                &swap.candidate_ctx,
14155                &mut swap.candidate_rx,
14156                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
14157            )
14158            .await;
14159            cutover(&swap);
14160            swap.handler.cleanup_connection(INCUMBENT).unwrap();
14161
14162            let captured = crate::router::test_log::captured_logs(&logs);
14163            println!("captured replacement registry logs:\n{captured}");
14164            let new_admissions: Vec<_> = captured
14165                .lines()
14166                .filter(|line| {
14167                    line.contains("connection_id=40")
14168                        && line.contains("swap candidate registered; not routable until cutover")
14169                })
14170                .collect();
14171            assert_eq!(
14172                new_admissions.len(),
14173                1,
14174                "unexpected admission logs: {captured}"
14175            );
14176            assert!(new_admissions[0].contains("module_id=aft"));
14177            assert!(new_admissions[0].contains("connection_id=40"));
14178            assert!(
14179                new_admissions[0].contains("swap candidate registered; not routable until cutover")
14180            );
14181            assert!(new_admissions[0].contains("ready=true"));
14182
14183            let old_admissions: Vec<_> = captured
14184                .lines()
14185                .filter(|line| {
14186                    line.contains("module registered module_id=aft ")
14187                        && line.contains("connection_id=30")
14188                })
14189                .collect();
14190            assert_eq!(
14191                old_admissions.len(),
14192                1,
14193                "unexpected incumbent admission: {captured}"
14194            );
14195
14196            let old_ends: Vec<_> = captured
14197                .lines()
14198                .filter(|line| {
14199                    line.contains("connection_id=30") && line.contains("module registration ended")
14200                })
14201                .collect();
14202            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
14203            assert!(old_ends[0].contains("module_id=aft"));
14204            assert!(old_ends[0].contains("connection_id=30"));
14205            assert!(old_ends[0].contains("reason=replaced"));
14206            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
14207            assert_eq!(
14208                captured
14209                    .lines()
14210                    .filter(|line| line.contains("module registration promoted"))
14211                    .count(),
14212                1,
14213                "unexpected promotion logs: {captured}"
14214            );
14215        }
14216
14217        /// An ack from the incumbent for a bind it was sent before cutover,
14218        /// arriving before the incumbent is drained. The incumbent is the live
14219        /// connection carrying every other client's routes, so the ack must
14220        /// not end it: the waiting client is told to retry, the reservation is
14221        /// given back, and the incumbent is told to drop just that binding.
14222        #[tokio::test]
14223        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14224            let mut swap = swap_with_incumbent().await;
14225            let handler = swap.handler.clone();
14226
14227            // A co-tenant route, bound on the incumbent before the swap.
14228            let cotenant = ConnectionId::new(31);
14229            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14230            let (cotenant_task, cotenant_bind) = relay_route_open(
14231                &handler,
14232                cotenant,
14233                &cotenant_ctx.egress,
14234                &mut swap.incumbent_rx,
14235                100,
14236                "aft",
14237                "swap-cotenant",
14238            )
14239            .await;
14240            handler
14241                .handle_control_frame(
14242                    &swap.incumbent_ctx,
14243                    route_bind_ack(cotenant_bind.header.corr),
14244                )
14245                .await
14246                .unwrap();
14247            assert!(cotenant_task.await.unwrap().is_empty());
14248            let (cotenant_channel, cotenant_epoch) =
14249                published_route(&cotenant_rx.recv().await.unwrap());
14250
14251            // A second route.open, relayed to the incumbent and not yet acked.
14252            let caller = ConnectionId::new(32);
14253            let (caller_ctx, mut caller_rx) = route_ctx(caller);
14254            let (caller_task, caller_bind) = relay_route_open(
14255                &handler,
14256                caller,
14257                &caller_ctx.egress,
14258                &mut swap.incumbent_rx,
14259                101,
14260                "aft",
14261                "swap-caller",
14262            )
14263            .await;
14264            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14265
14266            register_candidate(&swap, None);
14267            cutover(&swap);
14268
14269            // The incumbent acks after promotion and before any drain.
14270            let ack = handler
14271                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14272                .await;
14273            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14274            if module_loop_error.is_some() {
14275                // What the connection loop does with an untranslated router
14276                // error: end the connection, releasing every route on it.
14277                handler.cleanup_connection(INCUMBENT).unwrap();
14278            }
14279
14280            // 1. The incumbent's other routes survive.
14281            assert!(
14282                cotenant_rx.try_recv().is_err(),
14283                "the co-tenant route on the incumbent was torn down by one late ack: \
14284                 {module_loop_error:?}"
14285            );
14286            assert!(matches!(
14287                swap.forwarding
14288                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14289                    .unwrap(),
14290                DataRoute::Client(DataRouteState::Bound(_))
14291            ));
14292            assert_eq!(module_loop_error, None);
14293            assert!(swap
14294                .registry
14295                .get_module_by_connection(INCUMBENT)
14296                .unwrap()
14297                .is_some());
14298
14299            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
14300            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14301                .await
14302                .expect("the incumbent is told to drop the abandoned binding")
14303                .unwrap()
14304                .frame;
14305            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14306            assert_eq!(goodbye.header.channel, abandoned_channel);
14307            assert_eq!(goodbye.header.epoch, abandoned_epoch);
14308            assert!(swap.incumbent_rx.try_recv().is_err());
14309
14310            // 3. The waiting client gets a retryable refusal and no route.
14311            let response = caller_task.await.unwrap();
14312            assert_eq!(response.len(), 1);
14313            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14314            assert!(caller_rx.try_recv().is_err());
14315
14316            // 4. The reservation pair is given back, and the pending bind
14317            //    settled exactly once: one accepted open (the co-tenant) and one
14318            //    refused open (the caller), nothing counted twice.
14319            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14320            let counters = handler.counters().snapshot();
14321            assert_eq!(
14322                keyed_total(&counters, "route_open_accepted_by_principal"),
14323                1
14324            );
14325            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14326            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14327        }
14328
14329        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
14330        /// id would resolve to the promoted candidate and every new route.open
14331        /// would be refused as reloading, leaving neither process routable.
14332        #[tokio::test]
14333        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14334            let mut swap = swap_with_incumbent().await;
14335            register_candidate(&swap, None);
14336            let incumbent = cutover(&swap);
14337            swap.forwarding
14338                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14339                .unwrap()
14340                .expect("the incumbent is still registered");
14341
14342            let client = ConnectionId::new(33);
14343            let (client_ctx, mut client_rx) = route_ctx(client);
14344            let route_handler = swap.handler.clone();
14345            let open_ctx = RouteCtx {
14346                connection_id: client,
14347                egress: client_ctx.egress.clone(),
14348            };
14349            let mut route_task = tokio::spawn(async move {
14350                route_handler
14351                    .handle_control_frame(
14352                        &open_ctx,
14353                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14354                    )
14355                    .await
14356                    .unwrap()
14357            });
14358            let bind = tokio::select! {
14359                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14360                response = &mut route_task => {
14361                    let response = response.unwrap();
14362                    panic!(
14363                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
14364                        parse_error(&response[0])["code"]
14365                    );
14366                }
14367            };
14368            swap.handler
14369                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14370                .await
14371                .unwrap();
14372            assert!(route_task.await.unwrap().is_empty());
14373            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14374            match swap
14375                .forwarding
14376                .lookup_data_route(client, channel, epoch)
14377                .unwrap()
14378            {
14379                DataRoute::Client(DataRouteState::Bound(route)) => {
14380                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14381                }
14382                other => panic!("expected a bound route on the candidate, got {other:?}"),
14383            }
14384            assert!(swap.incumbent_rx.try_recv().is_err());
14385        }
14386
14387        /// A candidate declares itself ready with `catalog.update` on its own
14388        /// connection. If the connection-keyed registry lookups searched only the
14389        /// active slot, this would answer `not_registered` and the candidate
14390        /// would never become ready.
14391        #[tokio::test]
14392        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14393            let swap = swap_with_incumbent().await;
14394            register_candidate(&swap, Some(false));
14395            let update = Frame::build(
14396                FrameType::Request,
14397                control_flags(),
14398                0,
14399                0,
14400                55,
14401                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14402                    provides: manifest("aft", PROTOCOL_VERSION).provides,
14403                    capabilities: None,
14404                    ready: Some(true),
14405                })
14406                .unwrap(),
14407            )
14408            .unwrap();
14409
14410            let replies = swap
14411                .handler
14412                .handle_control_frame(&swap.candidate_ctx, update)
14413                .await
14414                .unwrap();
14415
14416            assert_eq!(replies.len(), 1);
14417            assert_eq!(
14418                replies[0].header.ty,
14419                FrameType::Response,
14420                "candidate catalog.update was refused: {:?}",
14421                serde_json::from_slice::<Value>(&replies[0].body).ok()
14422            );
14423            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14424            assert_eq!(
14425                swap.registry
14426                    .get_module("aft")
14427                    .unwrap()
14428                    .unwrap()
14429                    .connection_id,
14430                INCUMBENT
14431            );
14432        }
14433    }
14434
14435    /// The HELLO gate while the supervisor has a swap open: only the nonce it
14436    /// minted for the candidate admits a second process, into the candidate
14437    /// slot, and that check runs ahead of the reserved-module gate.
14438    mod swap_admission {
14439        use super::*;
14440
14441        const INCUMBENT_NONCE: &str = "incumbent-nonce";
14442        const CANDIDATE_NONCE: &str = "candidate-nonce";
14443
14444        fn handler_with_incumbent(
14445            module_id: &str,
14446            reserved: bool,
14447        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14448            let registry = Arc::new(Registry::default());
14449            let supervisor = SupervisorHandle::new();
14450            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14451            if reserved {
14452                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14453            }
14454            let handler =
14455                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
14456            let incumbent = handler
14457                .handle_control(
14458                    ConnectionId::new(1),
14459                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14460                )
14461                .unwrap();
14462            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14463            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14464            (registry, supervisor, handler)
14465        }
14466
14467        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
14468        /// admits every nonce, so while a swap is open the swap gate is the only
14469        /// thing between a key-holder and the candidate slot. A nonce the
14470        /// supervisor did not mint, or none at all, is refused, and neither the
14471        /// incumbent's registration nor the candidate slot moves.
14472        #[test]
14473        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
14474            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
14475
14476            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
14477                let replies = handler
14478                    .handle_control(
14479                        ConnectionId::new(connection),
14480                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
14481                    )
14482                    .unwrap();
14483                assert_eq!(replies[0].header.ty, FrameType::Error);
14484                assert_eq!(
14485                    parse_error(&replies[0])["code"],
14486                    "swap_token_invalid",
14487                    "nonce {nonce:?}"
14488                );
14489            }
14490            assert!(registry.get_candidate("aft").unwrap().is_none());
14491            assert_eq!(
14492                registry.get_module("aft").unwrap().unwrap().connection_id,
14493                ConnectionId::new(1)
14494            );
14495
14496            // Control: the minted token is admitted, into the candidate slot,
14497            // and only once.
14498            let admitted = handler
14499                .handle_control(
14500                    ConnectionId::new(4),
14501                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
14502                )
14503                .unwrap();
14504            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
14505            assert_eq!(
14506                registry
14507                    .get_candidate("aft")
14508                    .unwrap()
14509                    .unwrap()
14510                    .connection_id,
14511                ConnectionId::new(4)
14512            );
14513            assert_eq!(
14514                registry.get_module("aft").unwrap().unwrap().connection_id,
14515                ConnectionId::new(1),
14516                "the candidate must not take the active slot"
14517            );
14518            let replayed = handler
14519                .handle_control(
14520                    ConnectionId::new(5),
14521                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
14522                )
14523                .unwrap();
14524            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
14525
14526            // The case only this gate covers: the incumbent has died mid-swap,
14527            // so its duplicate refusal is gone too, and without the gate a
14528            // key-holder would take the id's ACTIVE slot.
14529            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
14530            let squatter = handler
14531                .handle_control(
14532                    ConnectionId::new(6),
14533                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
14534                )
14535                .unwrap();
14536            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
14537            assert!(
14538                registry.get_module("aft").unwrap().is_none(),
14539                "a squatter took the active slot of an id being swapped"
14540            );
14541        }
14542
14543        /// Design mutation arm (iii). A reserved module's candidate presents a
14544        /// nonce the reserved gate has never seen (that gate holds the
14545        /// incumbent's), so the swap gate must run first or the candidate is
14546        /// refused `reserved_module` and a reserved module can never be swapped.
14547        #[test]
14548        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
14549            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
14550
14551            let replies = handler
14552                .handle_control(
14553                    ConnectionId::new(2),
14554                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14555                )
14556                .unwrap();
14557
14558            assert_eq!(
14559                replies[0].header.ty,
14560                FrameType::HelloAck,
14561                "reserved candidate refused: {:?}",
14562                serde_json::from_slice::<Value>(&replies[0].body).ok()
14563            );
14564            assert_eq!(
14565                registry
14566                    .get_candidate("vault")
14567                    .unwrap()
14568                    .unwrap()
14569                    .connection_id,
14570                ConnectionId::new(2)
14571            );
14572        }
14573
14574        /// With no swap open the gate is inert: the incumbent's reserved gate
14575        /// and duplicate refusal behave exactly as before.
14576        #[test]
14577        fn without_an_open_swap_the_ordinary_gates_decide() {
14578            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
14579            supervisor.close_swap("vault");
14580
14581            let candidate = handler
14582                .handle_control(
14583                    ConnectionId::new(2),
14584                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14585                )
14586                .unwrap();
14587            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
14588            let duplicate = handler
14589                .handle_control(
14590                    ConnectionId::new(3),
14591                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
14592                )
14593                .unwrap();
14594            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14595            assert!(registry.get_candidate("vault").unwrap().is_none());
14596        }
14597    }
14598
14599    /// `scope.sync` and `scope.describe` through the real control handler: who
14600    /// may sync is decided by the registration and launch nonce of the module
14601    /// connection, never by the request body.
14602    mod scopes {
14603        mod idle_tests {
14604            use super::*;
14605            include!("scope_idle_tests.rs");
14606        }
14607        mod route_open_timings {
14608            use super::*;
14609            include!("route_open_timing_tests.rs");
14610        }
14611        mod activity {
14612            include!("activity_scope_tests.rs");
14613        }
14614        mod delta {
14615            include!("scope_handler_delta_tests.rs");
14616        }
14617        mod call_modes {
14618            include!("scope_call_mode_tests.rs");
14619        }
14620
14621        use subc_protocol::scope::{
14622            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
14623            ScopeStatus,
14624        };
14625
14626        use super::*;
14627
14628        const OWNER: &str = "prefrontal-core";
14629
14630        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
14631            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
14632        }
14633
14634        async fn call(
14635            handler: &ControlHandler,
14636            ctx: &RouteCtx,
14637            request: &ModuleControlRequestFromModule,
14638        ) -> Frame {
14639            let body = serde_json::to_vec(request).unwrap();
14640            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
14641            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
14642            assert_eq!(replies.len(), 1, "{replies:?}");
14643            replies.pop().unwrap()
14644        }
14645
14646        async fn sync(
14647            handler: &ControlHandler,
14648            ctx: &RouteCtx,
14649            generation: u64,
14650            scopes: Vec<ScopeRecord>,
14651        ) -> Result<ModuleControlResponseToModule, String> {
14652            let reply = call(
14653                handler,
14654                ctx,
14655                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
14656            )
14657            .await;
14658            match reply.header.ty {
14659                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
14660                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
14661            }
14662        }
14663
14664        async fn describe(
14665            handler: &ControlHandler,
14666            ctx: &RouteCtx,
14667            owner: &str,
14668            scope_ref: &str,
14669        ) -> ModuleControlResponseToModule {
14670            let reply = call(
14671                handler,
14672                ctx,
14673                &ModuleControlRequestFromModule::ScopeDescribe {
14674                    owner: Principal::Reserved {
14675                        module_id: owner.to_string(),
14676                    },
14677                    scope_ref: scope_ref.to_string(),
14678                },
14679            )
14680            .await;
14681            assert_eq!(
14682                reply.header.ty,
14683                FrameType::Response,
14684                "{:?}",
14685                parse_error(&reply)
14686            );
14687            serde_json::from_slice(&reply.body).unwrap()
14688        }
14689
14690        #[tokio::test]
14691        async fn scope_describe_never_echoes_bind_proof_or_hash() {
14692            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
14693            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
14694            let mut rig = rig().await;
14695            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
14696                .await;
14697            let _bound = rig
14698                .bound(
14699                    Some(AFT),
14700                    PLEXUS,
14701                    Some(
14702                        rig.selector("s", Some(1))
14703                            .with_bind_proof(Some(PROOF.into())),
14704                    ),
14705                )
14706                .await;
14707            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
14708            let output = serde_json::to_string(&reply).unwrap();
14709            assert!(output.contains("\"status\":\"live\""));
14710            assert!(!output.contains(PROOF));
14711            assert!(!output.contains(HASH));
14712        }
14713
14714        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
14715        async fn module(
14716            handler: &ControlHandler,
14717            connection: u64,
14718            module_id: &str,
14719            nonce: Option<&str>,
14720        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14721            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
14722            hello_via_sink(
14723                handler,
14724                &ctx,
14725                &mut rx,
14726                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
14727            )
14728            .await;
14729            (ctx, rx)
14730        }
14731
14732        /// `direct` and every other client connection has no registration, so
14733        /// it can neither sync nor own a scope.
14734        #[tokio::test]
14735        async fn a_client_connection_cannot_sync_or_describe() {
14736            let handler = ControlHandler::new(Arc::new(Registry::default()));
14737            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
14738            for request in [
14739                ModuleControlRequestFromModule::ScopeSync {
14740                    generation: 1,
14741                    scopes: vec![head("s", 1)],
14742                },
14743                ModuleControlRequestFromModule::ScopeDescribe {
14744                    owner: Principal::Direct,
14745                    scope_ref: "s".to_string(),
14746                },
14747            ] {
14748                let reply = call(&handler, &ctx, &request).await;
14749                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
14750            }
14751            assert!(
14752                !handler
14753                    .scopes
14754                    .read()
14755                    .unwrap()
14756                    .describe(
14757                        &Principal::Reserved {
14758                            module_id: OWNER.to_string()
14759                        },
14760                        "s"
14761                    )
14762                    .owner_synced
14763            );
14764        }
14765
14766        /// A module the supervisor did not spawn registers without a launch
14767        /// nonce, so it is never an owner's current launch.
14768        #[tokio::test]
14769        async fn a_module_without_a_supervised_launch_cannot_sync() {
14770            let handler = ControlHandler::new(Arc::new(Registry::default()));
14771            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
14772            assert_eq!(
14773                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
14774                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14775            );
14776        }
14777
14778        #[tokio::test]
14779        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
14780            let supervisor = SupervisorHandle::new();
14781            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14782            let handler = ControlHandler::new(Arc::new(Registry::default()))
14783                .with_supervisor(supervisor.clone());
14784            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14785            sync(&handler, &incumbent, 1, vec![head("s", 1)])
14786                .await
14787                .expect("the current launch syncs");
14788
14789            // A swap candidate registers with the swap token and is refused
14790            // while the incumbent keeps syncing.
14791            supervisor.open_swap(OWNER, "n2".to_string());
14792            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
14793            assert_eq!(
14794                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14795                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14796            );
14797            sync(&handler, &incumbent, 2, vec![head("s", 1)])
14798                .await
14799                .expect("the serving owner syncs during the swap");
14800
14801            // The swap fails and is rolled back. The candidate never held sync
14802            // authority, and still cannot sync.
14803            supervisor.close_swap(OWNER);
14804            assert_eq!(
14805                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14806                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14807            );
14808            sync(&handler, &incumbent, 3, vec![head("s", 1)])
14809                .await
14810                .expect("the serving owner syncs after the rollback");
14811            handler.cleanup_connection(candidate.connection_id).unwrap();
14812
14813            // A swap that cuts over. Promotion records the candidate's nonce as
14814            // the module's spawn nonce, which is what `set_spawn_nonce` does
14815            // here; the promoted connection then takes authority at any
14816            // generation and the superseded incumbent is refused.
14817            supervisor.open_swap(OWNER, "n3".to_string());
14818            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
14819            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
14820            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
14821                .await
14822                .expect("the promoted launch takes authority");
14823            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14824                panic!("unexpected reply {reply:?}");
14825            };
14826            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
14827            assert_eq!(
14828                sync(&handler, &incumbent, 4, Vec::new()).await,
14829                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14830            );
14831        }
14832
14833        /// Authority dies with its connection: the cleanup path releases it,
14834        /// so the owner's next connection takes it at any generation.
14835        #[tokio::test]
14836        async fn closing_the_authority_connection_frees_sync_authority() {
14837            let supervisor = SupervisorHandle::new();
14838            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14839            let handler = ControlHandler::new(Arc::new(Registry::default()))
14840                .with_supervisor(supervisor.clone());
14841            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14842            sync(&handler, &first, 10, vec![head("s", 1)])
14843                .await
14844                .unwrap();
14845            handler.cleanup_connection(first.connection_id).unwrap();
14846
14847            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
14848            sync(&handler, &second, 1, vec![head("s", 1)])
14849                .await
14850                .expect("the next connection takes the released authority");
14851        }
14852
14853        #[tokio::test]
14854        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
14855            let supervisor = SupervisorHandle::new();
14856            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14857            let handler =
14858                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
14859            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
14860            sync(&handler, &first, 10, vec![head("s", 1)])
14861                .await
14862                .unwrap();
14863            handler
14864                .handle_control_frame(
14865                    &first,
14866                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
14867                )
14868                .await
14869                .unwrap();
14870            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
14871            sync(&handler, &second, 1, vec![head("s", 1)])
14872                .await
14873                .expect("GOODBYE releases authority even if the old socket remains open");
14874        }
14875
14876        #[tokio::test]
14877        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
14878            let registry = Arc::new(Registry::default());
14879            let supervisor_handle = SupervisorHandle::new();
14880            let supervisor =
14881                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
14882                    .with_handle(supervisor_handle.clone())
14883                    .with_daemon_incarnation("incarnation-7".to_string());
14884            // Configured with enabled: false, so the supervisor lists the
14885            // module without spawning a process for it.
14886            supervisor
14887                .supervise_configured(
14888                    ModuleSpec {
14889                        module_id: OWNER.to_string(),
14890                        program: PathBuf::from("/nonexistent/prefrontal-core"),
14891                        args: Vec::new(),
14892                        env: Vec::new(),
14893                        reserved: false,
14894                        reserved_prefixes: Vec::new(),
14895                        protocol: ModuleProtocol::Subc,
14896                        overlap: Default::default(),
14897                    },
14898                    false,
14899                )
14900                .unwrap();
14901            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
14902            let handler =
14903                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
14904            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
14905
14906            // Configured but not yet synced: a reader waits for the owner.
14907            let ModuleControlResponseToModule::ScopeDescribe {
14908                status,
14909                daemon_incarnation,
14910                owner_synced,
14911                owner_configured,
14912                scope,
14913                ..
14914            } = describe(&handler, &reader, OWNER, "s").await
14915            else {
14916                panic!("not a describe reply");
14917            };
14918            assert_eq!(status, ScopeStatus::NotLive);
14919            assert_eq!(daemon_incarnation, "incarnation-7");
14920            assert!(!owner_synced);
14921            assert!(owner_configured);
14922            assert!(scope.is_none());
14923
14924            // Not a supervised module: the owner will never sync, and a reader
14925            // refuses rather than waits.
14926            let ModuleControlResponseToModule::ScopeDescribe {
14927                status,
14928                owner_configured,
14929                ..
14930            } = describe(&handler, &reader, "ghost", "s").await
14931            else {
14932                panic!("not a describe reply");
14933            };
14934            assert_eq!(status, ScopeStatus::NotLive);
14935            assert!(!owner_configured);
14936
14937            // Live, with the stamp fields and the computed owner_authorized.
14938            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
14939            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
14940            let ModuleControlResponseToModule::ScopeDescribe {
14941                status,
14942                scope_epoch,
14943                owner_synced,
14944                scope,
14945                ..
14946            } = describe(&handler, &reader, OWNER, "s").await
14947            else {
14948                panic!("not a describe reply");
14949            };
14950            assert_eq!(status, ScopeStatus::Live);
14951            assert_eq!(scope_epoch, Some(4));
14952            assert!(owner_synced);
14953            let stamp = scope.expect("a live scope carries its stamp");
14954            assert!(
14955                stamp.owner_authorized,
14956                "prefrontal-core is the default authority"
14957            );
14958            assert_eq!(stamp.kind, ScopeKind::Head);
14959        }
14960
14961        #[tokio::test]
14962        async fn scope_authority_owners_decides_owner_authorized() {
14963            let supervisor = SupervisorHandle::new();
14964            supervisor.set_spawn_nonce("broca", "b1".to_string());
14965            let handler = ControlHandler::new(Arc::new(Registry::default()))
14966                .with_supervisor(supervisor)
14967                .with_scope_authority_owners(vec!["broca".to_string()]);
14968            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
14969            let mut gated = head("s", 1);
14970            gated.attributes.agent_id = Some("agent".to_string());
14971            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
14972            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
14973                describe(&handler, &broca, "broca", "s").await
14974            else {
14975                panic!("not a describe reply");
14976            };
14977            assert!(scope.unwrap().owner_authorized);
14978        }
14979
14980        /// With route admission, the stamp, the commit re-check and drains in
14981        /// place, the feature is advertised: the module ops in HELLO_ACK, and
14982        /// `scopes/v1` in HELLO_ACK and `server.describe`.
14983        #[tokio::test]
14984        async fn scope_ops_and_the_scopes_capability_are_advertised() {
14985            let handler = ControlHandler::new(Arc::new(Registry::default()));
14986            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
14987            let ack = hello_via_sink(
14988                &handler,
14989                &ctx,
14990                &mut rx,
14991                hello_frame("m", PROTOCOL_VERSION, 1),
14992            )
14993            .await;
14994            let ack = parse_ack(&ack);
14995            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
14996                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
14997            }
14998            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
14999
15000            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
15001            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15002            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15003            let reply = handler
15004                .handle_control_frame(&client, frame)
15005                .await
15006                .unwrap()
15007                .pop()
15008                .unwrap();
15009            let ClientControlResponse::ServerDescribe { capabilities, .. } =
15010                serde_json::from_slice(&reply.body).unwrap()
15011            else {
15012                panic!("not a server.describe reply");
15013            };
15014            assert!(
15015                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
15016                "{capabilities:?}"
15017            );
15018        }
15019
15020        // ---- route admission, stamps, commit re-check and drains ----------
15021
15022        const PLEXUS: &str = "plexus";
15023        const OTHER: &str = "other";
15024        const AFT: &str = "aft";
15025        const BROCA: &str = "broca";
15026        const MAGIC: &str = "magic-context";
15027
15028        fn nonce(module_id: &str) -> String {
15029            format!("nonce-{module_id}")
15030        }
15031
15032        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15033            let (tx, rx) = mpsc::channel(64);
15034            (
15035                RouteCtx {
15036                    connection_id: ConnectionId::new(connection),
15037                    egress: FrameSink::new(tx),
15038                },
15039                rx,
15040            )
15041        }
15042
15043        /// A daemon with a configured owner (prefrontal-core) registered on its
15044        /// own module connection, two routable targets (plexus, other), and
15045        /// launch nonces minted for the modules that open routes as carriers.
15046        struct Rig {
15047            handler: ControlHandler,
15048            forwarding: Arc<ForwardingTable>,
15049            owner: RouteCtx,
15050            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15051            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
15052            generation: u64,
15053            next_connection: u64,
15054            _supervisor: Supervisor,
15055        }
15056
15057        async fn rig() -> Rig {
15058            rig_with_flow_support(true).await
15059        }
15060
15061        async fn rig_with_flow_support(flow_support: bool) -> Rig {
15062            rig_with_scope_capabilities(flow_support, false).await
15063        }
15064
15065        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
15066            let mut provides = Vec::new();
15067            if flow_support {
15068                provides.push("flow-scopes/v1");
15069            }
15070            if run_support {
15071                provides.push("agent-run-scopes/v1");
15072            }
15073            rig_with_target_capabilities(&provides).await
15074        }
15075
15076        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
15077            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
15078            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15079            // Decoder versions alone cannot admit scoped calls: only a capability
15080            // declaration promises the required run or flow behaviour.
15081            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15082            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
15083            Frame::build(
15084                FrameType::Hello,
15085                control_flags(),
15086                0,
15087                0,
15088                connection,
15089                serde_json::to_vec(&body).unwrap(),
15090            )
15091            .unwrap()
15092        }
15093
15094        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
15095            let registry = Arc::new(Registry::default());
15096            let forwarding = Arc::new(ForwardingTable::default());
15097            let supervisor_handle = SupervisorHandle::new();
15098            let supervisor =
15099                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
15100                    .with_handle(supervisor_handle.clone());
15101            supervisor
15102                .supervise_configured(
15103                    ModuleSpec {
15104                        module_id: OWNER.to_string(),
15105                        program: PathBuf::from("/nonexistent/prefrontal-core"),
15106                        args: Vec::new(),
15107                        env: Vec::new(),
15108                        reserved: false,
15109                        reserved_prefixes: Vec::new(),
15110                        protocol: ModuleProtocol::Subc,
15111                        overlap: Default::default(),
15112                    },
15113                    false,
15114                )
15115                .unwrap();
15116            for module_id in [OWNER, AFT, BROCA, MAGIC] {
15117                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
15118            }
15119            let handler =
15120                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15121                    .with_supervisor(supervisor_handle);
15122            let (owner, mut owner_rx) = wide_ctx(1);
15123            hello_via_sink(
15124                &handler,
15125                &owner,
15126                &mut owner_rx,
15127                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
15128            )
15129            .await;
15130            let mut modules = BTreeMap::new();
15131            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
15132                let (ctx, mut rx) = wide_ctx(connection);
15133                let hello = hello_with_capabilities(module_id, connection, provides);
15134                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
15135                modules.insert(module_id.to_string(), (ctx, rx));
15136            }
15137            Rig {
15138                handler,
15139                forwarding,
15140                owner,
15141                _owner_rx: owner_rx,
15142                modules,
15143                generation: 0,
15144                next_connection: 100,
15145                _supervisor: supervisor,
15146            }
15147        }
15148
15149        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
15150            ScopeCarrier::new(Principal::Reserved {
15151                module_id: module_id.to_string(),
15152            })
15153            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
15154        }
15155
15156        /// The scope most tests open under: aft carries to any module, broca
15157        /// only to plexus and other, and the owner delegates as agent-1.
15158        fn session(scope_epoch: u64) -> ScopeRecord {
15159            let mut record = head("s", scope_epoch);
15160            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
15161            record.attributes.agent_id = Some("agent-1".to_string());
15162            record.attributes.delegates = true;
15163            record
15164        }
15165
15166        impl Rig {
15167            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
15168                self.generation += 1;
15169                sync(&self.handler, &self.owner, self.generation, scopes)
15170                    .await
15171                    .expect("the owner's sync is accepted");
15172            }
15173
15174            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15175                ScopeSelector::new(
15176                    Principal::Reserved {
15177                        module_id: OWNER.to_string(),
15178                    },
15179                    scope_ref.to_string(),
15180                    scope_epoch,
15181                )
15182            }
15183
15184            fn open_frame(
15185                &mut self,
15186                opener: Option<&str>,
15187                target: &str,
15188                scope: Option<ScopeSelector>,
15189            ) -> (
15190                RouteCtx,
15191                mpsc::Receiver<crate::router::OutboundFrame>,
15192                Frame,
15193            ) {
15194                self.next_connection += 1;
15195                let (ctx, rx) = wide_ctx(self.next_connection);
15196                let root = unique_project_root("scoped-open");
15197                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15198                    target: RouteTarget::ToolProvider {
15199                        module_id: target.to_string(),
15200                    },
15201                    identity: BindIdentity::new(
15202                        root.path().to_path_buf(),
15203                        "unit".to_string(),
15204                        "session".to_string(),
15205                    ),
15206                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
15207                        module_id: module_id.to_string(),
15208                        launch_nonce: nonce(module_id),
15209                    }),
15210                    consumer_capabilities: None,
15211                    role_versions: None,
15212                    admission_facts: None,
15213                    scope,
15214                })
15215                .unwrap();
15216                let frame = Frame::build(
15217                    FrameType::Request,
15218                    control_flags(),
15219                    0,
15220                    0,
15221                    self.next_connection,
15222                    body,
15223                )
15224                .unwrap();
15225                (ctx, rx, frame)
15226            }
15227
15228            /// Open and expect a refusal before anything is relayed.
15229            async fn refused(
15230                &mut self,
15231                opener: Option<&str>,
15232                target: &str,
15233                scope: Option<ScopeSelector>,
15234            ) -> String {
15235                self.refusal_body(opener, target, scope).await["code"]
15236                    .as_str()
15237                    .unwrap()
15238                    .to_string()
15239            }
15240
15241            async fn refusal_body(
15242                &mut self,
15243                opener: Option<&str>,
15244                target: &str,
15245                scope: Option<ScopeSelector>,
15246            ) -> Value {
15247                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15248                let replies = tokio::time::timeout(
15249                    Duration::from_secs(2),
15250                    self.handler.handle_control_frame(&ctx, frame),
15251                )
15252                .await
15253                .expect("the open must be refused before waiting for a bind ack")
15254                .unwrap();
15255                assert_eq!(replies.len(), 1, "{replies:?}");
15256                assert_eq!(replies[0].header.ty, FrameType::Error);
15257                let (_, module_rx) = self.modules.get_mut(target).unwrap();
15258                assert!(
15259                    module_rx.try_recv().is_err(),
15260                    "a refused open relays nothing"
15261                );
15262                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15263                parse_error(&replies[0])
15264            }
15265
15266            /// Start an open and return its task and the bind the target got.
15267            async fn relayed(
15268                &mut self,
15269                opener: Option<&str>,
15270                target: &str,
15271                scope: Option<ScopeSelector>,
15272            ) -> Relayed {
15273                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15274                let handler = self.handler.clone();
15275                let task_ctx = ctx.clone();
15276                let task = tokio::spawn(async move {
15277                    handler
15278                        .handle_control_frame(&task_ctx, frame)
15279                        .await
15280                        .unwrap()
15281                });
15282                let (_, module_rx) = self.modules.get_mut(target).unwrap();
15283                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15284                    .await
15285                    .expect("the target receives the relayed route.bind")
15286                    .unwrap()
15287                    .frame;
15288                Relayed {
15289                    target: target.to_string(),
15290                    client: ctx,
15291                    client_rx: rx,
15292                    task,
15293                    bind,
15294                }
15295            }
15296
15297            async fn ack(&self, relayed: &Relayed) {
15298                let (module, _) = &self.modules[&relayed.target];
15299                self.handler
15300                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15301                    .await
15302                    .unwrap();
15303            }
15304
15305            /// Open, ack and return the bound route.
15306            async fn bound(
15307                &mut self,
15308                opener: Option<&str>,
15309                target: &str,
15310                scope: Option<ScopeSelector>,
15311            ) -> Bound {
15312                let relayed = self.relayed(opener, target, scope).await;
15313                self.ack(&relayed).await;
15314                let Relayed {
15315                    target,
15316                    client,
15317                    mut client_rx,
15318                    task,
15319                    bind,
15320                } = relayed;
15321                assert!(
15322                    task.await.unwrap().is_empty(),
15323                    "the open is answered by commit"
15324                );
15325                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15326                Bound {
15327                    target,
15328                    client,
15329                    client_rx,
15330                    channel,
15331                    epoch,
15332                    bind,
15333                }
15334            }
15335
15336            fn live(&self, route: &Bound) -> bool {
15337                matches!(
15338                    self.forwarding
15339                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15340                        .unwrap(),
15341                    DataRoute::Client(DataRouteState::Bound(_))
15342                )
15343            }
15344        }
15345
15346        struct Relayed {
15347            target: String,
15348            client: RouteCtx,
15349            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15350            task: tokio::task::JoinHandle<Vec<Frame>>,
15351            bind: Frame,
15352        }
15353
15354        struct Bound {
15355            target: String,
15356            client: RouteCtx,
15357            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15358            channel: u16,
15359            epoch: u32,
15360            bind: Frame,
15361        }
15362
15363        impl Bound {
15364            /// The reason of the `route.closed` this client was sent, after
15365            /// checking it also got a GOODBYE on exactly this route.
15366            fn closed_reason(&mut self) -> RouteCloseReason {
15367                let mut reason = None;
15368                let mut goodbye = false;
15369                while let Ok(outbound) = self.client_rx.try_recv() {
15370                    let frame = outbound.frame;
15371                    match frame.header.ty {
15372                        FrameType::Goodbye => {
15373                            assert_eq!(
15374                                (frame.header.channel, frame.header.epoch),
15375                                (self.channel, self.epoch)
15376                            );
15377                            goodbye = true;
15378                        }
15379                        FrameType::Push => {
15380                            let ClientControlPush::RouteClosed {
15381                                reason: r,
15382                                module_id,
15383                                ..
15384                            } = serde_json::from_slice(&frame.body).unwrap()
15385                            else {
15386                                panic!("unexpected push");
15387                            };
15388                            assert_eq!(module_id, self.target);
15389                            reason = Some(r);
15390                        }
15391                        other => panic!("unexpected frame {other:?}"),
15392                    }
15393                }
15394                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15395                reason.expect("the client is told why the route closed")
15396            }
15397
15398            fn untouched(&mut self) -> bool {
15399                self.client_rx.try_recv().is_err()
15400            }
15401
15402            fn stamp(&self) -> Option<ScopeStamp> {
15403                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15404                    ModuleControlRequest::RouteBind { scope, .. } => scope,
15405                    other => panic!("expected a route.bind, got {other:?}"),
15406                }
15407            }
15408        }
15409
15410        #[tokio::test]
15411        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15412        ) {
15413            let mut rig = rig().await;
15414            let mut record = session(1);
15415            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15416            record.child_owners = vec![Principal::Reserved {
15417                module_id: MAGIC.to_string(),
15418            }];
15419            rig.sync(vec![record]).await;
15420            let scope = || Some(rig_selector("s", Some(1)));
15421
15422            // Admitted: the owner, a bare carrier to any module, a targeted
15423            // carrier to its listed module.
15424            rig.bound(Some(OWNER), PLEXUS, scope()).await;
15425            rig.bound(Some(AFT), OTHER, scope()).await;
15426            rig.bound(Some(BROCA), PLEXUS, scope()).await;
15427
15428            // Refused scope_not_carrier: a targeted carrier to an unlisted
15429            // module, a module that is not listed at all (a child owner is not
15430            // a carrier), and a direct key-holder.
15431            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
15432                assert_eq!(
15433                    rig.refused(opener, target, scope()).await,
15434                    error_codes::SCOPE_NOT_CARRIER,
15435                    "{opener:?} -> {target}"
15436                );
15437            }
15438        }
15439
15440        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15441            ScopeSelector::new(
15442                Principal::Reserved {
15443                    module_id: OWNER.to_string(),
15444                },
15445                scope_ref.to_string(),
15446                scope_epoch,
15447            )
15448        }
15449
15450        #[tokio::test]
15451        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
15452            let mut rig = rig().await;
15453            rig.sync(vec![session(1)]).await;
15454            for opener in [OWNER, AFT] {
15455                assert_eq!(
15456                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
15457                        .await,
15458                    error_codes::SCOPE_EPOCH_REQUIRED,
15459                    "{opener}"
15460                );
15461            }
15462        }
15463
15464        #[tokio::test]
15465        async fn admission_separates_not_synced_not_live_and_ended() {
15466            let mut rig = rig().await;
15467            // Before the configured owner's first sync: retryable.
15468            let code = rig
15469                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15470                .await;
15471            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
15472            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
15473
15474            // An owner that is not configured will never sync: terminal.
15475            let ghost = ScopeSelector::new(
15476                Principal::Reserved {
15477                    module_id: "ghost".to_string(),
15478                },
15479                "s".to_string(),
15480                Some(1),
15481            );
15482            assert_eq!(
15483                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
15484                error_codes::SCOPE_NOT_LIVE
15485            );
15486
15487            rig.sync(vec![session(2)]).await;
15488            assert_eq!(
15489                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
15490                    .await,
15491                error_codes::SCOPE_NOT_LIVE
15492            );
15493            for epoch in [1, 3] {
15494                assert_eq!(
15495                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
15496                        .await,
15497                    error_codes::SCOPE_ENDED,
15498                    "epoch {epoch}"
15499                );
15500            }
15501            // Control: the live epoch is admitted.
15502            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
15503                .await;
15504        }
15505
15506        #[tokio::test]
15507        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
15508            let mut rig = rig().await;
15509            rig.sync(vec![session(1)]).await;
15510            let route = rig
15511                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15512                .await;
15513            let stamp = route.stamp().expect("a scoped bind carries the stamp");
15514            assert_eq!(stamp.scope_ref, "s");
15515            assert_eq!(stamp.scope_epoch, 1);
15516            assert_eq!(stamp.kind, ScopeKind::Head);
15517            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
15518            assert!(stamp.attributes.delegates);
15519            assert!(stamp.owner_authorized);
15520            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15521            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
15522
15523            // broca owns a scope of its own on its own module connection; it is
15524            // not in scope_authority_owners, so its stamp is not authorized.
15525            let (broca, mut broca_rx) = wide_ctx(50);
15526            hello_via_sink(
15527                &rig.handler,
15528                &broca,
15529                &mut broca_rx,
15530                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
15531            )
15532            .await;
15533            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
15534                .await
15535                .unwrap();
15536            let own = ScopeSelector::new(
15537                Principal::Reserved {
15538                    module_id: BROCA.to_string(),
15539                },
15540                "b".to_string(),
15541                Some(1),
15542            );
15543            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
15544            assert!(!route.stamp().unwrap().owner_authorized);
15545        }
15546
15547        #[tokio::test]
15548        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
15549            let mut rig = rig().await;
15550            let mut record = head("s", 1);
15551            record.carriers = vec![carrier(AFT, None)];
15552            let flow_id = "Flow:run-7/step_2!~";
15553            record.attributes.flow_id = Some(flow_id.to_string());
15554            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
15555                .await
15556                .unwrap();
15557            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15558                panic!("not a sync reply");
15559            };
15560            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
15561            let route = rig
15562                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15563                .await;
15564            assert!(rig.live(&route), "the stamped bind committed");
15565            let stamp = route.stamp().expect("a flow scope carries a stamp");
15566            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
15567            assert_eq!(stamp.attributes.agent_id, None);
15568            assert!(!stamp.attributes.delegates);
15569            assert!(stamp.owner_authorized);
15570            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15571            assert_eq!(unscoped.stamp(), None);
15572        }
15573
15574        #[tokio::test]
15575        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
15576            let mut rig = rig_with_flow_support(false).await;
15577            let mut record = session(1);
15578            record.attributes.flow_id = Some("flow:7".to_string());
15579            rig.sync(vec![record]).await;
15580            for opener in [OWNER, AFT] {
15581                let body = rig
15582                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15583                    .await;
15584                assert_eq!(body["code"], "target_flow_unsupported");
15585                let message = body["message"].as_str().unwrap();
15586                for required in [PLEXUS, "flow-scopes/v1"] {
15587                    assert!(message.contains(required), "{message}");
15588                }
15589            }
15590        }
15591
15592        #[tokio::test]
15593        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
15594            let mut rig = rig_with_flow_support(true).await;
15595            let mut record = session(1);
15596            record.attributes.flow_id = Some("flow:7".to_string());
15597            rig.sync(vec![record]).await;
15598            for opener in [OWNER, AFT] {
15599                let route = rig
15600                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15601                    .await;
15602                assert!(rig.live(&route));
15603                assert_eq!(
15604                    route.stamp().unwrap().attributes.flow_id.as_deref(),
15605                    Some("flow:7")
15606                );
15607            }
15608        }
15609
15610        #[tokio::test]
15611        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
15612            use std::future::Future;
15613
15614            let mut rig = rig_with_flow_support(true).await;
15615            let mut record = session(1);
15616            record.attributes.flow_id = Some("flow:7".to_string());
15617            rig.sync(vec![record]).await;
15618            let (client, mut client_rx, frame) =
15619                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
15620            // Hold the route-open response permit so admission sees the first
15621            // target but relay reservation cannot capture an endpoint yet.
15622            for _ in 0..64 {
15623                client.egress.try_send(route_bind_ack(1)).unwrap();
15624            }
15625            let handler = rig.handler.clone();
15626            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15627            std::future::poll_fn(|cx| {
15628                assert!(open.as_mut().poll(cx).is_pending());
15629                std::task::Poll::Ready(())
15630            })
15631            .await;
15632            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15633
15634            let old_connection = rig.modules[PLEXUS].0.connection_id;
15635            rig.handler.cleanup_connection(old_connection).unwrap();
15636            let (replacement, mut replacement_rx) = wide_ctx(200);
15637            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
15638            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15639            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15640            let hello = Frame::build(
15641                FrameType::Hello,
15642                control_flags(),
15643                0,
15644                0,
15645                200,
15646                serde_json::to_vec(&body).unwrap(),
15647            )
15648            .unwrap();
15649            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
15650
15651            client_rx.try_recv().unwrap();
15652            let replies = tokio::time::timeout(Duration::from_secs(2), open)
15653                .await
15654                .expect("the replacement is refused without waiting for a bind ack")
15655                .unwrap();
15656            assert_eq!(replies.len(), 1);
15657            let body = parse_error(&replies[0]);
15658            assert_eq!(body["code"], "target_flow_unsupported");
15659            for required in [PLEXUS, "flow-scopes/v1"] {
15660                assert!(body["message"].as_str().unwrap().contains(required));
15661            }
15662            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
15663            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15664            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15665            assert!(rig
15666                .handler
15667                .registry
15668                .get_module_by_connection(replacement.connection_id)
15669                .unwrap()
15670                .is_some());
15671        }
15672
15673        #[tokio::test]
15674        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
15675            use std::future::Future;
15676            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15677            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15678            const NEXT_HASH: &str =
15679                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15680            let mut rig = rig().await;
15681            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
15682            record.carriers.push(ScopeCarrier::new(Principal::Direct));
15683            rig.sync(vec![record.clone()]).await;
15684            let (client, mut rx, frame) = rig.open_frame(
15685                None,
15686                PLEXUS,
15687                Some(
15688                    rig.selector("s", Some(1))
15689                        .with_bind_proof(Some(PROOF.into())),
15690                ),
15691            );
15692            // Hold egress reservation while the scope owner changes its bind-proof
15693            // hash, so a proof valid at admission must be checked again at relay.
15694            for _ in 0..64 {
15695                client.egress.try_send(route_bind_ack(1)).unwrap();
15696            }
15697            let handler = rig.handler.clone();
15698            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15699            std::future::poll_fn(|cx| {
15700                assert!(open.as_mut().poll(cx).is_pending());
15701                std::task::Poll::Ready(())
15702            })
15703            .await;
15704            record.bind_proof_sha256 = Some(NEXT_HASH.into());
15705            rig.sync(vec![record]).await;
15706            rx.try_recv().unwrap();
15707            let replies = tokio::time::timeout(Duration::from_secs(2), open)
15708                .await
15709                .unwrap()
15710                .unwrap();
15711            assert_eq!(
15712                parse_error(&replies[0])["code"],
15713                error_codes::SCOPE_BIND_PROOF_MISMATCH
15714            );
15715            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
15716            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15717            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15718        }
15719
15720        #[tokio::test]
15721        async fn scope_bind_proof_reserved_openers_ignore_proof() {
15722            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15723            let mut rig = rig().await;
15724            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15725                .await;
15726            for opener in [OWNER, AFT] {
15727                for proof in [None, Some("malformed+proof=")] {
15728                    let route = rig
15729                        .bound(
15730                            Some(opener),
15731                            PLEXUS,
15732                            Some(
15733                                rig.selector("s", Some(1))
15734                                    .with_bind_proof(proof.map(str::to_string)),
15735                            ),
15736                        )
15737                        .await;
15738                    assert!(rig.live(&route));
15739                }
15740            }
15741        }
15742
15743        #[tokio::test]
15744        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
15745            use std::future::Future;
15746            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15747            const NEXT_HASH: &str =
15748                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15749            let mut rig = rig().await;
15750            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15751                .await;
15752            let (client, mut rx, frame) =
15753                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
15754            for _ in 0..64 {
15755                client.egress.try_send(route_bind_ack(1)).unwrap();
15756            }
15757            let handler = rig.handler.clone();
15758            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15759            std::future::poll_fn(|cx| {
15760                assert!(open.as_mut().poll(cx).is_pending());
15761                std::task::Poll::Ready(())
15762            })
15763            .await;
15764            rig.sync(vec![
15765                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
15766            ])
15767            .await;
15768            rx.try_recv().unwrap();
15769            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15770            let bind = std::future::poll_fn(|cx| {
15771                assert!(open.as_mut().poll(cx).is_pending());
15772                module_rx.poll_recv(cx)
15773            })
15774            .await
15775            .unwrap()
15776            .frame;
15777            handler
15778                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
15779                .await
15780                .unwrap();
15781            let replies = tokio::time::timeout(Duration::from_secs(2), open)
15782                .await
15783                .unwrap()
15784                .unwrap();
15785            assert_eq!(replies.len(), 1);
15786            // Proof rechecking allowed the relay. Independent tag settlement
15787            // still asks the opener to retry after any same-epoch content change.
15788            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
15789            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15790            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
15791            let route = rig
15792                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
15793                .await;
15794            assert!(rig.live(&route));
15795        }
15796
15797        #[tokio::test]
15798        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
15799        {
15800            let mut rig = rig_with_flow_support(false).await;
15801            rig.sync(vec![session(1)]).await;
15802            let route = rig
15803                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15804                .await;
15805            assert!(rig.live(&route));
15806            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
15807            let mut record = session(1);
15808            record.attributes.flow_id = Some("flow:7".to_string());
15809            rig.sync(vec![record]).await;
15810            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
15811            assert!(rig.live(&unscoped));
15812            assert_eq!(unscoped.stamp(), None);
15813        }
15814
15815        #[tokio::test]
15816        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
15817            let mut rig = rig().await;
15818            let mut record = session(1);
15819            record.attributes.flow_id = Some("flow:7".to_string());
15820            rig.sync(vec![record.clone()]).await;
15821            let before = rig.forwarding.published_scope_tag(OWNER, "s");
15822            let mut owner_route = rig
15823                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15824                .await;
15825            let mut carrier_route = rig
15826                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
15827                .await;
15828            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15829            rig.sync(vec![record.clone()]).await;
15830            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15831            assert!(rig.live(&owner_route) && owner_route.untouched());
15832            assert!(rig.live(&carrier_route) && carrier_route.untouched());
15833
15834            record.attributes.flow_id = Some("flow:8".to_string());
15835            rig.sync(vec![record]).await;
15836            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
15837            let before = before.unwrap();
15838            assert_eq!(after.scope_epoch, before.scope_epoch);
15839            assert!(after.version > before.version);
15840            for route in [&mut owner_route, &mut carrier_route] {
15841                assert!(!rig.live(route));
15842                assert_eq!(
15843                    route.closed_reason(),
15844                    RouteCloseReason::ScopeDelegationChanged
15845                );
15846            }
15847            assert!(rig.live(&unscoped) && unscoped.untouched());
15848            // Each provider also receives a GOODBYE for its drained route;
15849            // consume it before expecting the next route.bind on that sink.
15850            for target in [PLEXUS, OTHER] {
15851                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
15852                let goodbye = module_rx
15853                    .try_recv()
15854                    .expect("the provider sees the drain")
15855                    .frame;
15856                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15857                assert!(module_rx.try_recv().is_err());
15858            }
15859            let rebound = rig
15860                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15861                .await;
15862            assert_eq!(
15863                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
15864                Some("flow:8")
15865            );
15866        }
15867
15868        /// The owner's sync lands between admission and the module's ack. The
15869        /// open is refused by name, the module's other routes stay up, and the
15870        /// reserved pair is released. Changed content is retryable; an ended
15871        /// scope is not.
15872        #[tokio::test]
15873        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
15874            let mut rig = rig().await;
15875            rig.sync(vec![session(1)]).await;
15876            let mut cotenant = rig
15877                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15878                .await;
15879
15880            let mut changed = session(1);
15881            changed.child_owners.push(Principal::Reserved {
15882                module_id: MAGIC.to_string(),
15883            });
15884            let mut ended = None;
15885            for (code, next) in [
15886                (error_codes::SCOPE_CHANGED, vec![changed]),
15887                (error_codes::SCOPE_ENDED, Vec::new()),
15888            ] {
15889                let relayed = rig
15890                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15891                    .await;
15892                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
15893                ended = Some(next.is_empty());
15894                rig.sync(next).await;
15895                rig.ack(&relayed).await;
15896                let replies = relayed.task.await.unwrap();
15897                assert_eq!(replies.len(), 1, "{replies:?}");
15898                assert_eq!(parse_error(&replies[0])["code"], code);
15899                assert_eq!(
15900                    subc_protocol::error_codes::is_retryable_route_open(code),
15901                    code == error_codes::SCOPE_CHANGED
15902                );
15903                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15904                // The module is told to drop just the binding it created.
15905                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15906                // Collected, because ending the scope also closes the co-tenant
15907                // route, whose GOODBYE comes first.
15908                let mut goodbyes = Vec::new();
15909                while let Ok(outbound) = plexus_rx.try_recv() {
15910                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
15911                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
15912                }
15913                assert!(
15914                    goodbyes.contains(&(bind_channel, bind_epoch)),
15915                    "{goodbyes:?}"
15916                );
15917                assert!(rig
15918                    .handler
15919                    .registry
15920                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
15921                    .unwrap()
15922                    .is_some());
15923            }
15924            assert_eq!(ended, Some(true));
15925            // The co-tenant stayed up through the change, and closed only when
15926            // the scope ended, by the drain rule rather than by the commit.
15927            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
15928        }
15929
15930        /// Each row of the drain table on one set of routes: the owner's, a
15931        /// bare carrier's, and a targeted carrier's to each of its targets.
15932        #[tokio::test]
15933        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
15934            struct Case {
15935                name: &'static str,
15936                change: fn(&mut ScopeRecord),
15937                /// Closed routes by index: owner->plexus, aft->plexus,
15938                /// broca->plexus, broca->other.
15939                closed: [Option<RouteCloseReason>; 4],
15940            }
15941            use RouteCloseReason::*;
15942            let cases = [
15943                Case {
15944                    name: "a carrier entry removed",
15945                    change: |r| {
15946                        r.carriers.retain(|c| {
15947                            c.principal
15948                                != Principal::Reserved {
15949                                    module_id: AFT.to_string(),
15950                                }
15951                        })
15952                    },
15953                    closed: [None, Some(ScopeCarrierRemoved), None, None],
15954                },
15955                Case {
15956                    name: "a target removed from a carrier",
15957                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
15958                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
15959                },
15960                Case {
15961                    name: "a bare carrier narrowed to targets",
15962                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
15963                    closed: [None, Some(ScopeCarrierRemoved), None, None],
15964                },
15965                Case {
15966                    name: "delegates turned off",
15967                    change: |r| r.attributes.delegates = false,
15968                    closed: [Some(ScopeDelegationChanged); 4],
15969                },
15970                Case {
15971                    name: "agent_id changed",
15972                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
15973                    closed: [Some(ScopeDelegationChanged); 4],
15974                },
15975                Case {
15976                    name: "a carrier added, child owners changed, the record re-sent",
15977                    change: |r| {
15978                        r.carriers.push(carrier(MAGIC, None));
15979                        r.child_owners.push(Principal::Reserved {
15980                            module_id: MAGIC.to_string(),
15981                        });
15982                    },
15983                    closed: [None; 4],
15984                },
15985                Case {
15986                    name: "a target added",
15987                    change: |r| {
15988                        r.carriers[1]
15989                            .targets
15990                            .as_mut()
15991                            .unwrap()
15992                            .push("third".to_string())
15993                    },
15994                    closed: [None; 4],
15995                },
15996                Case {
15997                    name: "delegates turned on",
15998                    change: |r| r.attributes.delegates = true,
15999                    closed: [None; 4],
16000                },
16001            ];
16002            for case in cases {
16003                let mut rig = rig().await;
16004                rig.sync(vec![session(1)]).await;
16005                let scope = || Some(rig_selector("s", Some(1)));
16006                let mut routes = [
16007                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
16008                    rig.bound(Some(AFT), PLEXUS, scope()).await,
16009                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
16010                    rig.bound(Some(BROCA), OTHER, scope()).await,
16011                ];
16012                let mut record = session(1);
16013                (case.change)(&mut record);
16014                rig.sync(vec![record]).await;
16015                for (index, expected) in case.closed.iter().enumerate() {
16016                    let route = &mut routes[index];
16017                    match expected {
16018                        Some(reason) => {
16019                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
16020                            assert_eq!(
16021                                route.closed_reason(),
16022                                *reason,
16023                                "{}: route {index}",
16024                                case.name
16025                            );
16026                        }
16027                        None => {
16028                            assert!(rig.live(route), "{}: route {index} closed", case.name);
16029                            assert!(
16030                                route.untouched(),
16031                                "{}: route {index} was told something",
16032                                case.name
16033                            );
16034                        }
16035                    }
16036                }
16037            }
16038        }
16039
16040        #[tokio::test]
16041        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
16042            // Removed, and replaced by a higher epoch.
16043            for next in [Vec::new(), vec![session(2)]] {
16044                let mut rig = rig().await;
16045                rig.sync(vec![session(1)]).await;
16046                let mut route = rig
16047                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16048                    .await;
16049                rig.sync(next).await;
16050                assert!(!rig.live(&route));
16051                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
16052            }
16053
16054            // A child whose parent ends: its routes close as parent-ended, the
16055            // child stays live, and routes under the parent close as ended.
16056            let mut rig = rig().await;
16057            let mut child = session(1);
16058            child.scope_ref = "child".to_string();
16059            child.kind = ScopeKind::Worker;
16060            child.parent = Some(ScopeParent::new(
16061                Principal::Reserved {
16062                    module_id: OWNER.to_string(),
16063                },
16064                "s".to_string(),
16065                1,
16066            ));
16067            rig.sync(vec![session(1), child.clone()]).await;
16068            let mut child_route = rig
16069                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
16070                .await;
16071            assert_eq!(
16072                child_route.stamp().unwrap().parent_state,
16073                Some(ParentState::Linked)
16074            );
16075            rig.sync(vec![child]).await;
16076            assert!(!rig.live(&child_route));
16077            assert_eq!(
16078                child_route.closed_reason(),
16079                RouteCloseReason::ScopeParentEnded
16080            );
16081        }
16082
16083        #[tokio::test]
16084        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
16085        ) {
16086            let mut rig = rig().await;
16087            rig.sync(vec![session(1)]).await;
16088            let mut route = rig
16089                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16090                .await;
16091            let before = rig.forwarding.published_scope_tag(OWNER, "s");
16092            rig.sync(vec![session(1)]).await;
16093            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16094            assert!(rig.live(&route) && route.untouched());
16095
16096            // A call in flight on the route when another carrier is added. A
16097            // forwarded REQUEST holds one credit on the route's flow until the
16098            // module answers; the router takes it exactly like this.
16099            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
16100                .forwarding
16101                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16102                .unwrap()
16103            else {
16104                panic!("the route is bound");
16105            };
16106            binding.flow.acquire_tagged(9, false).await.unwrap();
16107            let mut widened = session(1);
16108            widened.carriers.push(carrier(MAGIC, None));
16109            rig.sync(vec![widened]).await;
16110            assert!(rig.live(&route) && route.untouched());
16111            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16112            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
16113            // The call's credit is still held on an open flow, so its answer
16114            // will be delivered: closing the route would have closed the flow.
16115            assert_eq!(binding.flow.in_flight(), 1);
16116            binding
16117                .flow
16118                .acquire_tagged(10, false)
16119                .await
16120                .expect("the flow is still open");
16121        }
16122
16123        /// A swap's superseded endpoint keeps its routes until drained; ending
16124        /// the scope closes them there too.
16125        #[tokio::test]
16126        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
16127            let mut rig = rig().await;
16128            rig.sync(vec![session(1)]).await;
16129            let mut on_incumbent = rig
16130                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16131                .await;
16132
16133            // Swap plexus: register a candidate and cut over, leaving the
16134            // incumbent superseded with the route still on it.
16135            let (candidate, _candidate_rx) = wide_ctx(9);
16136            let registration = rig
16137                .handler
16138                .registry
16139                .register_candidate_with_control_ops(
16140                    manifest(PLEXUS, PROTOCOL_VERSION),
16141                    PROTOCOL_VERSION,
16142                    candidate.connection_id,
16143                    module_baseline_control_ops(),
16144                )
16145                .unwrap();
16146            rig.forwarding
16147                .register_candidate_module_connection(
16148                    candidate.connection_id,
16149                    PLEXUS.to_string(),
16150                    PROTOCOL_VERSION,
16151                    manifest_concurrency(&registration.manifest),
16152                    candidate.egress.clone(),
16153                )
16154                .unwrap();
16155            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
16156            rig.handler
16157                .registry
16158                .promote_candidate(PLEXUS)
16159                .unwrap()
16160                .unwrap();
16161            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
16162
16163            rig.sync(Vec::new()).await;
16164            assert!(!rig.live(&on_incumbent));
16165            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
16166            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16167            let goodbye = incumbent_rx
16168                .try_recv()
16169                .expect("the superseded endpoint is told")
16170                .frame;
16171            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16172        }
16173    }
16174}
16175
16176#[cfg(test)]
16177mod concurrency_default_exposure_tests {
16178    use super::*;
16179
16180    fn hello_body(role_json: &str) -> Vec<u8> {
16181        format!(
16182            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
16183        )
16184        .into_bytes()
16185    }
16186
16187    fn manifest_from(body: &[u8]) -> ModuleManifest {
16188        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
16189        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
16190            .expect("manifest parses")
16191    }
16192
16193    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
16194
16195    #[test]
16196    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
16197        let body = hello_body(&format!(
16198            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
16199        ));
16200        let manifest = manifest_from(&body);
16201        // Precondition: serde really resolved it to the default, so the typed
16202        // manifest alone cannot answer the question this probe exists for.
16203        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16204        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
16205    }
16206
16207    #[test]
16208    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
16209        let body = hello_body(&format!(
16210            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
16211        ));
16212        let manifest = manifest_from(&body);
16213        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16214        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16215    }
16216
16217    #[test]
16218    fn non_management_roles_are_never_reported() {
16219        let body = hello_body(
16220            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
16221        );
16222        let manifest = manifest_from(&body);
16223        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16224    }
16225}