Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    path::{Path, PathBuf},
5    sync::{Arc, Mutex, RwLock},
6    time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
21    error_codes,
22    manifest::{
23        validate_hello_capability_grammar, validate_hello_event_declarations,
24        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
25        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
26    },
27    scope::{
28        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
29        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
30        SCOPE_SYNC_OP,
31    },
32    session::{
33        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
34        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
35        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
36        ROLE_VERSIONS_FIELD,
37    },
38    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
39    Priority, RouteTarget, PROTOCOL_VERSION,
40};
41use tokio::time::{timeout_at, Instant};
42use tracing::{debug, info, warn};
43
44use crate::{
45    capability_requirements::{
46        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
47        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
48        RuntimeModule,
49    },
50    daemon_config::RestartRequiredSection,
51    forwarding::{
52        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
53        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
54        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
55    },
56    observability::{
57        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
58    },
59    provenance::{
60        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
61    },
62    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
63    router::{RouteCtx, RouterError},
64    scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
65    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
66    stderr_tail::{CaptureState, TailEntry},
67    supervise::{
68        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
69        SupervisorHandle, SwapHelloAdmission,
70    },
71    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
72};
73
74/// Lowest envelope version this subc build will negotiate.
75///
76/// Module HELLO negotiation is exact: peers must use the daemon's locked
77/// protocol version. Older and newer peers receive `version_unsupported` and
78/// are not registered.
79pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
80
81const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
82const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
83const CAP_PING_PONG: &str = "ping_pong_v1";
84const CAP_SESSION_ATTACH: &str = "session_attach_v1";
85const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
86
87const SUBC_CONTROL_OPS: &[&str] = &[
88    ops::ACTIVITY_RECORD,
89    ops::ACTIVITY_QUERY,
90    ops::ACTIVITY_SUBSCRIBE,
91    ops::SERVER_DESCRIBE,
92    ops::CATALOG_LIST,
93    ops::ROUTE_OPEN,
94    ops::ROUTE_POLL,
95    ops::ROUTE_CLOSING,
96    ops::ROUTE_CLOSED,
97    ops::SUPERVISOR_LIST,
98    ops::SUPERVISOR_RESTART,
99    ops::SUPERVISOR_SWAP,
100    ops::SUPERVISOR_RELOAD,
101    ops::SUPERVISOR_RESCAN,
102    ops::SUPERVISOR_RELEASE_RESERVED,
103    ops::SUPERVISOR_SET_ENABLED,
104    ops::SUPERVISOR_HEALTH_PROBE,
105    ops::SUPERVISOR_HEALTH,
106    ops::SUPERVISOR_STDERR_TAIL,
107    ops::SUPERVISOR_TERMINALS,
108    ops::SUPERVISOR_ROUTES,
109    ops::SUPERVISOR_PROVENANCE,
110    ops::SUPERVISOR_SPAWN_SNAPSHOT,
111    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
112];
113
114const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
115    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
116    "supervisor.live_roots",
117    SCOPE_SYNC_OP,
118    SCOPE_APPLY_OP,
119    SCOPE_DESCRIBE_OP,
120    "operator.confirm",
121];
122
123/// Module-originated ops the daemon answers but does not advertise in
124/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
125/// belongs to is incomplete, so no module is told it works before it does.
126const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
127
128const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
129
130/// How long subc waits for a module to ack a relayed route.bind before returning
131/// `module_timeout`. The ack waits on the module's own configure, which for AFT
132/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
133/// and DB-open work — on a cold page cache or a large repo that legitimately
134/// exceeds a couple of seconds. The default is generous because rejecting a VALID
135/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
136/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
137pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
138
139/// How many CONSECUTIVE full-budget relay timeouts against one target module
140/// open that module's bind-relay breaker.
141///
142/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
143/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
144/// so three consecutive full-budget timeouts take ~36s to observe: every client
145/// whose open contributed to opening the breaker had already given up on its
146/// own. That is what makes opening the breaker unable to turn a call that would
147/// have succeeded into a refusal — it can only make an already-failing module
148/// fail faster.
149///
150/// Two would be reachable inside one default deadline. One would convict a
151/// module on a single cold-cache bind, which is exactly the valid-but-slow case
152/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
153pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
154
155/// How long a module's bind-relay breaker stays open before exactly one
156/// `route.open` is let through as a probe.
157///
158/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
159/// re-pays a full-budget stall almost continuously, and the breaker stops being
160/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
161/// deadline: a client that starts retrying after the module recovers has to get
162/// a probe opportunity inside its own deadline, or the breaker converts a
163/// recovered module into a failed call — the failure it exists to prevent,
164/// pointed the other way.
165///
166/// 20s sits between those with room on both sides, and it caps what a wedged
167/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
168/// rather than one per `route.open` per connection. The stall that motivated
169/// this, with its measurements, is written up in
170/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
171/// waited the whole budget out.
172pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
173
174const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
175const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
176
177fn reload_verdict(
178    configured: &Path,
179    spawned_from: Option<&Path>,
180    image: subc_control::RunningImageAgreement,
181) -> PendingReloadVerdict {
182    let path = match spawned_from {
183        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
184            ReloadPathAgreement::Match
185        }
186        Some(spawned_from) => ReloadPathAgreement::Mismatch {
187            configured: configured.to_path_buf(),
188            spawned_from: spawned_from.to_path_buf(),
189        },
190        None => ReloadPathAgreement::Unavailable {
191            reason: if matches!(
192                image,
193                subc_control::RunningImageAgreement::Unavailable {
194                    reason: subc_control::RunningImageUnavailableReason::NotRunning
195                }
196            ) {
197                ReloadPathUnavailableReason::NotRunning
198            } else {
199                ReloadPathUnavailableReason::SpawnedPathUnavailable
200            },
201        },
202    };
203    PendingReloadVerdict { path, image }
204}
205
206fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
207    if configured == spawned_from {
208        return true;
209    }
210    #[cfg(windows)]
211    {
212        // Windows spawn records the resolved verbatim path (\\?\...), while
213        // configuration keeps its original spelling. Resolve both with the OS
214        // instead of stripping prefixes or folding case, which could conflate
215        // distinct names in case-sensitive directories. Different resolved
216        // paths still differ even if their files have identical content.
217        match (
218            std::fs::canonicalize(configured),
219            std::fs::canonicalize(spawned_from),
220        ) {
221            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
222            _ => false,
223        }
224    }
225    #[cfg(not(windows))]
226    {
227        false
228    }
229}
230
231#[derive(Clone)]
232struct DaemonProvenanceFacts {
233    build: DaemonBuildProvenance,
234    pid: Option<u32>,
235    started_at_ms: Option<u64>,
236    start_clock: Option<crate::clock::StartClock>,
237    executable_path: Option<PathBuf>,
238    executable_identity: Option<SpawnedFileIdentity>,
239    process_start_time: Option<u64>,
240    probe: ExecutableIdentityProbe,
241}
242
243impl Default for DaemonProvenanceFacts {
244    fn default() -> Self {
245        Self {
246            build: DaemonBuildProvenance {
247                build_git_sha: None,
248                build_lock_digest: None,
249            },
250            pid: None,
251            started_at_ms: None,
252            start_clock: None,
253            executable_path: None,
254            executable_identity: None,
255            process_start_time: None,
256            probe: ExecutableIdentityProbe::default(),
257        }
258    }
259}
260
261#[derive(Debug, Clone)]
262struct SupervisorRescanContext {
263    supervisor: Supervisor,
264    config_path: PathBuf,
265    configured_port: Option<u16>,
266    storage_config: Option<crate::daemon_config::StorageConfig>,
267    admission_facts_carrier_module_id: Option<String>,
268    admission_facts_targets: Option<Vec<String>>,
269    scope_authority_owners: Vec<String>,
270}
271
272/// Refusal labels passed to `observe_route_open_refusal` that mean the target
273/// module is not serving right now, and so open or extend an outage in the
274/// route outage tracker. Every one of them is only reachable after the target
275/// was found in the registry, which is what keeps an arbitrary client-chosen
276/// id from ever creating tracker state.
277///
278/// Deliberately absent: `not_registered` and `removed` (the id may be
279/// anything a client sent, and a removed module is gone on purpose),
280/// `protocol_none` (such a module never serves routes, so nothing is out),
281/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
282/// capability and admission-facts refusals (they refuse the caller, not a
283/// module outage), and `relay_reservation_failed` (its code ranges over
284/// capacity limits as well as a vanished connection). Capacity, breaker,
285/// relay-timeout and module-rejection refusals do not pass through that
286/// function at all; the breaker logs its own transitions.
287///
288/// The two not-serving refusals that bypass that function record themselves
289/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
290/// `required_capability_unprovided` is not tracked: the module itself is up,
291/// and the outage belongs to the missing provider.
292const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
293    "reloading",
294    "supervisor_not_live",
295    "registration_not_active",
296    "no_forwarding_connection",
297    "relay_send_failed",
298];
299
300/// Real channel-0 control handler for subc itself.
301#[derive(Clone)]
302pub struct ControlHandler {
303    registry: Arc<Registry>,
304    activity: Option<crate::activity::ActivityLedger>,
305    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
306    forwarding: Arc<ForwardingTable>,
307    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
308    supervisor: SupervisorHandle,
309    subc_capabilities: Arc<[String]>,
310    /// Daemon-wide route.bind relay budget. Used as the fallback when the
311    /// target module has no per-module override in
312    /// `route_bind_relay_timeouts`.
313    route_bind_relay_timeout: Duration,
314    /// Per-module route.bind relay budget overrides, keyed by module id. When
315    /// `handle_route_open` resolves the deadline for a target module, a
316    /// per-module entry wins over the daemon-wide value above.
317    route_bind_relay_timeouts: BTreeMap<String, Duration>,
318    /// Per-target-module bind-relay breaker state. Shared with the forwarding
319    /// table, which is where a new module connection resets it.
320    route_bind_breakers: RouteBindBreakers,
321    /// Live relay admissions keyed by target module. Shared through the
322    /// forwarding table so cloned or separately built handlers enforce one cap.
323    route_bind_concurrency: RouteBindConcurrency,
324    /// Start and end of each module's not-serving period as seen by
325    /// `route.open`, so an outage gets one line at each edge instead of only
326    /// the per-refusal INFO lines. Taken from the forwarding table, so every
327    /// handler built over one table shares it.
328    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
329    /// Consecutive relay timeouts that open a module's breaker.
330    route_bind_breaker_threshold: u32,
331    /// How long a breaker stays open before one probe is admitted.
332    route_bind_breaker_cooldown: Duration,
333    health_probe_timeout: Duration,
334    /// Central storage policy. When set, each registering module receives its
335    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
336    storage_config: Option<crate::daemon_config::StorageConfig>,
337    /// The machine id established at boot, served on every HELLO_ACK and on
338    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
339    /// changes the file, never this value. `None` serves no id.
340    machine_id: Option<crate::machine_id::MachineId>,
341    admission_facts_carrier_module_id: Option<String>,
342    admission_facts_targets: Option<Vec<String>>,
343    /// Scope records with their sync authorities and tombstones; see
344    /// `crate::scopes`. Shared by clones of this handler, so every connection
345    /// reads and writes one table.
346    scopes: Arc<RwLock<ScopeTable>>,
347    /// Wakes the expiry loop when an accepted scope change may have added a
348    /// deadline, so the loop can wait without ticking while no live scope has
349    /// one. A permit is stored when nothing waits, so no wake is lost.
350    scope_deadline_added: Arc<tokio::sync::Notify>,
351    /// The configured `scope_authority_owners`, kept so a rescan can report a
352    /// changed value as needing a daemon restart; rescan never applies it.
353    scope_authority_owners: Vec<String>,
354    /// The launch nonce each module connection presented at HELLO, which is how
355    /// a `scope.sync` is matched to the owner's current launch.
356    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
357    rescan: Option<SupervisorRescanContext>,
358    connected_clients: ConnectedClients,
359    counters: DaemonCounters,
360    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
361    daemon_provenance: DaemonProvenanceFacts,
362    #[cfg(test)]
363    control_dispatch_delay: Option<Duration>,
364    #[cfg(test)]
365    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
366    /// Holds `route.open` after the route.bind is queued to the module and
367    /// before it is marked enqueued, so a test can make the module's reply (or
368    /// a client teardown) land exactly inside that window.
369    #[cfg(test)]
370    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
371}
372
373impl fmt::Debug for ControlHandler {
374    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
375        f.debug_struct("ControlHandler")
376            .field("registry", &self.registry)
377            .field("forwarding", &self.forwarding)
378            .field("process_liveness", &self.process_liveness.is_some())
379            .field("supervisor", &self.supervisor)
380            .field("subc_capabilities", &self.subc_capabilities)
381            .finish()
382    }
383}
384
385struct RouteOpenRequest {
386    target: RouteTarget,
387    identity: BindIdentity,
388    consumer_identity: Option<ConsumerIdentity>,
389    consumer_capabilities: Option<Vec<String>>,
390    role_versions: Option<BTreeMap<String, String>>,
391    admission_facts: Option<serde_json::Value>,
392    scope: Option<ScopeSelector>,
393}
394
395struct RouteBindReservationGuard {
396    forwarding: Arc<ForwardingTable>,
397    endpoint: ModuleEndpointId,
398    relay_corr: u64,
399    armed: bool,
400}
401
402struct ModuleControlRpcGuard {
403    forwarding: Arc<ForwardingTable>,
404    endpoint: ModuleEndpointId,
405    corr: u64,
406    armed: bool,
407}
408
409impl ModuleControlRpcGuard {
410    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
411        Self {
412            forwarding,
413            endpoint,
414            corr,
415            armed: true,
416        }
417    }
418
419    fn disarm(&mut self) {
420        self.armed = false;
421    }
422}
423
424impl Drop for ModuleControlRpcGuard {
425    fn drop(&mut self) {
426        if self.armed {
427            let _ = self
428                .forwarding
429                .cancel_module_control_rpc(self.endpoint, self.corr);
430        }
431    }
432}
433
434impl RouteBindReservationGuard {
435    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
436        Self {
437            forwarding,
438            endpoint,
439            relay_corr,
440            armed: true,
441        }
442    }
443
444    fn release_and_disarm(&mut self) {
445        if !self.armed {
446            return;
447        }
448        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
449            self.endpoint,
450            self.relay_corr,
451            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
452        ) {
453            send_goodbye_target_best_effort(
454                &self.forwarding.counters(),
455                &target,
456                "canceled route.bind",
457            );
458        }
459        self.armed = false;
460    }
461
462    fn disarm(&mut self) {
463        self.armed = false;
464    }
465}
466
467impl Drop for RouteBindReservationGuard {
468    fn drop(&mut self) {
469        self.release_and_disarm();
470    }
471}
472
473/// Per-target-module circuit breaker around the `route.bind` relay.
474///
475/// The connection reader is serial per connection, so a module whose `on_bind`
476/// sits on the ack blocks every LATER frame on the connections that call it,
477/// including calls to unrelated modules. This does not make any module's bind
478/// fast; it stops the daemon paying the full budget again and again for a
479/// condition it has already observed.
480///
481/// State is keyed by TARGET MODULE and shared by every connection: a wedged
482/// module wedges everyone, so what one connection learned should protect the
483/// rest.
484///
485/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
486/// relay to that module has actually timed out, and is removed again when a
487/// relay is accepted or the module reconnects, so it cannot grow with traffic
488/// or with modules that behave.
489///
490/// # Why a `std` mutex here is not the head-of-line defect again
491///
492/// Acquisition never awaits. The critical section is a hash lookup plus a few
493/// integer updates, with no I/O and no `.await` inside it, so a reader task
494/// cannot be descheduled behind it the way it can behind
495/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
496/// primitive, held for the same kind of work, as the refusal counter this very
497/// path already increments.
498///
499/// It is also NOT on the data-plane splice path: only `route.open` and module
500/// registration touch it, so bound-route frames gain no state check and no
501/// contention.
502#[derive(Debug, Clone, Default)]
503pub(crate) struct RouteBindBreakers {
504    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
505}
506
507#[derive(Debug, Clone, Default)]
508pub(crate) struct RouteBindConcurrency {
509    modules: Arc<Mutex<HashMap<String, usize>>>,
510}
511
512struct RouteBindConcurrencyGuard {
513    concurrency: RouteBindConcurrency,
514    module_id: String,
515}
516
517impl RouteBindConcurrency {
518    /// Admit without waiting. Waiting here would move the bind stall from the
519    /// module reply to a semaphore and restore reader head-of-line blocking.
520    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
521        let mut modules = self
522            .modules
523            .lock()
524            .expect("route.bind concurrency mutex poisoned");
525        let in_flight = modules.entry(module_id.to_string()).or_default();
526        if *in_flight >= limit {
527            return Err(*in_flight);
528        }
529        *in_flight += 1;
530        Ok(RouteBindConcurrencyGuard {
531            concurrency: self.clone(),
532            module_id: module_id.to_string(),
533        })
534    }
535}
536
537impl Drop for RouteBindConcurrencyGuard {
538    fn drop(&mut self) {
539        let mut modules = self
540            .concurrency
541            .modules
542            .lock()
543            .expect("route.bind concurrency mutex poisoned");
544        let remove = {
545            let in_flight = modules
546                .get_mut(&self.module_id)
547                .expect("admitted route.bind has a concurrency entry");
548            *in_flight -= 1;
549            *in_flight == 0
550        };
551        if remove {
552            modules.remove(&self.module_id);
553        }
554    }
555}
556
557#[derive(Debug, Default)]
558struct ModuleBreakerState {
559    /// Relay timeouts observed with no accepted relay in between.
560    consecutive_timeouts: u32,
561    /// `Some` while the breaker is open: the instant the cooldown expires and
562    /// the next arrival may probe. `None` means closed.
563    cooldown_until: Option<Instant>,
564    /// A half-open probe has been admitted and has not settled yet. This is
565    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
566    /// that read the cooldown, so concurrent opens arriving at the moment the
567    /// cooldown expires cannot all decide that they are the probe.
568    probe_in_flight: Option<Arc<()>>,
569}
570
571/// What the breaker decided for one `route.open`, before any relay work.
572enum RouteBindAdmission<'a> {
573    Admitted {
574        guard: RouteBindBreakerGuard<'a>,
575        /// This open is the single half-open probe, so the transition is worth
576        /// one log line.
577        probe: bool,
578    },
579    Refused {
580        consecutive_timeouts: u32,
581        /// What is left of the cooldown. Zero when the refusal is because the
582        /// one probe is already in flight rather than because the cooldown has
583        /// not elapsed.
584        retry_in: Duration,
585        probe_in_flight: bool,
586    },
587}
588
589/// An outstanding admission, which must be told how its relay settled.
590///
591/// `Drop` settles it as inconclusive, so an early return between admission and
592/// the relay -- or the whole handler being cancelled when the client
593/// disconnects -- releases a half-open probe slot instead of leaving the
594/// breaker wedged half-open with no further probes.
595struct RouteBindBreakerGuard<'a> {
596    breakers: RouteBindBreakers,
597    module_id: &'a str,
598    probe_token: Option<Arc<()>>,
599    settled: bool,
600}
601
602impl RouteBindBreakerGuard<'_> {
603    /// The module answered within the budget and took the bind. THE ONLY
604    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
605    /// breaker, which is a transition worth logging.
606    fn record_accepted(&mut self) -> bool {
607        self.settled = true;
608        self.breakers.record_accepted(self.module_id)
609    }
610
611    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
612    /// COUNTS TOWARD OPENING.
613    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
614        self.settled = true;
615        self.breakers.record_timeout(
616            self.module_id,
617            self.probe_token.as_ref(),
618            threshold,
619            cooldown,
620        )
621    }
622
623    /// Everything else: the module REJECTED the bind, its connection went away
624    /// mid-relay, or the waiter was cancelled.
625    ///
626    /// None of these is evidence that a module is slow, and each already has
627    /// its own refusal with its own code. A module that rejects a bind in
628    /// microseconds is healthy and must never be convicted for it; a module
629    /// that died has said nothing about the module that replaces it. So these
630    /// neither increment nor reset the count -- they only release a probe slot.
631    fn record_inconclusive(&mut self) {
632        self.settled = true;
633        self.breakers
634            .record_inconclusive(self.module_id, self.probe_token.as_ref());
635    }
636}
637
638impl Drop for RouteBindBreakerGuard<'_> {
639    fn drop(&mut self) {
640        if !self.settled {
641            self.breakers
642                .record_inconclusive(self.module_id, self.probe_token.as_ref());
643        }
644    }
645}
646
647/// The breaker moved to open, reported so the caller can log it outside the
648/// lock. Opening is rare and load-bearing; the refusals that follow are
649/// frequent and are counted rather than logged.
650struct BreakerOpened {
651    consecutive_timeouts: u32,
652    /// True when a failed probe re-opened an already-open breaker, which reads
653    /// very differently in a log from a first opening.
654    reopened_after_probe: bool,
655}
656
657impl RouteBindBreakers {
658    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
659        self.modules
660            .lock()
661            .expect("route.bind breaker mutex poisoned")
662    }
663
664    /// Decide whether this `route.open` may attempt its relay. Takes the map
665    /// lock and nothing else, and never awaits.
666    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
667        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
668            probe: probe_token.is_some(),
669            guard: RouteBindBreakerGuard {
670                breakers: self.clone(),
671                module_id,
672                probe_token,
673                settled: false,
674            },
675        };
676
677        let mut modules = self.lock();
678        let Some(state) = modules.get_mut(module_id) else {
679            return admitted(None);
680        };
681        let Some(cooldown_until) = state.cooldown_until else {
682            return admitted(None);
683        };
684        if state.probe_in_flight.is_some() {
685            return RouteBindAdmission::Refused {
686                consecutive_timeouts: state.consecutive_timeouts,
687                retry_in: Duration::ZERO,
688                probe_in_flight: true,
689            };
690        }
691        let now = Instant::now();
692        if now < cooldown_until {
693            return RouteBindAdmission::Refused {
694                consecutive_timeouts: state.consecutive_timeouts,
695                retry_in: cooldown_until - now,
696                probe_in_flight: false,
697            };
698        }
699        let token = Arc::new(());
700        state.probe_in_flight = Some(Arc::clone(&token));
701        admitted(Some(token))
702    }
703
704    fn record_accepted(&self, module_id: &str) -> bool {
705        self.lock()
706            .remove(module_id)
707            .is_some_and(|state| state.cooldown_until.is_some())
708    }
709
710    fn record_timeout(
711        &self,
712        module_id: &str,
713        probe_token: Option<&Arc<()>>,
714        threshold: u32,
715        cooldown: Duration,
716    ) -> Option<BreakerOpened> {
717        let mut modules = self.lock();
718        let state = modules.entry(module_id.to_string()).or_default();
719        let was_open = state.cooldown_until.is_some();
720        let was_probe = Self::owns_probe(state, probe_token);
721        if was_probe {
722            state.probe_in_flight = None;
723        }
724        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
725        if state.consecutive_timeouts < threshold {
726            return None;
727        }
728        state.cooldown_until = Some(Instant::now() + cooldown);
729        Some(BreakerOpened {
730            consecutive_timeouts: state.consecutive_timeouts,
731            reopened_after_probe: was_open && was_probe,
732        })
733    }
734
735    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
736        // A relay can finish after the breaker was reset or after it opened
737        // again and started a new probe. Only the guard whose token matches the
738        // active probe may release it, so a late relay never frees a newer probe.
739        state
740            .probe_in_flight
741            .as_ref()
742            .zip(token)
743            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
744    }
745
746    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
747        if let Some(state) = self.lock().get_mut(module_id) {
748            if Self::owns_probe(state, probe_token) {
749                state.probe_in_flight = None;
750            }
751        }
752    }
753
754    /// Discard what was learned about a module, because the process it was
755    /// learned about is gone. Returns the discarded count when it was non-zero.
756    ///
757    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
758    /// `module_id` is a configuration identity that outlives any particular
759    /// child; what the breaker observed was the process behind the module
760    /// connection of the moment. When a new connection registers under that id
761    /// the verdict's subject no longer exists, so the verdict is stale by
762    /// construction rather than merely likely to be wrong. Keeping it would
763    /// apply a dead process's record to a live one, which is the same defect
764    /// class this breaker exists to stop the daemon committing.
765    ///
766    /// A half-open probe in flight is discarded with the rest: it was a
767    /// question about the old process.
768    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
769        self.lock()
770            .remove(module_id)
771            .map(|state| state.consecutive_timeouts)
772            .filter(|discarded| *discarded > 0)
773    }
774
775    /// Open breakers, for the `server.describe` counters object. `None` when
776    /// none is open, so the key stays absent rather than present-and-empty.
777    ///
778    /// This is the operator's answer to "is this module refusing instantly or
779    /// is it fine?", which look identical from a client that retries and then
780    /// succeeds.
781    fn open_snapshot(&self) -> Option<serde_json::Value> {
782        let now = Instant::now();
783        let modules = self.lock();
784        let open = modules
785            .iter()
786            .filter_map(|(module_id, state)| {
787                let cooldown_until = state.cooldown_until?;
788                Some((
789                    module_id.clone(),
790                    serde_json::json!({
791                        "consecutive_timeouts": state.consecutive_timeouts,
792                        "cooldown_remaining_ms":
793                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
794                        "probe_in_flight": state.probe_in_flight.is_some(),
795                    }),
796                ))
797            })
798            .collect::<serde_json::Map<String, serde_json::Value>>();
799        (!open.is_empty()).then_some(serde_json::Value::Object(open))
800    }
801}
802
803impl ControlHandler {
804    pub fn new(registry: Arc<Registry>) -> Self {
805        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
806    }
807
808    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
809        let counters = forwarding.counters();
810        // Taken from the forwarding table rather than created here, so that the
811        // breaker a `route.open` consults is the same one a module's
812        // registration resets, however many handlers are built over one table.
813        let route_bind_breakers = forwarding.route_bind_breakers();
814        let route_bind_concurrency = forwarding.route_bind_concurrency();
815        let route_outages = forwarding.route_outages();
816        Self {
817            registry,
818            activity: None,
819            activity_streams: Arc::new(Mutex::new(HashMap::new())),
820            forwarding,
821            process_liveness: None,
822            supervisor: SupervisorHandle::new(),
823            subc_capabilities: Arc::from([
824                CAP_MANIFEST_REGISTRATION.to_string(),
825                CAP_CHANNEL_LIFECYCLE.to_string(),
826                CAP_PING_PONG.to_string(),
827                CAP_SESSION_ATTACH.to_string(),
828                CAP_ADMISSION_FACTS_RELAY.to_string(),
829                CAP_SCOPES_V1.to_string(),
830                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
831                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
832                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
833            ]),
834            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
835            route_bind_relay_timeouts: BTreeMap::new(),
836            route_bind_breakers,
837            route_bind_concurrency,
838            route_outages,
839            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
840            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
841            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
842            storage_config: None,
843            machine_id: None,
844            admission_facts_carrier_module_id: None,
845            admission_facts_targets: None,
846            scopes: Arc::new(RwLock::new(ScopeTable::new(
847                crate::daemon_config::default_scope_authority_owners(),
848            ))),
849            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
850            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
851            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
852            rescan: None,
853            connected_clients: ConnectedClients::new(),
854            counters,
855            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
856            daemon_provenance: DaemonProvenanceFacts::default(),
857            #[cfg(test)]
858            control_dispatch_delay: None,
859            #[cfg(test)]
860            provenance_probe_override: None,
861            #[cfg(test)]
862            route_bind_relay_sent_hold: None,
863        }
864    }
865
866    /// Set the central storage policy: registering modules then receive their
867    /// resolved storage descriptor in HELLO_ACK.
868    pub fn with_storage_config(
869        mut self,
870        storage_config: Option<crate::daemon_config::StorageConfig>,
871    ) -> Self {
872        self.storage_config = storage_config;
873        self
874    }
875
876    /// Open the ledger on its own writer thread; an unavailable sink does not
877    /// prevent the daemon from serving routing or supervisor requests.
878    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
879        self.activity = Some(crate::activity::ActivityLedger::open(
880            run_dir.join("activity.jsonl"),
881            incarnation,
882        ));
883        self.supervisor.set_activity(self.activity.clone());
884        self
885    }
886
887    /// Set the machine id served to every registering module (HELLO_ACK) and on
888    /// `server.describe`.
889    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
890        self.machine_id = machine_id;
891        self
892    }
893
894    /// Configure the exact reserved module and target ids permitted to relay
895    /// opaque admission facts. Config-file loading validates this authority;
896    /// this builder keeps the same policy available to embedded test daemons.
897    pub fn with_admission_facts_config(
898        mut self,
899        carrier_module_id: Option<String>,
900        targets: Option<Vec<String>>,
901    ) -> Self {
902        self.admission_facts_carrier_module_id = carrier_module_id;
903        self.admission_facts_targets = targets;
904        self
905    }
906
907    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
908    /// Replaces the scope table with an empty one under the new list, so call it
909    /// while building the handler, before any module can sync.
910    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
911        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
912        self.scope_authority_owners = owners;
913        self
914    }
915
916    /// Override the route.bind relay timeout. Used by tests that assert the
917    /// timeout path so they don't block on the production-safe default.
918    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
919        self.route_bind_relay_timeout = timeout;
920        self
921    }
922
923    /// Install per-module route.bind relay budget overrides. A module id
924    /// listed here wins over the daemon-wide default set via
925    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
926    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
927    /// the same `Duration` the bind path will use.
928    pub fn with_route_bind_relay_timeouts(
929        mut self,
930        timeouts: impl IntoIterator<Item = (String, Duration)>,
931    ) -> Self {
932        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
933        self
934    }
935
936    /// Resolve the route.bind relay budget for a specific target module id.
937    /// Per-module overrides win; the daemon-wide value (set via
938    /// `with_route_bind_relay_timeout` or the built-in default) is the
939    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
940    /// the same resolution `handle_route_open` will use.
941    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
942        self.route_bind_relay_timeouts
943            .get(module_id)
944            .copied()
945            .unwrap_or(self.route_bind_relay_timeout)
946    }
947
948    /// Override the per-module bind-relay breaker policy.
949    ///
950    /// Used by tests, which cannot spend three production budgets opening a
951    /// breaker or twenty seconds waiting for its cooldown. The production
952    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
953    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
954    /// reasoning for the numbers.
955    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
956        self.route_bind_breaker_threshold = threshold.max(1);
957        self.route_bind_breaker_cooldown = cooldown;
958        self
959    }
960
961    #[cfg(test)]
962    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
963        self.health_probe_timeout = timeout;
964        self
965    }
966
967    #[cfg(test)]
968    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
969        self.control_dispatch_delay = Some(delay);
970        self
971    }
972
973    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
974    /// announce it is parked, once to be released. Build the barrier for two
975    /// parties, the test being the other one.
976    #[cfg(test)]
977    pub(crate) fn with_route_bind_relay_sent_hold(
978        mut self,
979        hold: Arc<tokio::sync::Barrier>,
980    ) -> Self {
981        self.route_bind_relay_sent_hold = Some(hold);
982        self
983    }
984
985    pub fn with_process_liveness(
986        mut self,
987        process_liveness: Arc<dyn ModuleProcessLiveness>,
988    ) -> Self {
989        self.process_liveness = Some(process_liveness);
990        self
991    }
992
993    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
994        self.supervisor = supervisor;
995        self.supervisor.set_activity(self.activity.clone());
996        self
997    }
998
999    pub fn with_daemon_provenance(
1000        mut self,
1001        pid: u32,
1002        started_at_ms: u64,
1003        executable_path: Option<PathBuf>,
1004        build_git_sha: Option<String>,
1005        build_lock_digest: Option<String>,
1006    ) -> Self {
1007        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1008        let process_start_time = process_start_time(pid);
1009        self.daemon_provenance = DaemonProvenanceFacts {
1010            build: DaemonBuildProvenance {
1011                build_git_sha,
1012                build_lock_digest,
1013            },
1014            pid: Some(pid),
1015            started_at_ms: Some(started_at_ms),
1016            start_clock: None,
1017            executable_path,
1018            executable_identity,
1019            process_start_time,
1020            probe: ExecutableIdentityProbe::default(),
1021        };
1022        self
1023    }
1024
1025    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1026        self.daemon_provenance.start_clock = Some(clock);
1027        self
1028    }
1029
1030    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1031    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1032        self.scopes
1033            .write()
1034            .unwrap_or_else(|poisoned| poisoned.into_inner())
1035            .set_wall_clock(Arc::new(clock));
1036        self
1037    }
1038
1039    #[cfg(test)]
1040    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1041        self.provenance_probe_override = Some(result);
1042        self
1043    }
1044
1045    /// Install the configured module set and its reserved capability bindings.
1046    /// Bindings are configuration-scoped and may point at a provider that has not
1047    /// been installed yet, so this does not require the bound module to exist.
1048    pub fn with_capability_config(
1049        self,
1050        modules: impl IntoIterator<Item = (String, bool)>,
1051        reserved_capabilities: BTreeMap<String, String>,
1052    ) -> Self {
1053        self.capability_evaluator
1054            .configure(modules, reserved_capabilities);
1055        self
1056    }
1057
1058    pub fn with_supervisor_rescan(
1059        mut self,
1060        supervisor: Supervisor,
1061        config_path: impl Into<PathBuf>,
1062        configured_port: Option<u16>,
1063    ) -> Self {
1064        self.rescan = Some(SupervisorRescanContext {
1065            supervisor,
1066            config_path: config_path.into(),
1067            configured_port,
1068            storage_config: self.storage_config.clone(),
1069            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1070            admission_facts_targets: self.admission_facts_targets.clone(),
1071            scope_authority_owners: self.scope_authority_owners.clone(),
1072        });
1073        self
1074    }
1075
1076    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1077        self.connected_clients = connected_clients;
1078        self
1079    }
1080
1081    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1082        Arc::clone(&self.forwarding)
1083    }
1084
1085    pub(crate) fn counters(&self) -> DaemonCounters {
1086        self.counters.clone()
1087    }
1088
1089    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1090    /// requirement event without depending on an operator polling a status command.
1091    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1092        tokio::spawn(async move {
1093            loop {
1094                self.capability_evaluator
1095                    .wait_for_change_or_deadline()
1096                    .await;
1097                self.refresh_capability_requirements();
1098            }
1099        });
1100    }
1101
1102    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1103    /// deadline passed during sleep is enforced on the first tick after wake.
1104    /// While no live scope carries a deadline the loop waits for an accepted
1105    /// scope change instead of ticking, so a daemon without deadlines never
1106    /// wakes for expiry.
1107    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1108        tokio::spawn(async move {
1109            loop {
1110                let has_deadlines = self
1111                    .scopes
1112                    .read()
1113                    .unwrap_or_else(|poisoned| poisoned.into_inner())
1114                    .has_deadlines();
1115                if !has_deadlines {
1116                    self.scope_deadline_added.notified().await;
1117                    continue;
1118                }
1119                tokio::time::sleep(Duration::from_secs(1)).await;
1120                if let Err(error) = self.sweep_expired_scopes() {
1121                    warn!(%error, "scope expiry sweep failed");
1122                }
1123            }
1124        });
1125    }
1126
1127    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1128        let mut table = self
1129            .scopes
1130            .write()
1131            .unwrap_or_else(|poisoned| poisoned.into_inner());
1132        let swept = table.sweep_expired();
1133        let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1134        drop(table);
1135        self.close_scope_drained_routes(drained);
1136        Ok(())
1137    }
1138
1139    fn publish_scope_changes(
1140        &self,
1141        changes: &[crate::scopes::ScopeTagChange],
1142        expired: &[crate::scopes::ScopeExpired],
1143    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1144        if changes.is_empty() {
1145            return Ok(Vec::new());
1146        }
1147        let drained = self
1148            .forwarding
1149            .publish_scope_changes(changes)
1150            .map_err(RouterError::Forwarding)?;
1151        let mut counts = HashMap::new();
1152        for route in &drained {
1153            *counts
1154                .entry((
1155                    route.scope.owner.as_str(),
1156                    route.scope.scope_ref.as_str(),
1157                    route.scope.tag.scope_epoch,
1158                ))
1159                .or_insert(0usize) += 1;
1160        }
1161        for scope in expired {
1162            let routes_closed = counts
1163                .get(&(
1164                    scope.owner.as_str(),
1165                    scope.scope_ref.as_str(),
1166                    scope.scope_epoch,
1167                ))
1168                .copied()
1169                .unwrap_or(0);
1170            warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1171                scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1172                routes_closed, "scope expired");
1173        }
1174        Ok(drained)
1175    }
1176
1177    fn runtime_capability_snapshot(
1178        &self,
1179    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1180        let runtime = self
1181            .supervisor
1182            .list()
1183            .into_iter()
1184            .map(|module| {
1185                let status = module.status().map_err(|err| {
1186                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1187                })?;
1188                Ok(RuntimeModule {
1189                    module_id: status.module_id,
1190                    state: status.state,
1191                    enabled: status.enabled,
1192                })
1193            })
1194            .collect::<Result<Vec<_>, RouterError>>()?;
1195        let (_, registrations) = self.registry.list_modules().map_err(|err| {
1196            RouterError::backend(
1197                0,
1198                0,
1199                format!("failed to list capability registrations: {err}"),
1200            )
1201        })?;
1202        let registrations = registrations
1203            .into_iter()
1204            .map(|registration| RegisteredModule {
1205                module_id: registration.manifest.module_id,
1206                module_version: registration.manifest.module_version,
1207                capabilities: registration.manifest.capabilities,
1208            })
1209            .collect();
1210        Ok((runtime, registrations))
1211    }
1212
1213    /// The capability side effects of a module becoming the active registration
1214    /// for its id: cache its manifest (warning if its claims drifted), run the
1215    /// deny census when its declarations call for one, and recompute the
1216    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1217    /// candidate's does not, and the supervisor does it at promotion instead,
1218    /// through [`crate::supervise::SwapPromotionObserver`].
1219    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1220        let cached_registration = RegisteredModule {
1221            module_id: registration.manifest.module_id.clone(),
1222            module_version: registration.manifest.module_version.clone(),
1223            capabilities: registration.manifest.capabilities.clone(),
1224        };
1225        if self.capability_evaluator.record_hello(&cached_registration) {
1226            warn!(
1227                module_id = %cached_registration.module_id,
1228                "capability claims drifted from the cached manifest"
1229            );
1230        }
1231        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1232            self.enforce_capability_denies();
1233        }
1234        self.refresh_capability_requirements();
1235    }
1236
1237    /// Point the shared supervisor handle at this handler for swap promotions.
1238    /// Called wherever a handler is put behind the `Arc` the router serves, so
1239    /// it can be held weakly.
1240    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1241        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1242            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1243        self.supervisor.set_swap_promotion_observer(observer);
1244    }
1245
1246    pub fn refresh_capability_requirements(&self) {
1247        match self.runtime_capability_snapshot() {
1248            Ok((runtime, registrations)) => {
1249                log_requirement_events(
1250                    self.capability_evaluator
1251                        .evaluate_now(&runtime, &registrations),
1252                );
1253            }
1254            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1255        }
1256    }
1257
1258    /// Reconcile only live, attested route bindings after a capability deny edge
1259    /// or target claim was added. This is deliberately a control-plane census:
1260    /// the opaque forwarding hot path must not grow a per-frame capability check.
1261    fn enforce_capability_denies(&self) {
1262        let (_, registrations) = match self.registry.list_modules() {
1263            Ok(snapshot) => snapshot,
1264            Err(err) => {
1265                warn!(error = %err, "failed to read registrations for capability deny census");
1266                return;
1267            }
1268        };
1269        let manifests = registrations
1270            .into_iter()
1271            .map(|registration| {
1272                (
1273                    registration.manifest.module_id.clone(),
1274                    registration.manifest,
1275                )
1276            })
1277            .collect::<BTreeMap<_, _>>();
1278        let census = match self.forwarding.route_census(None) {
1279            Ok(census) => census,
1280            Err(err) => {
1281                warn!(error = %err, "failed to read route census for capability deny enforcement");
1282                return;
1283            }
1284        };
1285
1286        for (target_module_id, routes) in census {
1287            let Some(target_manifest) = manifests.get(&target_module_id) else {
1288                continue;
1289            };
1290            let mut closed_routes = Vec::new();
1291            let mut module_goodbyes = Vec::new();
1292            for route in routes {
1293                let Principal::Reserved {
1294                    module_id: opening_module_id,
1295                } = &route.principal
1296                else {
1297                    continue;
1298                };
1299                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1300                    continue;
1301                };
1302                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1303                    continue;
1304                };
1305
1306                match self.forwarding.release_client_route(
1307                    route.goodbye_target.connection_id,
1308                    route.goodbye_target.channel,
1309                    route.goodbye_target.epoch,
1310                ) {
1311                    Ok(RouteRelease::Removed(module_goodbye)) => {
1312                        warn!(
1313                            opening_module_id,
1314                            target_module_id,
1315                            capability,
1316                            "force-closing route because an attested capability deny edge now matches"
1317                        );
1318                        closed_routes.push(route);
1319                        module_goodbyes.push(module_goodbye);
1320                    }
1321                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1322                    Err(err) => warn!(
1323                        opening_module_id,
1324                        target_module_id,
1325                        capability,
1326                        error = %err,
1327                        "failed to force-close capability-denied route"
1328                    ),
1329                }
1330            }
1331
1332            if closed_routes.is_empty() {
1333                continue;
1334            }
1335            send_route_control_pushes(
1336                &self.forwarding,
1337                closed_routes,
1338                ClientControlPush::RouteClosed {
1339                    module_id: target_module_id,
1340                    channels: Vec::new(),
1341                    reason: RouteCloseReason::CapabilityDenied,
1342                    drained: false,
1343                    abandoned: 0,
1344                    excluded_subscriptions: 0,
1345                    terminal: Some(false),
1346                },
1347            );
1348            self.emit_route_goodbyes(module_goodbyes);
1349        }
1350    }
1351
1352    /// Why a registered module is not accepting new route binds, or `None` when
1353    /// it is. This is the module's effective readiness: its declared readiness
1354    /// first, then every `need: required` capability it declares evaluating to
1355    /// `provided`. `route.open` and `catalog.list` both read it here so the
1356    /// catalog never reports a module routable that `route.open` would refuse.
1357    fn not_ready_reason(
1358        &self,
1359        registration: &crate::registry::ModuleRegistration,
1360    ) -> Option<NotReadyReason> {
1361        if !registration.ready {
1362            return Some(NotReadyReason {
1363                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1364                capability: None,
1365            });
1366        }
1367        self.first_unprovided_required_capability(registration)
1368            .map(|capability| NotReadyReason {
1369                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1370                capability: Some(capability),
1371            })
1372    }
1373
1374    /// The lexicographically first capability this registration declares
1375    /// `need: required` whose evaluator verdict is not `provided`.
1376    ///
1377    /// The verdicts are the capability evaluator's own; nothing here decides
1378    /// what "provided" means. The evaluator counts a capability provided as
1379    /// soon as a module claiming it has REGISTERED, not once that module is
1380    /// ready. That distinction is what keeps two modules that require each
1381    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1382    /// is ready", each would wait for the other to become ready first and
1383    /// neither ever would. Do not tighten it to readiness.
1384    ///
1385    /// A required capability with no verdict at all means this registration's
1386    /// HELLO or catalog.update landed after the last recompute; recompute once
1387    /// rather than let a missing verdict read as either answer. If it is still
1388    /// missing (the recompute itself failed) the capability counts as
1389    /// unprovided: the refusal is retryable, and routing a module whose
1390    /// required provider is unknown is the outcome this check exists to stop.
1391    fn first_unprovided_required_capability(
1392        &self,
1393        registration: &crate::registry::ModuleRegistration,
1394    ) -> Option<String> {
1395        let required = registration
1396            .manifest
1397            .capabilities
1398            .iter()
1399            .flat_map(|declarations| declarations.requires.iter())
1400            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1401            .map(|requirement| requirement.capability.as_str())
1402            .collect::<BTreeSet<_>>();
1403        if required.is_empty() {
1404            return None;
1405        }
1406        let module_id = registration.manifest.module_id.as_str();
1407        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1408        if required
1409            .iter()
1410            .any(|capability| verdict(capability).is_none())
1411        {
1412            self.refresh_capability_requirements();
1413        }
1414        required
1415            .into_iter()
1416            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1417            .map(str::to_string)
1418    }
1419
1420    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1421        self.capability_evaluator
1422            .statuses()
1423            .into_iter()
1424            .map(capability_requirement_status)
1425            .collect()
1426    }
1427
1428    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1429    /// registration-release watch. The signal is what the supervisor waits on
1430    /// before spawning a replacement, so it must only fire once forwarding
1431    /// teardown is also done (see [`Self::cleanup_connection`] /
1432    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1433    /// admitted but its forwarding endpoint could not be installed.
1434    fn deregister_connection(
1435        &self,
1436        connection_id: ConnectionId,
1437        reason: RegistrationEndReason,
1438    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1439        self.registry
1440            .deregister_connection_with_reason(connection_id, reason)
1441    }
1442
1443    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1444        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1445            return None;
1446        }
1447        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1448            parse_client_control_request(&frame.body)
1449        else {
1450            return None;
1451        };
1452        Some(target_module_id(&target).to_string())
1453    }
1454
1455    pub(crate) fn route_open_capacity_refusal(
1456        &self,
1457        ctx: &RouteCtx,
1458        frame: &Frame,
1459        target_module_id: &str,
1460        in_flight: usize,
1461        limit: usize,
1462    ) -> Result<Frame, RouterError> {
1463        self.route_open_admission_refusal_frame(
1464            ctx,
1465            frame,
1466            target_module_id,
1467            "open_admission_full",
1468            (in_flight, limit),
1469            format!(
1470                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1471            ),
1472        )
1473    }
1474
1475    fn route_open_target_capacity_refusal(
1476        &self,
1477        ctx: &RouteCtx,
1478        frame: &Frame,
1479        target_module_id: &str,
1480        in_flight: usize,
1481    ) -> Result<Frame, RouterError> {
1482        self.route_open_admission_refusal_frame(
1483            ctx,
1484            frame,
1485            target_module_id,
1486            "target_binds_full",
1487            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1488            format!(
1489                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1490            ),
1491        )
1492    }
1493
1494    /// Admission pressure clears as existing binds settle, so its refusal must
1495    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1496    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1497    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1498    /// cannot currently reach its target; `module_timeout` would falsely claim
1499    /// that a wait expired. A new, cleaner code would be terminal to deployed
1500    /// clients, so it requires a client-tolerance rollout before daemon emission.
1501    fn route_open_admission_refusal_frame(
1502        &self,
1503        ctx: &RouteCtx,
1504        frame: &Frame,
1505        target_module_id: &str,
1506        reason: &'static str,
1507        (in_flight, limit): (usize, usize),
1508        message: impl Into<String>,
1509    ) -> Result<Frame, RouterError> {
1510        let code = error_codes::TARGET_UNAVAILABLE;
1511        self.counters.increment_route_open_refused(code);
1512        info!(
1513            target: "control",
1514            code,
1515            reason,
1516            module_id = ?target_module_id,
1517            connection_id = ctx.connection_id.get(),
1518            in_flight,
1519            limit,
1520            "route.open refused"
1521        );
1522        control_error_frame(frame, code, message.into())
1523    }
1524
1525    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1526    ///
1527    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1528    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1529    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1530    #[cfg(test)]
1531    pub fn handle_control(
1532        &self,
1533        connection_id: ConnectionId,
1534        frame: Frame,
1535    ) -> Result<Vec<Frame>, RouterError> {
1536        match frame.header.ty {
1537            FrameType::Ping => Ok(vec![pong(&frame)?]),
1538            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1539            FrameType::Goodbye => self.handle_goodbye(connection_id),
1540            ty => Ok(vec![control_error_frame(
1541                &frame,
1542                "unsupported_control_frame",
1543                format!("unsupported channel-0 frame {ty:?}"),
1544            )?]),
1545        }
1546    }
1547
1548    pub async fn handle_control_frame(
1549        &self,
1550        ctx: &RouteCtx,
1551        frame: Frame,
1552    ) -> Result<Vec<Frame>, RouterError> {
1553        self.handle_control_frame_timed(ctx, frame, None).await
1554    }
1555
1556    pub(crate) async fn handle_control_frame_timed(
1557        &self,
1558        ctx: &RouteCtx,
1559        frame: Frame,
1560        dispatch_started_at: Option<StdInstant>,
1561    ) -> Result<Vec<Frame>, RouterError> {
1562        match frame.header.ty {
1563            FrameType::Ping => Ok(vec![pong(&frame)?]),
1564            FrameType::Hello => {
1565                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1566            }
1567            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1568            FrameType::Cancel => {
1569                // A Cancel on channel 0 names a waiting operator.confirm or a
1570                // subscription. Successful cancellation has no ordinary reply.
1571                if self
1572                    .forwarding
1573                    .operator_confirms()
1574                    .cancel(ctx.connection_id, frame.header.corr)
1575                    || self
1576                        .supervisor
1577                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1578                    || self.cancel_activity_subscription(ctx, &frame)?
1579                {
1580                    Ok(Vec::new())
1581                } else {
1582                    Ok(vec![control_error_frame(
1583                        &frame,
1584                        "unknown_subscription",
1585                        "no subscription has this correlation id",
1586                    )?])
1587                }
1588            }
1589            FrameType::Request => {
1590                // This additive operation is not part of the existing exhaustive
1591                // module-control enum. Probe the op before decoding that enum.
1592                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1593                if op
1594                    .as_ref()
1595                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1596                {
1597                    return self.handle_activity_record(ctx, frame).await;
1598                }
1599                if op.as_ref().is_some_and(|probe| {
1600                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1601                }) {
1602                    return self.handle_activity_read(ctx, frame).await;
1603                }
1604                if op
1605                    .as_ref()
1606                    .is_some_and(|probe| probe.op == "operator.confirm")
1607                {
1608                    return self.handle_operator_confirm(ctx, frame);
1609                }
1610                if self
1611                    .forwarding
1612                    .module_endpoint_for_connection(ctx.connection_id)
1613                    .map_err(RouterError::Forwarding)?
1614                    .is_some()
1615                {
1616                    if !is_known_module_request_op(&frame.body) {
1617                        return Ok(vec![control_error_frame(
1618                            &frame,
1619                            "unsupported_control_frame",
1620                            "module-originated channel-0 REQUEST is not supported",
1621                        )?]);
1622                    }
1623                    let request = match parse_module_control_request_from_module(&frame.body) {
1624                        Ok(request) => request,
1625                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1626                            return Ok(vec![control_error_frame(
1627                                &frame,
1628                                "unsupported_control_frame",
1629                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1630                            )?])
1631                        }
1632                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1633                            return Ok(vec![control_error_frame(
1634                                &frame,
1635                                "invalid_control_body",
1636                                format!("malformed module control body: {err}"),
1637                            )?])
1638                        }
1639                    };
1640                    let op = module_control_request_op(&request);
1641                    let corr = frame.header.corr;
1642                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1643                    let result =
1644                        self.handle_module_control_request(ctx.connection_id, frame, request);
1645                    log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1646                    return result;
1647                }
1648
1649                if is_known_module_request_op(&frame.body) {
1650                    return Ok(vec![control_error_frame(
1651                        &frame,
1652                        "not_registered",
1653                        "catalog.update requires an active module registration owned by this connection",
1654                    )?]);
1655                }
1656
1657                let request = match parse_client_control_request(&frame.body) {
1658                    Ok(request) => request,
1659                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1660                        return Ok(vec![control_error_frame(
1661                            &frame,
1662                            "unknown_control_op",
1663                            format!("unknown client control op: {err}"),
1664                        )?])
1665                    }
1666                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1667                        return Ok(vec![control_error_frame(
1668                            &frame,
1669                            "invalid_control_body",
1670                            format!("malformed client control body: {err}"),
1671                        )?])
1672                    }
1673                };
1674                let op = client_control_request_op(&request);
1675                let corr = frame.header.corr;
1676                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1677                #[cfg(test)]
1678                if let Some(delay) = self.control_dispatch_delay {
1679                    tokio::time::sleep(delay).await;
1680                }
1681                let result = self
1682                    .handle_client_control_request(ctx, frame, request)
1683                    .await;
1684                log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1685                result
1686            }
1687            FrameType::Push => {
1688                let Some(endpoint) = self
1689                    .forwarding
1690                    .module_endpoint_for_connection(ctx.connection_id)
1691                    .map_err(RouterError::Forwarding)?
1692                else {
1693                    return Ok(vec![control_error_frame(
1694                        &frame,
1695                        "unsupported_control_frame",
1696                        "client-originated channel-0 PUSH is not supported",
1697                    )?]);
1698                };
1699                self.handle_status_update(endpoint, frame)
1700            }
1701            FrameType::Response | FrameType::Error
1702                if self
1703                    .forwarding
1704                    .module_endpoint_for_connection(ctx.connection_id)
1705                    .map_err(RouterError::Forwarding)?
1706                    .is_some() =>
1707            {
1708                self.handle_module_relay_response(ctx.connection_id, frame)
1709            }
1710            ty => Ok(vec![control_error_frame(
1711                &frame,
1712                "unsupported_control_frame",
1713                format!("unsupported channel-0 frame {ty:?}"),
1714            )?]),
1715        }
1716    }
1717
1718    pub fn cleanup_connection(
1719        &self,
1720        connection_id: ConnectionId,
1721    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1722        self.cleanup_connection_with_end_reason(
1723            connection_id,
1724            RegistrationEndReason::ConnectionClosed,
1725        )
1726    }
1727
1728    fn cleanup_connection_with_end_reason(
1729        &self,
1730        connection_id: ConnectionId,
1731        requested_reason: RegistrationEndReason,
1732    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1733        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1734            self.registry
1735                .get_module_by_connection(connection_id)?
1736                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
1737                .and_then(|module| module.registration_end_reason().ok().flatten())
1738                .unwrap_or(requested_reason)
1739        } else {
1740            requested_reason
1741        };
1742        let crash_closed = self
1743            .registry
1744            .get_module_by_connection(connection_id)?
1745            .and_then(|registration| {
1746                self.forwarding
1747                    .module_endpoint_for_connection(connection_id)
1748                    .ok()
1749                    .flatten()
1750                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1751                    .map(|routes| (registration.manifest.module_id, routes))
1752            });
1753        let crash_closed = crash_closed.map(|(module_id, routes)| {
1754            let terminal = match self.supervisor.get(&module_id) {
1755                None => false,
1756                Some(module) => match module.will_recover_after_connection_loss() {
1757                    Ok(will_recover) => !will_recover,
1758                    Err(err) => {
1759                        warn!(
1760                            %module_id,
1761                            error = %err,
1762                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
1763                        );
1764                        false
1765                    }
1766                },
1767            };
1768            // The forwarding table gates all providers at the start of daemon
1769            // shutdown, before their connections are closed. An ordinary
1770            // module disconnect still reports crash if that gate is not set.
1771            let (reason, drained) = match self
1772                .forwarding
1773                .daemon_drain_result_for_connection(connection_id)
1774            {
1775                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
1776                Ok(None) => (RouteCloseReason::Crash, false),
1777                Err(err) => {
1778                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1779                    (RouteCloseReason::Crash, false)
1780                }
1781            };
1782            (module_id, routes, reason, drained, terminal)
1783        });
1784        let registrations = self.deregister_connection(connection_id, end_reason);
1785        let cleanup = if crash_closed.is_some() {
1786            self.forwarding.cleanup_connection_counted(connection_id)
1787        } else {
1788            // A module connection's teardown needs the count of abandoned
1789            // route.bind relays for its route.closed notice below. Any other
1790            // connection, such as a client's, sends no such notice and needs
1791            // only its routes released, so it uses the route-only wrapper and
1792            // reports zero.
1793            self.forwarding
1794                .cleanup_connection(connection_id)
1795                .map(|released| crate::forwarding::ConnectionCleanup {
1796                    released,
1797                    abandoned_relays: 0,
1798                })
1799        };
1800        // The route.closed push waits for forwarding teardown because only
1801        // teardown knows how many pending route.bind relays it aborted. It still
1802        // goes out before the GOODBYEs for the released routes, and its targets
1803        // were captured above, before teardown removed those routes.
1804        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
1805            let abandoned = cleanup
1806                .as_ref()
1807                .map_or(0, |cleanup| cleanup.abandoned_relays);
1808            send_route_control_pushes(
1809                &self.forwarding,
1810                routes,
1811                ClientControlPush::RouteClosed {
1812                    module_id,
1813                    channels: Vec::new(),
1814                    reason,
1815                    drained,
1816                    abandoned,
1817                    excluded_subscriptions: 0,
1818                    terminal: Some(terminal),
1819                },
1820            );
1821        }
1822        if let Ok(cleanup) = cleanup {
1823            self.emit_route_goodbyes(cleanup.released);
1824        }
1825        // Signal the registration-release watch only now that BOTH registry and
1826        // forwarding teardown are done, so a supervisor waiting to spawn a
1827        // replacement never observes release while old routes still exist.
1828        if matches!(&registrations, Ok(r) if !r.is_empty()) {
1829            crate::supervise::notify_registration_release();
1830            self.capability_evaluator.wake_deadline_loop();
1831            self.refresh_capability_requirements();
1832        }
1833        self.supervisor.remove_spawn_subscribers(connection_id);
1834        self.activity_streams
1835            .lock()
1836            .unwrap_or_else(|p| p.into_inner())
1837            .retain(|(id, _), task| {
1838                if *id == connection_id {
1839                    task.abort();
1840                    false
1841                } else {
1842                    true
1843                }
1844            });
1845        // Sync authority dies with its connection, so the owner's next
1846        // connection can take it; the owner's scopes stay as they are.
1847        self.hello_launch_nonces
1848            .lock()
1849            .unwrap_or_else(|poisoned| poisoned.into_inner())
1850            .forget(connection_id);
1851        self.scopes
1852            .write()
1853            .unwrap_or_else(|poisoned| poisoned.into_inner())
1854            .release_connection(connection_id);
1855        registrations
1856    }
1857
1858    pub(crate) fn handle_route_goodbye(
1859        &self,
1860        connection_id: ConnectionId,
1861        route_channel: u16,
1862        route_epoch: u32,
1863    ) -> Result<bool, RouterError> {
1864        debug!(
1865            connection_id = connection_id.get(),
1866            route_channel, route_epoch, "handling route GOODBYE"
1867        );
1868        let RouteRelease::Removed(released_route) = self
1869            .forwarding
1870            .release_client_route(connection_id, route_channel, route_epoch)
1871            .map_err(RouterError::Forwarding)?
1872        else {
1873            return Ok(false);
1874        };
1875        self.emit_route_goodbyes(vec![released_route]);
1876        Ok(true)
1877    }
1878
1879    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1880        for released in released_routes {
1881            let frame = match Frame::build_with_version(
1882                released.negotiated_ver,
1883                FrameType::Goodbye,
1884                control_flags(),
1885                released.channel,
1886                released.epoch,
1887                0,
1888                Vec::new(),
1889            ) {
1890                Ok(frame) => frame,
1891                Err(err) => {
1892                    warn!(
1893                        route_channel = released.channel,
1894                        error = %err,
1895                        "failed to build route GOODBYE frame"
1896                    );
1897                    continue;
1898                }
1899            };
1900            if !released.close_on_delivery_failure() {
1901                crate::forwarding::send_module_route_goodbye(
1902                    &self.counters,
1903                    &released.sink,
1904                    frame,
1905                    released.module_id.as_deref(),
1906                    "client route released",
1907                );
1908                continue;
1909            }
1910            if let Err(err) = released.sink.try_send(frame) {
1911                warn!(
1912                    target_connection_id = released.connection_id.get(),
1913                    route_channel = released.channel,
1914                    error = %err,
1915                    "route GOODBYE was not delivered to client; closing target connection"
1916                );
1917                if self
1918                    .forwarding
1919                    .escalate_client_delivery_failure(
1920                        released.connection_id,
1921                        released.channel,
1922                        released.epoch,
1923                        CloseReason::new(
1924                            "route_goodbye_delivery_failed",
1925                            format!(
1926                                "failed to enqueue route GOODBYE for channel {}: {err}",
1927                                released.channel
1928                            ),
1929                        ),
1930                        crate::forwarding::UndeliveredFrame {
1931                            module_id: released.module_id.as_deref(),
1932                            sink: &released.sink,
1933                        },
1934                    )
1935                    .unwrap_or(false)
1936                {
1937                    self.counters.increment_goodbye_relay_client_failed();
1938                }
1939            }
1940        }
1941    }
1942
1943    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
1944    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
1945    /// own commit failed after the module had already accepted). Without this, a
1946    /// module that accepts late keeps a binding subc has torn down, so a later
1947    /// frame on that module channel could misdeliver if the channel is reused.
1948    ///
1949    /// Never closes the shared module connection on failure: a dropped notification
1950    /// only wastes a bounded amount of warm module-side state, which the module's
1951    /// own idle reaper reclaims. Only call this once the route.bind relay was
1952    /// actually enqueued to the module — if the relay send itself failed, the
1953    /// module never created a binding and there is nothing to tear down.
1954    fn send_abandoned_route_bind_goodbye(
1955        &self,
1956        module_sink: &crate::FrameSink,
1957        negotiated_ver: u8,
1958        module_channel: u16,
1959        module_epoch: u32,
1960    ) {
1961        let frame = match Frame::build_with_version(
1962            negotiated_ver,
1963            FrameType::Goodbye,
1964            control_flags(),
1965            module_channel,
1966            module_epoch,
1967            0,
1968            Vec::new(),
1969        ) {
1970            Ok(frame) => frame,
1971            Err(err) => {
1972                warn!(
1973                    route_channel = module_channel,
1974                    error = %err,
1975                    "failed to build GOODBYE for abandoned route.bind"
1976                );
1977                return;
1978            }
1979        };
1980        crate::forwarding::send_module_route_goodbye(
1981            &self.counters,
1982            module_sink,
1983            frame,
1984            None,
1985            "abandoned route.bind",
1986        );
1987    }
1988
1989    fn handle_hello(
1990        &self,
1991        connection_id: ConnectionId,
1992        sink: Option<crate::FrameSink>,
1993        frame: Frame,
1994    ) -> Result<Vec<Frame>, RouterError> {
1995        debug!(
1996            connection_id = connection_id.get(),
1997            corr = frame.header.corr,
1998            "handling HELLO"
1999        );
2000        // A module connection has one identity for its entire lifetime. A second
2001        // registration would leave the old registry owner behind while replacing
2002        // its forwarding endpoint and launch nonce.
2003        if self
2004            .registry
2005            .get_module_by_connection(connection_id)
2006            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2007            .is_some()
2008        {
2009            return Ok(vec![control_error_frame(
2010                &frame,
2011                "invalid_hello",
2012                "connection is already registered as a module",
2013            )?]);
2014        }
2015        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2016            Ok(value) => value,
2017            Err(err) => {
2018                return Ok(vec![control_error_frame(
2019                    &frame,
2020                    "invalid_hello",
2021                    format!("malformed HELLO body: {err}"),
2022                )?])
2023            }
2024        };
2025        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2026            return Ok(vec![control_error_frame(
2027                &frame,
2028                "invalid_capability_grammar",
2029                err.to_string(),
2030            )?]);
2031        }
2032        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2033            return Ok(vec![control_error_frame(
2034                &frame,
2035                "invalid_manifest",
2036                err.to_string(),
2037            )?]);
2038        }
2039        if let Err(err) = validate_hello_event_declarations(&hello_value) {
2040            return Ok(vec![control_error_frame(
2041                &frame,
2042                "invalid_event_declaration",
2043                err.to_string(),
2044            )?]);
2045        }
2046        if let Some(provenance) = hello_value
2047            .get("manifest")
2048            .and_then(|manifest| manifest.get("provenance"))
2049        {
2050            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2051                return Ok(vec![control_error_frame(
2052                    &frame,
2053                    "invalid_manifest",
2054                    format!("malformed manifest provenance: {err}"),
2055                )?]);
2056            }
2057        }
2058        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2059            Ok(hello) => hello,
2060            Err(err) => {
2061                return Ok(vec![control_error_frame(
2062                    &frame,
2063                    "invalid_hello",
2064                    format!("malformed HELLO body: {err}"),
2065                )?])
2066            }
2067        };
2068
2069        if hello.protocol_ver != hello.manifest.protocol_ver {
2070            return Ok(vec![control_error_frame(
2071                &frame,
2072                "invalid_manifest",
2073                format!(
2074                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2075                    hello.protocol_ver, hello.manifest.protocol_ver
2076                ),
2077            )?]);
2078        }
2079
2080        if hello.manifest.module_id.trim().is_empty() {
2081            return Ok(vec![control_error_frame(
2082                &frame,
2083                "invalid_manifest",
2084                "manifest module_id must not be empty",
2085            )?]);
2086        }
2087
2088        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2089            Ok(negotiated_ver) => negotiated_ver,
2090            Err(message) => {
2091                return Ok(vec![control_error_frame(
2092                    &frame,
2093                    "version_unsupported",
2094                    message,
2095                )?])
2096            }
2097        };
2098
2099        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
2100        // swap is open for this id, the only HELLO admitted as a second process
2101        // is the one carrying the candidate's launch nonce (the swap token), and
2102        // it registers into the candidate slot rather than being refused as a
2103        // duplicate. Run after the reserved gate, a reserved module's candidate
2104        // would be refused `reserved_module` for presenting a nonce that gate
2105        // does not know. See `SupervisorHandle::swap_hello_admission`.
2106        let swap_admission = self
2107            .supervisor
2108            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2109        if swap_admission == SwapHelloAdmission::Refused {
2110            warn!(
2111                module_id = %hello.manifest.module_id,
2112                connection_id = connection_id.get(),
2113                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2114            );
2115            return Ok(vec![control_error_frame(
2116                &frame,
2117                "swap_token_invalid",
2118                format!(
2119                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2120                    hello.manifest.module_id
2121                ),
2122            )?]);
2123        }
2124        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2125
2126        // Reserved-module identity gate: a module_id configured `reserved` may be
2127        // registered ONLY by the process subc spawned for it, proven by echoing the
2128        // one-time launch nonce subc injected. A non-reserved id has no recorded
2129        // nonce and always passes. This blocks a key-holder from impersonating a
2130        // security-boundary module (e.g. the credential vault) while the real one is
2131        // down/restarting and its registration slot is momentarily free. A swap
2132        // candidate has already proven the same thing with its own nonce above.
2133        if let Some(rejection) = (!swap_candidate)
2134            .then(|| {
2135                self.supervisor.reserved_hello_rejection(
2136                    &hello.manifest.module_id,
2137                    hello.launch_nonce.as_deref(),
2138                )
2139            })
2140            .flatten()
2141        {
2142            let message = match rejection {
2143                ReservedHelloRejection::Exact { module_id } => format!(
2144                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2145                ),
2146                ReservedHelloRejection::Prefix {
2147                    prefix,
2148                    owner_module_id,
2149                } => format!(
2150                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2151                    hello.manifest.module_id
2152                ),
2153            };
2154            return Ok(vec![control_error_frame(
2155                &frame,
2156                "reserved_module",
2157                message,
2158            )?]);
2159        }
2160
2161        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2162            &hello.manifest.module_id,
2163            hello.manifest.capabilities.as_ref(),
2164        );
2165        if let Some(refusal) = reserved_capability_refusals.first() {
2166            let capability = refusal.capability.clone();
2167            let bound_module = refusal.claimants[0].clone();
2168            log_duplicate_claim_events(reserved_capability_refusals);
2169            return Ok(vec![control_error_frame(
2170                &frame,
2171                "reserved_capability",
2172                format!(
2173                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2174                    capability, bound_module, hello.manifest.module_id
2175                ),
2176            )?]);
2177        }
2178
2179        // A connection that already opened client routes must not also register as
2180        // a module: cleanup would then release only one side and leak the other.
2181        if self
2182            .forwarding
2183            .connection_has_client_routes(connection_id)
2184            .map_err(RouterError::Forwarding)?
2185        {
2186            return Ok(vec![control_error_frame(
2187                &frame,
2188                "invalid_hello",
2189                "connection has open client routes and cannot also register as a module",
2190            )?]);
2191        }
2192
2193        // Kept for scope sync authority, which goes only to the connection that
2194        // presented the module's current launch nonce. Recorded before the
2195        // registration is attempted: a connection whose registration then fails
2196        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2197        self.hello_launch_nonces
2198            .lock()
2199            .unwrap_or_else(|poisoned| poisoned.into_inner())
2200            .record(connection_id, hello.launch_nonce.as_deref());
2201        let control_ops = effective_module_control_ops(hello.control_ops);
2202        // Built before anything is registered so an encoding failure leaves no
2203        // registry or forwarding state behind.
2204        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2205        if swap_candidate {
2206            return self.register_swap_candidate(
2207                connection_id,
2208                sink,
2209                &frame,
2210                hello.manifest,
2211                negotiated_ver,
2212                control_ops,
2213                hello_ack,
2214            );
2215        }
2216        let registration = match self.registry.register_with_control_ops(
2217            hello.manifest,
2218            negotiated_ver,
2219            connection_id,
2220            control_ops,
2221        ) {
2222            Ok(registration) => registration,
2223            Err(RegistryError::DuplicateModuleId { module_id }) => {
2224                return Ok(vec![control_error_frame(
2225                    &frame,
2226                    "duplicate_module_id",
2227                    format!(
2228                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2229                    ),
2230                )?])
2231            }
2232            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2233                return Ok(vec![control_error_frame(
2234                    &frame,
2235                    "invalid_module_id",
2236                    err.to_string(),
2237                )?])
2238            }
2239            Err(err) => {
2240                return Ok(vec![control_error_frame(
2241                    &frame,
2242                    "registry_error",
2243                    err.to_string(),
2244                )?])
2245            }
2246        };
2247
2248        let reply = if let Some(sink) = sink {
2249            // The forwarding table's module store is also the daemon-to-module
2250            // control-RPC lane, so every HELLO gets a live endpoint even when the
2251            // manifest has no routable provider role. Non-routable modules still
2252            // cannot receive route.bind in production: `handle_route_open` checks
2253            // the registry manifest with `target_has_required_role` before the
2254            // only production call to `begin_route_bind_relay_for` below that
2255            // route.open path. The remaining direct relay callers are unit tests
2256            // and benchmark harnesses that construct forwarding state explicitly.
2257            //
2258            // The HELLO_ACK is queued by the forwarding table itself, before the
2259            // endpoint becomes visible, and is NOT returned as a reply. A module
2260            // reads HELLO_ACK first and exits on anything else; a reply is only
2261            // written after this handler returns, by which time a route.open on
2262            // another connection could already have queued a route.bind request
2263            // for this module ahead of it.
2264            let concurrency = manifest_concurrency(&registration.manifest);
2265            if let Err(err) = self.forwarding.register_module_connection_acked(
2266                connection_id,
2267                registration.manifest.module_id.clone(),
2268                negotiated_ver,
2269                concurrency,
2270                sink,
2271                hello_ack,
2272            ) {
2273                // Forwarding registration failed, so there is no forwarding
2274                // state to tear down. Remove the registry entry and signal the
2275                // release watch directly.
2276                if matches!(
2277                    self.deregister_connection(
2278                        connection_id,
2279                        RegistrationEndReason::RegistrationFailed,
2280                    ),
2281                    Ok(r) if !r.is_empty()
2282                ) {
2283                    crate::supervise::notify_registration_release();
2284                }
2285                return Ok(vec![control_error_frame(
2286                    &frame,
2287                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2288                        "invalid_hello"
2289                    } else {
2290                        forwarding_error_code(&err)
2291                    },
2292                    err.to_string(),
2293                )?]);
2294            }
2295            Vec::new()
2296        } else {
2297            // No sink means no forwarding endpoint, so nothing can be routed
2298            // ahead of the ack; it goes out as the reply.
2299            vec![hello_ack]
2300        };
2301
2302        // Exposure over assumption: Concurrency's serde default is pinned to the
2303        // pre-field behavior (ModuleManaged), so a management surface that is
2304        // genuinely Serial and just never declared it inherits concurrent
2305        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2306        // turns "no module has been bitten yet" into the checkable claim "no
2307        // module is exposed" -- one read of the boot log instead of a fleet
2308        // audit. Detected from the raw HELLO bytes because the serde default
2309        // deliberately erases the absent/declared distinction from the type.
2310        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2311            info!(
2312                module_id = %registration.manifest.module_id,
2313                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2314            );
2315        }
2316
2317        self.apply_registration_capabilities(&registration);
2318
2319        self.supervisor
2320            .record_activity_registration(&registration.manifest.module_id);
2321
2322        info!(
2323            module_id = %registration.manifest.module_id,
2324            module_version = %registration.manifest.module_version,
2325            negotiated_ver,
2326            routable_provider = manifest_provides_routable_role(&registration.manifest),
2327            connection_id = connection_id.get(),
2328            "module registered"
2329        );
2330
2331        Ok(reply)
2332    }
2333
2334    /// Register a HELLO the swap gate admitted into the candidate slot of the
2335    /// registry and of forwarding, where it is reachable over its own
2336    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2337    /// nothing routes to it until the supervisor cuts over.
2338    ///
2339    /// Registry first, then forwarding, the same order as an ordinary HELLO;
2340    /// a forwarding failure removes the registry entry again. The capability
2341    /// census is not run: it describes routable modules, and this one is not
2342    /// routable until promotion.
2343    #[allow(clippy::too_many_arguments)]
2344    fn register_swap_candidate(
2345        &self,
2346        connection_id: ConnectionId,
2347        sink: Option<crate::FrameSink>,
2348        frame: &Frame,
2349        manifest: ModuleManifest,
2350        negotiated_ver: u8,
2351        control_ops: Vec<String>,
2352        hello_ack: Frame,
2353    ) -> Result<Vec<Frame>, RouterError> {
2354        let module_id = manifest.module_id.clone();
2355        let registration = match self.registry.register_candidate_with_control_ops(
2356            manifest,
2357            negotiated_ver,
2358            connection_id,
2359            control_ops,
2360        ) {
2361            Ok(registration) => registration,
2362            Err(RegistryError::DuplicateModuleId { module_id }) => {
2363                return Ok(vec![control_error_frame(
2364                    frame,
2365                    "duplicate_module_id",
2366                    format!(
2367                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2368                    ),
2369                )?])
2370            }
2371            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2372                return Ok(vec![control_error_frame(
2373                    frame,
2374                    "invalid_module_id",
2375                    err.to_string(),
2376                )?])
2377            }
2378            Err(err) => {
2379                return Ok(vec![control_error_frame(
2380                    frame,
2381                    "registry_error",
2382                    err.to_string(),
2383                )?])
2384            }
2385        };
2386        let reply = if let Some(sink) = sink {
2387            // Same ordering as an ordinary HELLO: the forwarding table queues
2388            // the HELLO_ACK before the candidate endpoint is inserted, because
2389            // a module exits if its first frame after HELLO is anything else.
2390            let concurrency = manifest_concurrency(&registration.manifest);
2391            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2392                connection_id,
2393                module_id.clone(),
2394                negotiated_ver,
2395                concurrency,
2396                sink,
2397                hello_ack,
2398            ) {
2399                if matches!(
2400                    self.deregister_connection(
2401                        connection_id,
2402                        RegistrationEndReason::RegistrationFailed,
2403                    ),
2404                    Ok(r) if !r.is_empty()
2405                ) {
2406                    crate::supervise::notify_registration_release();
2407                }
2408                return Ok(vec![control_error_frame(
2409                    frame,
2410                    forwarding_error_code(&err),
2411                    err.to_string(),
2412                )?]);
2413            }
2414            Vec::new()
2415        } else {
2416            vec![hello_ack]
2417        };
2418        self.supervisor.mark_swap_candidate_admitted(&module_id);
2419        info!(
2420            module_id = %module_id,
2421            module_version = %registration.manifest.module_version,
2422            negotiated_ver,
2423            ready = registration.ready,
2424            connection_id = connection_id.get(),
2425            "swap candidate registered; not routable until cutover"
2426        );
2427        Ok(reply)
2428    }
2429
2430    fn build_hello_ack(
2431        &self,
2432        frame: &Frame,
2433        negotiated_ver: u8,
2434        module_id: &str,
2435    ) -> Result<Frame, RouterError> {
2436        let ack = ModuleHelloAckBody {
2437            negotiated_ver,
2438            subc_ops: module_subc_ops(),
2439            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2440            storage: self
2441                .storage_config
2442                .as_ref()
2443                .map(|cfg| cfg.descriptor_for(module_id)),
2444            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2445        };
2446        let body = serde_json::to_vec(&ack).map_err(|err| {
2447            RouterError::backend(
2448                0,
2449                frame.header.corr,
2450                format!("failed to encode HELLO_ACK: {err}"),
2451            )
2452        })?;
2453
2454        Frame::build_with_version(
2455            negotiated_ver,
2456            FrameType::HelloAck,
2457            control_flags(),
2458            0,
2459            0,
2460            frame.header.corr,
2461            body,
2462        )
2463        .map_err(RouterError::FrameBuild)
2464    }
2465
2466    async fn handle_client_control_request(
2467        &self,
2468        ctx: &RouteCtx,
2469        frame: Frame,
2470        request: ClientControlRequest,
2471    ) -> Result<Vec<Frame>, RouterError> {
2472        match request {
2473            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2474            ClientControlRequest::CatalogList { module_id } => {
2475                self.handle_catalog_list(frame, module_id)
2476            }
2477            ClientControlRequest::RouteOpen {
2478                target,
2479                identity,
2480                consumer_identity,
2481                consumer_capabilities,
2482                role_versions,
2483                admission_facts,
2484                scope,
2485            } => {
2486                self.handle_route_open(
2487                    ctx,
2488                    frame,
2489                    RouteOpenRequest {
2490                        target,
2491                        identity,
2492                        consumer_identity,
2493                        consumer_capabilities,
2494                        role_versions,
2495                        admission_facts,
2496                        scope,
2497                    },
2498                )
2499                .await
2500            }
2501            ClientControlRequest::RoutePoll {
2502                route_channel,
2503                route_epoch,
2504                kind,
2505            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2506            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2507            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2508                self.handle_supervisor_spawn_snapshot(frame)
2509            }
2510            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2511                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2512            }
2513            ClientControlRequest::SupervisorRestart {
2514                module_id,
2515                drain_timeout_ms,
2516            } => {
2517                self.log_supervisor_request_received(
2518                    ctx,
2519                    frame.header.corr,
2520                    ops::SUPERVISOR_RESTART,
2521                    Some(&module_id),
2522                    None,
2523                )?;
2524                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2525                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2526                    .await
2527            }
2528            ClientControlRequest::SupervisorSwap {
2529                module_id,
2530                ready_timeout_ms,
2531            } => {
2532                self.log_supervisor_request_received(
2533                    ctx,
2534                    frame.header.corr,
2535                    ops::SUPERVISOR_SWAP,
2536                    Some(&module_id),
2537                    None,
2538                )?;
2539                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2540                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2541                    .await
2542            }
2543            ClientControlRequest::SupervisorReload { module_id } => {
2544                self.log_supervisor_request_received(
2545                    ctx,
2546                    frame.header.corr,
2547                    ops::SUPERVISOR_RELOAD,
2548                    Some(&module_id),
2549                    None,
2550                )?;
2551                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2552                self.handle_supervisor_reload(frame, module_id, requester)
2553                    .await
2554            }
2555            ClientControlRequest::SupervisorRescan { preview } => {
2556                if !preview {
2557                    self.log_supervisor_request_received(
2558                        ctx,
2559                        frame.header.corr,
2560                        ops::SUPERVISOR_RESCAN,
2561                        None,
2562                        None,
2563                    )?;
2564                }
2565                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2566                self.handle_supervisor_rescan(frame, preview, requester)
2567                    .await
2568            }
2569            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2570                self.log_supervisor_request_received(
2571                    ctx,
2572                    frame.header.corr,
2573                    ops::SUPERVISOR_RELEASE_RESERVED,
2574                    Some(&module_id),
2575                    None,
2576                )?;
2577                self.handle_supervisor_release_reserved(frame, module_id)
2578                    .await
2579            }
2580            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2581                self.log_supervisor_request_received(
2582                    ctx,
2583                    frame.header.corr,
2584                    ops::SUPERVISOR_SET_ENABLED,
2585                    Some(&module_id),
2586                    Some(enabled),
2587                )?;
2588                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2589                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2590                    .await
2591            }
2592            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2593                self.handle_supervisor_health_probe(frame, module_id).await
2594            }
2595            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2596            ClientControlRequest::SupervisorRoutes { module_id } => {
2597                self.handle_supervisor_routes(frame, module_id)
2598            }
2599            ClientControlRequest::SupervisorProvenance { module_id } => {
2600                self.handle_supervisor_provenance(frame, module_id).await
2601            }
2602            ClientControlRequest::SupervisorStderrTail {
2603                module_id,
2604                max_lines,
2605                max_bytes,
2606            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2607            ClientControlRequest::SupervisorTerminals { module_id } => {
2608                self.handle_supervisor_terminals(frame, module_id).await
2609            }
2610        }
2611    }
2612
2613    fn log_supervisor_request_received(
2614        &self,
2615        ctx: &RouteCtx,
2616        corr: u64,
2617        op: &'static str,
2618        module_id: Option<&str>,
2619        enabled: Option<bool>,
2620    ) -> Result<(), RouterError> {
2621        let caller = self.supervisor_requester(ctx, corr)?;
2622        let caller = principal_label(&caller);
2623        // `Option` fields are recorded only when present, so a request with no
2624        // module id (rescan) or no enabled flag simply omits that field.
2625        info!(
2626            target: "control",
2627            op,
2628            module_id,
2629            enabled,
2630            connection_id = ctx.connection_id.get(),
2631            caller = %caller,
2632            "supervisor request received"
2633        );
2634        Ok(())
2635    }
2636
2637    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2638        Ok(self
2639            .registry
2640            .get_module_by_connection(ctx.connection_id)
2641            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2642            .map(|registration| Principal::Reserved {
2643                module_id: registration.manifest.module_id,
2644            })
2645            .unwrap_or(Principal::Direct))
2646    }
2647
2648    fn handle_module_control_request(
2649        &self,
2650        connection_id: ConnectionId,
2651        frame: Frame,
2652        request: ModuleControlRequestFromModule,
2653    ) -> Result<Vec<Frame>, RouterError> {
2654        match request {
2655            ModuleControlRequestFromModule::CatalogUpdate {
2656                provides,
2657                capabilities,
2658                ready,
2659            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2660            ModuleControlRequestFromModule::LiveRoots {} => {
2661                let registered = self
2662                    .registry
2663                    .get_module_by_connection(connection_id)
2664                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2665                let Some(registration) = registered else {
2666                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2667                };
2668                let response = self
2669                    .forwarding
2670                    .live_roots(&registration.manifest.module_id)
2671                    .map_err(RouterError::Forwarding)?;
2672                Ok(vec![control_response_body_frame(
2673                    &frame,
2674                    &response,
2675                    "ModuleControlResponseToModule::LiveRoots",
2676                )?])
2677            }
2678            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2679                self.handle_scope_sync(connection_id, frame, generation, scopes)
2680            }
2681            ModuleControlRequestFromModule::ScopeApply {
2682                generation,
2683                upsert,
2684                end,
2685            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2686            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2687                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2688            }
2689        }
2690    }
2691
2692    fn handle_operator_confirm(
2693        &self,
2694        ctx: &RouteCtx,
2695        frame: Frame,
2696    ) -> Result<Vec<Frame>, RouterError> {
2697        use crate::operator_confirm::{audit, Outcome};
2698        let registration = self
2699            .registry
2700            .get_module_by_connection(ctx.connection_id)
2701            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2702        let Some(registration) = registration else {
2703            let outcome = Outcome::refusal("not_registered");
2704            audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2705            return Ok(vec![outcome.frame(&frame)]);
2706        };
2707        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2708            Ok(request) => request,
2709            Err(_) => {
2710                let outcome = Outcome::refusal("invalid_control_body");
2711                audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2712                return Ok(vec![outcome.frame(&frame)]);
2713            }
2714        };
2715        let module_id = registration.manifest.module_id;
2716        // Read the launch nonce (under its own lock) before taking the forwarding
2717        // table's lock below: holding forwarding while waiting on another daemon
2718        // lock risks a lock-order deadlock with paths that take them the other way.
2719        let nonce = self
2720            .hello_launch_nonces
2721            .lock()
2722            .unwrap_or_else(|p| p.into_inner())
2723            .nonce(ctx.connection_id)
2724            .map(str::to_owned);
2725        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2726            self.supervisor
2727                .spawned_consumer_authorized(&module_id, nonce)
2728        });
2729        let confirms = self.forwarding.operator_confirms();
2730        self.forwarding
2731            .with_operator_route(
2732                ctx.connection_id,
2733                request.route_channel,
2734                request.route_epoch,
2735                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2736            )
2737            .map_err(RouterError::Forwarding)
2738    }
2739
2740    /// `scope.sync`: the owner is the module registered on this connection.
2741    /// A connection with no registration (every client connection, `direct`
2742    /// included) is refused `not_registered` before the table is consulted.
2743    fn handle_scope_sync(
2744        &self,
2745        connection_id: ConnectionId,
2746        frame: Frame,
2747        generation: u64,
2748        scopes: Vec<ScopeRecord>,
2749    ) -> Result<Vec<Frame>, RouterError> {
2750        self.handle_scope_change(connection_id, frame, generation, scopes, None)
2751    }
2752
2753    fn handle_scope_change(
2754        &self,
2755        connection_id: ConnectionId,
2756        frame: Frame,
2757        generation: u64,
2758        scopes: Vec<ScopeRecord>,
2759        end: Option<Vec<ScopeEnd>>,
2760    ) -> Result<Vec<Frame>, RouterError> {
2761        let is_apply = end.is_some();
2762        let Some(registration) = self
2763            .registry
2764            .get_module_by_connection(connection_id)
2765            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2766        else {
2767            return Ok(vec![control_error_frame(
2768                &frame,
2769                "not_registered",
2770                "scope.sync and scope.apply require an active module registration owned by this connection",
2771            )?]);
2772        };
2773        let owner = registration.manifest.module_id;
2774        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2775        let is_current_launch = |connection: ConnectionId| {
2776            self.hello_launch_nonces
2777                .lock()
2778                .unwrap_or_else(|poisoned| poisoned.into_inner())
2779                .presented(connection, current_nonce.as_deref())
2780        };
2781        // Lock order is the scope table, then the forwarding table: the new
2782        // tags are published, and the routes the change closes are selected,
2783        // while the scope table is still write-locked, so no admission can read
2784        // a record whose tag is not yet published.
2785        let mut table = self
2786            .scopes
2787            .write()
2788            .unwrap_or_else(|poisoned| poisoned.into_inner());
2789        let outcome = match end {
2790            Some(end) => table.apply(
2791                &owner,
2792                connection_id,
2793                is_current_launch,
2794                generation,
2795                scopes,
2796                end,
2797            ),
2798            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
2799        };
2800        let drained = match &outcome {
2801            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
2802            Err(_) => Vec::new(),
2803        };
2804        let has_deadlines = outcome.is_ok() && table.has_deadlines();
2805        drop(table);
2806        if has_deadlines {
2807            self.scope_deadline_added.notify_one();
2808        }
2809        match outcome {
2810            Ok(applied) => {
2811                let counts = ScopeOutcomeCounts::of(&applied.results);
2812                info!(
2813                    owner = %owner,
2814                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2815                    generation,
2816                    records = applied.results.len(),
2817                    created = counts.created,
2818                    replaced = counts.replaced,
2819                    updated = counts.updated,
2820                    unchanged = counts.unchanged,
2821                    refused = counts.refused,
2822                    ended = applied.ended.len(),
2823                    tag_changes = applied.tag_changes.len(),
2824                    routes_closed = drained.len(),
2825                    "scope change accepted"
2826                );
2827                // An accepted scope change can still refuse individual records, and the
2828                // owner is the only party that sees the reply. Name them here so
2829                // an operator can tell a refused session from a missing one
2830                // without the owner's logs. Capped so a call that refuses
2831                // thousands cannot flood the log; the count above is complete.
2832                for refused in applied
2833                    .results
2834                    .iter()
2835                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2836                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2837                {
2838                    warn!(
2839                        owner = %owner,
2840                        generation,
2841                        scope_ref = %refused.scope_ref,
2842                        scope_epoch = refused.scope_epoch,
2843                        code = refused.code.as_deref().unwrap_or(""),
2844                        "scope record refused"
2845                    );
2846                }
2847                self.close_scope_drained_routes(drained);
2848                let response = if is_apply {
2849                    ModuleControlResponseToModule::ScopeApply {
2850                        generation,
2851                        results: applied.results,
2852                        end_results: applied.end_results,
2853                        ended: applied.ended,
2854                    }
2855                } else {
2856                    ModuleControlResponseToModule::ScopeSync {
2857                        generation,
2858                        results: applied.results,
2859                        ended: applied.ended,
2860                    }
2861                };
2862                Ok(vec![control_response_body_frame(
2863                    &frame,
2864                    &response,
2865                    "scope change response",
2866                )?])
2867            }
2868            Err(refusal) => {
2869                info!(
2870                    owner = %owner,
2871                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2872                    generation,
2873                    code = refusal.code,
2874                    "scope change refused"
2875                );
2876                Ok(vec![control_error_frame(
2877                    &frame,
2878                    refusal.code,
2879                    refusal.message,
2880                )?])
2881            }
2882        }
2883    }
2884
2885    /// Tell both ends of each route a scope change closed. The module gets a
2886    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
2887    /// the client's route handle. The client also gets `route.closed` with the
2888    /// scope reason, one push per module and reason, so it can tell a revoked
2889    /// route from an ordinary close and not reopen it.
2890    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2891        if drained.is_empty() {
2892            return;
2893        }
2894        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2895            BTreeMap::new();
2896        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2897        for route in drained {
2898            warn!(
2899                module_id = %route.module_id,
2900                reason = ?route.reason,
2901                client_connection_id = route.client.connection_id.get(),
2902                route_channel = route.client.channel,
2903                "closing route because its scope changed"
2904            );
2905            pushes
2906                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2907                .or_insert_with(|| (route.reason, Vec::new()))
2908                .1
2909                .push(EndpointRoute {
2910                    goodbye_target: route.client.clone(),
2911                    principal: Principal::Unverified,
2912                    bound_at: Instant::now(),
2913                    draining: false,
2914                    drain_reason: None,
2915                });
2916            goodbyes.push(route.module);
2917            goodbyes.push(route.client);
2918        }
2919        for ((module_id, _), (reason, routes)) in pushes {
2920            send_route_control_pushes(
2921                &self.forwarding,
2922                routes,
2923                ClientControlPush::RouteClosed {
2924                    module_id,
2925                    channels: Vec::new(),
2926                    reason,
2927                    drained: false,
2928                    abandoned: 0,
2929                    excluded_subscriptions: 0,
2930                    terminal: Some(false),
2931                },
2932            );
2933        }
2934        self.emit_route_goodbyes(goodbyes);
2935    }
2936
2937    /// `scope.describe`: any registered module may read any scope, because a
2938    /// provider must read the scope a route it serves is stamped with.
2939    fn handle_scope_describe(
2940        &self,
2941        connection_id: ConnectionId,
2942        frame: Frame,
2943        owner: Principal,
2944        scope_ref: String,
2945    ) -> Result<Vec<Frame>, RouterError> {
2946        let registered = self
2947            .registry
2948            .get_module_by_connection(connection_id)
2949            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2950        if registered.is_none() {
2951            return Ok(vec![control_error_frame(
2952                &frame,
2953                "not_registered",
2954                "scope.describe requires an active module registration owned by this connection",
2955            )?]);
2956        }
2957        let description = self
2958            .scopes
2959            .read()
2960            .unwrap_or_else(|poisoned| poisoned.into_inner())
2961            .describe(&owner, &scope_ref);
2962        let owner_configured = match &owner {
2963            // Ask whether the owner is configured (`is_configured`), not
2964            // whether it is on the roster (`get(..).is_some()`): a supervised
2965            // module's process can register and describe a scope before the
2966            // supervisor has put it on the roster.
2967            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2968            _ => false,
2969        };
2970        let response = ModuleControlResponseToModule::ScopeDescribe {
2971            status: description.status,
2972            scope_epoch: description.scope_epoch,
2973            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2974            owner_synced: description.owner_synced,
2975            owner_configured,
2976            scope: description.stamp,
2977        };
2978        Ok(vec![control_response_body_frame(
2979            &frame,
2980            &response,
2981            "ModuleControlResponseToModule::ScopeDescribe",
2982        )?])
2983    }
2984
2985    fn handle_catalog_update(
2986        &self,
2987        connection_id: ConnectionId,
2988        frame: Frame,
2989        provides: Vec<ProviderRole>,
2990        capabilities: Option<CapabilityDeclarations>,
2991        ready: Option<bool>,
2992    ) -> Result<Vec<Frame>, RouterError> {
2993        self.refresh_capability_requirements();
2994        let Some(registration) = self
2995            .registry
2996            .get_module_by_connection(connection_id)
2997            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2998        else {
2999            return Ok(vec![control_error_frame(
3000                &frame,
3001                "not_registered",
3002                "catalog.update requires an active module registration owned by this connection",
3003            )?]);
3004        };
3005
3006        if let Some(message) =
3007            catalog_update_frozen_field_message(&registration.manifest, &provides)
3008        {
3009            return Ok(vec![control_error_frame(
3010                &frame,
3011                "catalog_update_frozen_field",
3012                message,
3013            )?]);
3014        }
3015
3016        let mut candidate = registration.manifest.clone();
3017        candidate.provides = provides.clone();
3018        candidate.capabilities = capabilities
3019            .clone()
3020            .or_else(|| registration.manifest.capabilities.clone());
3021        if let Err(err) = candidate.validate_capability_grammar() {
3022            return Ok(vec![control_error_frame(
3023                &frame,
3024                "invalid_capability_grammar",
3025                err.to_string(),
3026            )?]);
3027        }
3028
3029        // Updates must honor the same reserved owner as initial registration;
3030        // otherwise an empty HELLO could acquire the claim after admission.
3031        let mut conflicts = self
3032            .capability_evaluator
3033            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3034        if let Some(conflict) = conflicts.first() {
3035            let message = format!(
3036                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3037                conflict.capability, conflict.claimants[0], candidate.module_id
3038            );
3039            for conflict in &mut conflicts {
3040                conflict.source = DuplicateClaimSource::CatalogUpdate;
3041            }
3042            log_duplicate_claim_events(conflicts);
3043            return Ok(vec![control_error_frame(
3044                &frame,
3045                "reserved_capability",
3046                message,
3047            )?]);
3048        }
3049
3050        let updated = self
3051            .registry
3052            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3053            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3054        if updated.is_none() {
3055            return Ok(vec![control_error_frame(
3056                &frame,
3057                "not_registered",
3058                "catalog.update requires an active module registration owned by this connection",
3059            )?]);
3060        }
3061        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3062            log_duplicate_claim_events(
3063                self.capability_evaluator
3064                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
3065            );
3066        }
3067        if capability_census_trigger(
3068            registration.manifest.capabilities.as_ref(),
3069            updated
3070                .as_ref()
3071                .and_then(|entry| entry.manifest.capabilities.as_ref()),
3072        ) {
3073            self.enforce_capability_denies();
3074        }
3075        self.refresh_capability_requirements();
3076
3077        let response = ModuleControlResponseToModule::CatalogUpdate {};
3078        control_response_body_frame(
3079            &frame,
3080            &response,
3081            "ModuleControlResponseToModule::CatalogUpdate",
3082        )
3083        .map(|frame| vec![frame])
3084    }
3085
3086    pub(crate) fn with_activity_ledger_config(
3087        mut self,
3088        run_dir: &Path,
3089        incarnation: String,
3090        config: crate::activity::ActivityConfig,
3091    ) -> Self {
3092        self.activity = Some(crate::activity::ActivityLedger::open_config(
3093            run_dir.join("activity.jsonl"),
3094            incarnation,
3095            config,
3096        ));
3097        self.supervisor.set_activity(self.activity.clone());
3098        self
3099    }
3100
3101    pub(crate) async fn configure_activity(
3102        &self,
3103        config: crate::activity::ActivityConfig,
3104    ) -> Result<(), &'static str> {
3105        match &self.activity {
3106            Some(ledger) => ledger.configure(config).await,
3107            None => Ok(()),
3108        }
3109    }
3110
3111    fn cancel_activity_subscription(
3112        &self,
3113        ctx: &RouteCtx,
3114        frame: &Frame,
3115    ) -> Result<bool, RouterError> {
3116        let task = self
3117            .activity_streams
3118            .lock()
3119            .unwrap_or_else(|p| p.into_inner())
3120            .remove(&(ctx.connection_id, frame.header.corr));
3121        let Some(task) = task else {
3122            return Ok(false);
3123        };
3124        task.abort();
3125        let end = Frame::build_with_version(
3126            response_version(frame),
3127            FrameType::StreamEnd,
3128            control_flags(),
3129            0,
3130            0,
3131            frame.header.corr,
3132            Vec::new(),
3133        )
3134        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3135        let sink = ctx.egress.clone();
3136        tokio::spawn(async move {
3137            let _ = sink.send(end).await;
3138        });
3139        Ok(true)
3140    }
3141
3142    async fn handle_activity_read(
3143        &self,
3144        ctx: &RouteCtx,
3145        frame: Frame,
3146    ) -> Result<Vec<Frame>, RouterError> {
3147        use subc_control::{
3148            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3149            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3150        };
3151        #[derive(Deserialize)]
3152        struct Identity {
3153            #[serde(default)]
3154            consumer_identity: Option<ConsumerIdentity>,
3155        }
3156        // A registered module connection already identifies its caller. Ignore
3157        // consumer_identity there so a module cannot borrow a mirror's identity.
3158        // Only separate consumer connections use the Direct/nonce trust boundary.
3159        let principal = match self
3160            .forwarding
3161            .module_id_for_connection(ctx.connection_id)
3162            .map_err(RouterError::Forwarding)?
3163        {
3164            Some(module_id) => Ok(Principal::Reserved { module_id }),
3165            None => {
3166                let identity: Identity = match serde_json::from_slice(&frame.body) {
3167                    Ok(identity) => identity,
3168                    Err(_) => {
3169                        return Ok(vec![control_error_frame(
3170                            &frame,
3171                            ACTIVITY_MALFORMED,
3172                            "malformed activity read",
3173                        )?])
3174                    }
3175                };
3176                self.route_open_principal(&frame, identity.consumer_identity)?
3177            }
3178        };
3179        let admitted = matches!(principal, Ok(Principal::Direct))
3180            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3181        if !admitted {
3182            return Ok(vec![control_error_frame(
3183                &frame,
3184                ACTIVITY_READ_FORBIDDEN,
3185                "activity reads require an operator or trusted mirror principal",
3186            )?]);
3187        }
3188        let Some(ledger) = &self.activity else {
3189            return Ok(vec![control_error_frame(
3190                &frame,
3191                ACTIVITY_UNAVAILABLE,
3192                "activity ledger unavailable",
3193            )?]);
3194        };
3195        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3196            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3197        {
3198            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3199                Ok(request) => ledger.query(request).await,
3200                Err(_) => Err(ACTIVITY_MALFORMED),
3201            };
3202            return match result {
3203                Ok(reply) => Ok(vec![control_response_body_frame(
3204                    &frame,
3205                    &reply,
3206                    ops::ACTIVITY_QUERY,
3207                )?]),
3208                Err(code) => Ok(vec![control_error_frame(
3209                    &frame,
3210                    code,
3211                    "activity query refused",
3212                )?]),
3213            };
3214        }
3215        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3216            Ok(request) => request,
3217            Err(_) => {
3218                return Ok(vec![control_error_frame(
3219                    &frame,
3220                    ACTIVITY_MALFORMED,
3221                    "malformed activity subscription",
3222                )?])
3223            }
3224        };
3225        let mut subscription = match ledger.subscribe(request.since).await {
3226            Ok(subscription) => subscription,
3227            Err(code) => {
3228                return Ok(vec![control_error_frame(
3229                    &frame,
3230                    code,
3231                    "activity subscription refused",
3232                )?])
3233            }
3234        };
3235        let key = (ctx.connection_id, frame.header.corr);
3236        let sink = ctx.egress.clone();
3237        let mut streams = self
3238            .activity_streams
3239            .lock()
3240            .unwrap_or_else(|p| p.into_inner());
3241        streams.retain(|_, task| !task.is_finished());
3242        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3243            return Ok(vec![control_error_frame(
3244                &frame,
3245                "duplicate_subscription",
3246                "activity subscription correlation id already in use",
3247            )?]);
3248        }
3249        let task = tokio::spawn(async move {
3250            loop {
3251                let entry = if let Some(entry) = subscription.replay.pop_front() {
3252                    Some(entry)
3253                } else {
3254                    subscription.receiver.recv().await
3255                };
3256                let Some(entry) = entry else {
3257                    break;
3258                };
3259                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3260                    return;
3261                };
3262                let Ok(next) = Frame::build_with_version(
3263                    response_version(&frame),
3264                    FrameType::StreamData,
3265                    control_flags(),
3266                    0,
3267                    0,
3268                    frame.header.corr,
3269                    body,
3270                ) else {
3271                    return;
3272                };
3273                if sink.send(next).await.is_err() {
3274                    return;
3275                }
3276            }
3277            // Dropping a full subscriber's sender drains its accepted entries
3278            // before the terminal error, preserving their order on the wire.
3279            if let Some(code) = subscription.terminal_code() {
3280                if let Ok(end) =
3281                    control_error_frame(&frame, code, "activity subscriber fell behind")
3282                {
3283                    let _ = sink.send(end).await;
3284                }
3285            }
3286        });
3287        streams.insert(key, task.abort_handle());
3288        Ok(Vec::new())
3289    }
3290
3291    async fn handle_activity_record(
3292        &self,
3293        ctx: &RouteCtx,
3294        frame: Frame,
3295    ) -> Result<Vec<Frame>, RouterError> {
3296        use subc_control::{
3297            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3298            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3299        };
3300        let result = async {
3301            let module = self
3302                .forwarding
3303                .module_id_for_connection(ctx.connection_id)
3304                .ok()
3305                .flatten()
3306                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3307            let request: ActivityRecordRequest =
3308                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3309            crate::activity::validate(&module, &request)?;
3310            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3311            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3312            let at_ms = std::time::SystemTime::now()
3313                .duration_since(std::time::UNIX_EPOCH)
3314                .unwrap_or_default()
3315                .as_millis() as u64;
3316            ledger.record(module, request, actor, at_ms).await
3317        }
3318        .await;
3319        match result {
3320            Ok(entry) => Ok(vec![control_response_body_frame(
3321                &frame,
3322                &ActivityRecordResponse { seq: entry.seq },
3323                "activity.record",
3324            )?]),
3325            Err(code) => Ok(vec![control_error_frame(
3326                &frame,
3327                code,
3328                "activity recording refused",
3329            )?]),
3330        }
3331    }
3332
3333    fn activity_actor(
3334        &self,
3335        connection: ConnectionId,
3336        module: &str,
3337        on: Option<&subc_control::ActivityOn>,
3338    ) -> Result<subc_control::ActivityActor, &'static str> {
3339        use crate::forwarding::{DataRoute, DataRouteState};
3340        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3341        let principal = |id: &str| Principal::Reserved {
3342            module_id: id.to_owned(),
3343        };
3344        let mut actor = ActivityActor {
3345            principal: principal(module),
3346            scope_owner: None,
3347            agent_id: None,
3348            run_id: None,
3349            flow_id: None,
3350            project_root: None,
3351        };
3352        let scope = match on {
3353            None => return Ok(actor),
3354            Some(ActivityOn::Scope(scope)) => {
3355                let selector = subc_protocol::scope::ScopeSelector {
3356                    owner: principal(module),
3357                    scope_ref: scope.scope_ref.clone(),
3358                    scope_epoch: Some(scope.epoch),
3359                };
3360                self.scopes
3361                    .read()
3362                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3363                    .admit(&principal(module), module, &selector, true)
3364                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3365            }
3366            Some(ActivityOn::Route(route)) => {
3367                let DataRoute::Module(DataRouteState::Bound(binding)) = self
3368                    .forwarding
3369                    .lookup_data_route(connection, route.channel, route.epoch)
3370                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3371                else {
3372                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3373                };
3374                if binding.module_id != module {
3375                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3376                }
3377                actor.principal = binding.principal.clone();
3378                actor.project_root = binding
3379                    .project_root
3380                    .as_ref()
3381                    .map(|root| root.as_path().to_path_buf());
3382                let Some(bound) = &binding.scope else {
3383                    return Ok(actor);
3384                };
3385                let owner = principal(&bound.owner);
3386                let selector = subc_protocol::scope::ScopeSelector {
3387                    owner: owner.clone(),
3388                    scope_ref: bound.scope_ref.clone(),
3389                    scope_epoch: Some(bound.tag.scope_epoch),
3390                };
3391                let admitted = self
3392                    .scopes
3393                    .read()
3394                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3395                    .admit(&owner, module, &selector, true)
3396                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3397                if admitted.tag != bound.tag {
3398                    return Err(ACTIVITY_ACTOR_UNKNOWN);
3399                }
3400                admitted
3401            }
3402        };
3403        actor.scope_owner = Some(scope.stamp.owner);
3404        actor.agent_id = scope.stamp.attributes.agent_id;
3405        actor.run_id = scope.stamp.attributes.run_id;
3406        actor.flow_id = scope.stamp.attributes.flow_id;
3407        Ok(actor)
3408    }
3409
3410    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3411        self.refresh_capability_requirements();
3412        // A bare connection count is ambiguous between many clients holding a
3413        // route each and one client accumulating hundreds, so publish the
3414        // concentration alongside it. Route state is best-effort here: a
3415        // diagnostic endpoint must still answer if the forwarding lock is
3416        // contended.
3417        let mut counters = self.counters.snapshot();
3418        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3419            obj.insert(
3420                "activity_unparseable_lines".into(),
3421                ledger.read_errors().into(),
3422            );
3423            obj.insert(
3424                "activity_lifecycle_append_failures".into(),
3425                ledger.lifecycle_failures().into(),
3426            );
3427            obj.insert(
3428                "activity_started_digest_skips".into(),
3429                ledger.lifecycle_digest_skips().into(),
3430            );
3431        }
3432        if let (Ok((connections_with_routes, max)), Some(obj)) = (
3433            self.forwarding.client_route_concentration(),
3434            counters.as_object_mut(),
3435        ) {
3436            obj.insert(
3437                "client_connections_with_routes".into(),
3438                connections_with_routes.into(),
3439            );
3440            obj.insert("max_routes_on_one_connection".into(), max.into());
3441        }
3442        // A module that is being fast-refused and a module that is fine look
3443        // identical from a client that retries and succeeds, so name the open
3444        // breakers here. This rides the existing free-form counters object
3445        // rather than a new wire field, so no sibling that deserializes
3446        // `ServerDescribe` has to be rebuilt to keep reading it.
3447        if let (Some(open_breakers), Some(obj)) = (
3448            self.route_bind_breakers.open_snapshot(),
3449            counters.as_object_mut(),
3450        ) {
3451            obj.insert("route_bind_breakers_open".into(), open_breakers);
3452        }
3453        let response = ClientControlResponse::ServerDescribe {
3454            protocol_ver: PROTOCOL_VERSION,
3455            subc_ops: subc_ops(),
3456            capabilities: self.subc_capabilities.as_ref().to_vec(),
3457            connected_clients: self.connected_clients.count(),
3458            counters: Some(counters),
3459            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3460            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3461            capability_requirements: self.capability_requirement_statuses(),
3462            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3463        };
3464        Ok(vec![control_response_body_frame(
3465            &frame,
3466            &response,
3467            "ClientControlResponse::ServerDescribe",
3468        )?])
3469    }
3470
3471    fn handle_catalog_list(
3472        &self,
3473        frame: Frame,
3474        module_id: Option<String>,
3475    ) -> Result<Vec<Frame>, RouterError> {
3476        let (generation, modules) = self.registry.list_modules().map_err(|err| {
3477            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3478        })?;
3479        let entries = modules
3480            .into_iter()
3481            .filter(|registration| {
3482                module_id
3483                    .as_deref()
3484                    .map(|wanted| registration.manifest.module_id == wanted)
3485                    .unwrap_or(true)
3486            })
3487            .map(|registration| {
3488                let not_ready = self.not_ready_reason(&registration);
3489                let roles = registration.manifest.provides;
3490                CatalogEntry::new(
3491                    registration.manifest.module_id,
3492                    roles,
3493                    registration.control_ops,
3494                )
3495                .with_ready(not_ready.is_none())
3496                .with_not_ready(not_ready)
3497                .with_module_version(Some(registration.manifest.module_version))
3498                .with_capabilities(registration.manifest.capabilities)
3499                .with_self_signals(registration.manifest.self_signals)
3500                .with_events(registration.manifest.events)
3501            })
3502            .collect();
3503        let response = ClientControlResponse::CatalogList {
3504            generation,
3505            modules: entries,
3506            subc_ops: subc_ops(),
3507        };
3508        Ok(vec![control_response_body_frame(
3509            &frame,
3510            &response,
3511            "ClientControlResponse::CatalogList",
3512        )?])
3513    }
3514
3515    fn route_open_principal(
3516        &self,
3517        frame: &Frame,
3518        consumer_identity: Option<ConsumerIdentity>,
3519    ) -> Result<Result<Principal, Frame>, RouterError> {
3520        let Some(consumer_identity) = consumer_identity else {
3521            return Ok(Ok(Principal::Direct));
3522        };
3523
3524        if self.supervisor.spawned_consumer_authorized(
3525            &consumer_identity.module_id,
3526            &consumer_identity.launch_nonce,
3527        ) {
3528            return Ok(Ok(Principal::Reserved {
3529                module_id: consumer_identity.module_id,
3530            }));
3531        }
3532
3533        Ok(Err(control_error_frame(
3534            frame,
3535            "bad_consumer_identity",
3536            format!(
3537                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3538                consumer_identity.module_id
3539            ),
3540        )?))
3541    }
3542
3543    /// Ordinary `route.open` refusals go through here; admission and breaker
3544    /// refusals log separately with their capacity or breaker state. The daemon can
3545    /// attest which code it sent: without the event, a client's "the daemon
3546    /// refused me" and the daemon's own view could only be reconciled by
3547    /// argument. Malformed input (`invalid_project_root`) does not come here;
3548    /// rejecting a request that was never a valid open is not a refusal of one.
3549    fn route_open_refusal_frame(
3550        &self,
3551        ctx: &RouteCtx,
3552        frame: &Frame,
3553        module_id: &str,
3554        reason: &'static str,
3555        code: &'static str,
3556        message: impl Into<String>,
3557    ) -> Result<Frame, RouterError> {
3558        self.observe_route_open_refusal(ctx, module_id, reason, code);
3559        control_error_frame(frame, code, message.into())
3560    }
3561
3562    /// Refuse a `route.open` because the target module's bind-relay breaker is
3563    /// open, without attempting the relay.
3564    ///
3565    /// The wire code is `module_timeout`, which is the truth (the module has
3566    /// not been answering binds) and which both SDKs already classify as
3567    /// retryable with capped backoff. Reusing it is what keeps this change out
3568    /// of both SDKs; the daemon-side distinction lives in the counter key
3569    /// instead.
3570    ///
3571    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
3572    /// While a breaker is open this fires on every open to that module, and the
3573    /// stall written up in `docs/designs/route-open-head-of-line.md` already
3574    /// produced 261 lines about a single module inside 3000 lines of daemon
3575    /// log. The rare transitions are logged at warn/info instead and the volume
3576    /// is carried by the counter, so the evidence survives without the flood.
3577    /// The debug line keeps a per-refusal record reachable for whoever turns
3578    /// the level up.
3579    fn route_open_breaker_refusal_frame(
3580        &self,
3581        ctx: &RouteCtx,
3582        frame: &Frame,
3583        module_id: &str,
3584        consecutive_timeouts: u32,
3585        retry_in: Duration,
3586        probe_in_flight: bool,
3587    ) -> Result<Frame, RouterError> {
3588        self.counters
3589            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3590        debug!(
3591            target: "control",
3592            code = "module_timeout",
3593            module_id = ?module_id,
3594            connection_id = ctx.connection_id.get(),
3595            consecutive_timeouts,
3596            retry_in_ms = retry_in.as_millis() as u64,
3597            probe_in_flight,
3598            "route.open refused by open bind-relay breaker"
3599        );
3600        // Say what a caller can act on. An open bind-relay breaker means the
3601        // module timed out accepting several new routes in a row. The module
3602        // is still running and its established routes keep working; only new
3603        // route.open requests are refused until the cooldown ends and one
3604        // test route (the probe) gets through. A message that only counts
3605        // failed relays reads as "the module is down" to a worker that sees it.
3606        let detail = if probe_in_flight {
3607            "one test route is already being tried; retry once it settles".to_string()
3608        } else {
3609            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3610        };
3611        control_error_frame(
3612            frame,
3613            "module_timeout",
3614            format!(
3615                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3616                 timed out in a row); {detail}; its established routes are unaffected"
3617            ),
3618        )
3619    }
3620
3621    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
3622    /// requester's bytes (an unknown target is whatever the client sent) and
3623    /// is Debug-formatted so control characters land in the log escaped
3624    /// rather than as terminal sequences for whoever tails it.
3625    ///
3626    /// `reason` names the check that refused, because one wire code has
3627    /// several senders: after a module registers, `target_unavailable` can
3628    /// come from a missing role, an inactive registration, a supervisor that
3629    /// has not marked the process live, a missing forwarding connection, or a
3630    /// failed relay, and a log that records only the code cannot say which of
3631    /// them fired. It is a static, daemon-chosen label per branch, so it is
3632    /// safe to print plainly and stays a closed set.
3633    fn observe_route_open_refusal(
3634        &self,
3635        ctx: &RouteCtx,
3636        module_id: &str,
3637        reason: &'static str,
3638        code: &'static str,
3639    ) {
3640        self.counters.increment_route_open_refused(code);
3641        info!(
3642            target: "control",
3643            code,
3644            reason,
3645            module_id = ?module_id,
3646            connection_id = ctx.connection_id.get(),
3647            "route.open refused"
3648        );
3649        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3650            self.route_outages.record_not_serving(module_id, reason);
3651        }
3652    }
3653
3654    /// Record an ACCEPTED route.open.
3655    ///
3656    /// Refusals have been logged and counted since the attestation work; accepts
3657    /// were invisible, so the daemon knew every principal it stamped and wrote
3658    /// none of them down. The party that attests the identity was the only party
3659    /// not recording it, which left a credential vault unable to name the sender
3660    /// of a call that reached it (claustrum #43) and left the launch-nonce
3661    /// concurrency question unanswerable from the outside.
3662    ///
3663    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
3664    /// `code`/`module_id`/`connection_id` returns both directions of the same
3665    /// decision rather than two shapes a reader has to join by hand.
3666    ///
3667    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
3668    /// and the difference carries information rather than being an
3669    /// inconsistency. This line is only reachable after a successful bind to a
3670    /// REGISTERED module, so the value has already passed HELLO validation
3671    /// including the path-hazard refusal and cannot contain control bytes. A
3672    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
3673    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
3674    ///
3675    /// Bare is also what every other daemon line already emits (`module
3676    /// registered`, `configured module supervised`). Shipping `?module_id` here
3677    /// made this instrument the only one in the file whose ids did not answer
3678    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
3679    /// line whose whole purpose is being grepped beside its sibling.
3680    ///
3681    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
3682    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
3683    /// forwards every field type through it and a bare `&str` and a `?`-escaped
3684    /// one are recorded identically. A test written against that harness passes
3685    /// either way -- I wrote one, measured it, and deleted it rather than ship a
3686    /// green assertion that cannot fail. The same limit applies to the escaping
3687    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
3688    /// it reads as a guard on the Debug escaping and cannot detect its removal.
3689    /// Fencing either needs the real formatter, not the capture layer.
3690    ///
3691    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
3692    /// unix-socket options and subc is loopback TCP, so there is no peer identity
3693    /// to record. The ephemeral port would decay within minutes and answer only a
3694    /// live question. The identity question is instead answered by counting
3695    /// distinct live connections presenting one module's `consumer_identity` --
3696    /// "is anyone else holding this secret" rather than "is this the right
3697    /// process".
3698    fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3699        self.route_outages.record_accepted(module_id);
3700        self.counters.increment_route_open_accepted(principal);
3701        info!(
3702            target: "control",
3703            principal,
3704            module_id,
3705            connection_id = ctx.connection_id.get(),
3706            "route.open accepted"
3707        );
3708    }
3709
3710    fn supervised_absent_route_open_refusal_frame(
3711        &self,
3712        ctx: &RouteCtx,
3713        frame: &Frame,
3714        module_id: &str,
3715        code: &'static str,
3716        status: &crate::supervise::ModuleStatus,
3717    ) -> Result<Frame, RouterError> {
3718        self.counters.increment_route_open_refused(code);
3719        info!(
3720            target: "control",
3721            code,
3722            reason = "supervised_not_registered",
3723            module_id = ?module_id,
3724            connection_id = ctx.connection_id.get(),
3725            state = %status.state,
3726            enabled = status.enabled,
3727            live = status.live,
3728            "route.open refused"
3729        );
3730        // A supervised module whose process has not registered is not
3731        // serving, whatever the reason; the supervisor knows this id, so it is
3732        // safe to track.
3733        self.route_outages
3734            .record_not_serving(module_id, "supervised_not_registered");
3735        let message = if let Some(pid) = status.waiting_for_previous_pid {
3736            format!("waiting: previous instance (pid {pid}) is still running but hidden; it cannot be identified")
3737        } else {
3738            format!(
3739                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3740                status.state, status.enabled, status.live
3741            )
3742        };
3743        control_error_frame(frame, code, message)
3744    }
3745
3746    async fn handle_route_open(
3747        &self,
3748        ctx: &RouteCtx,
3749        frame: Frame,
3750        request: RouteOpenRequest,
3751    ) -> Result<Vec<Frame>, RouterError> {
3752        let RouteOpenRequest {
3753            target,
3754            mut identity,
3755            consumer_identity,
3756            consumer_capabilities,
3757            role_versions,
3758            admission_facts,
3759            scope,
3760        } = request;
3761        let target_module_id = target_module_id(&target).to_string();
3762        if self
3763            .registry
3764            .get_module_by_connection(ctx.connection_id)
3765            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3766            .is_some()
3767        {
3768            return Ok(vec![control_error_frame(
3769                &frame,
3770                "invalid_request",
3771                "module connections cannot open client routes",
3772            )?]);
3773        }
3774        debug!(
3775            connection_id = ctx.connection_id.get(),
3776            corr = frame.header.corr,
3777            module_id = %target_module_id,
3778            "handling route.open"
3779        );
3780
3781        // A malformed declaration is refused first, before anything about the
3782        // target is looked up: the same body would be refused against any
3783        // module, so the caller learns nothing by retrying or waiting. An empty
3784        // map declares nothing and travels as no field at all, so a provider
3785        // only ever sees a missing field or a non-empty one.
3786        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3787        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3788            self.observe_route_open_refusal(
3789                ctx,
3790                &target_module_id,
3791                "invalid_role_versions",
3792                error_codes::INVALID_REQUEST,
3793            );
3794            return Ok(vec![control_error_body_frame(
3795                &frame,
3796                ErrorBody {
3797                    code: error_codes::INVALID_REQUEST.to_string(),
3798                    message: error.to_string(),
3799                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3800                },
3801            )?]);
3802        }
3803
3804        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
3805            if status.waiting_for_previous_pid.is_some() {
3806                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3807                    ctx,
3808                    &frame,
3809                    &target_module_id,
3810                    "module_warming",
3811                    &status,
3812                )?]);
3813            }
3814        }
3815
3816        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
3817        // opposite. Below, a caller learns whether a module is unregistered,
3818        // supervised-but-down (with state/enabled/live), or registered without the
3819        // requested role. Elsewhere that is an enumeration leak: a probe learning
3820        // the shape of a fleet it cannot otherwise see.
3821        //
3822        // It is not one here, and the reason is the ACCESS MODEL rather than
3823        // anything about these errors. Reaching route.open requires the
3824        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
3825        // connection file, so any caller who completes it already runs as this
3826        // user -- and can read subc.jsonc for the module list and `ck module
3827        // status` for live state. The reply discloses nothing the caller cannot
3828        // read more easily from disk, while the precision is load-bearing:
3829        // `unknown_module` is retryable and a missing role is not.
3830        //
3831        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
3832        // remote transport, a sandboxed caller, a shared-host mode -- THAT
3833        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
3834        let Some(registration) = self
3835            .registry
3836            .get_module(&target_module_id)
3837            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3838        else {
3839            if let Some((status, warming)) =
3840                self.supervisor_status(&target_module_id, frame.header.corr)?
3841            {
3842                // BEFORE the two availability codes below, because for a module
3843                // that speaks no subc wire both of them are false comfort: they
3844                // say "not right now" and are retried, and this module will
3845                // never register no matter how long the caller waits. The
3846                // absence here is the declaration being honoured, not a module
3847                // that is late.
3848                if status.protocol == ModuleProtocol::None {
3849                    return Ok(vec![self.route_open_refusal_frame(
3850                        ctx,
3851                        &frame,
3852                        &target_module_id,
3853                        "protocol_none",
3854                        error_codes::MODULE_NO_PROTOCOL,
3855                        format!(
3856                            "module_id '{target_module_id}' is declared protocol: none; \
3857                             it speaks no subc wire and serves no routes"
3858                        ),
3859                    )?]);
3860                }
3861                let code = if warming {
3862                    "module_warming"
3863                } else {
3864                    "target_unavailable"
3865                };
3866                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3867                    ctx,
3868                    &frame,
3869                    &target_module_id,
3870                    code,
3871                    &status,
3872                )?]);
3873            }
3874            if let Some(removed_ago_ms) =
3875                self.supervisor.removal_tombstone_age_ms(&target_module_id)
3876            {
3877                return Ok(vec![self.route_open_refusal_frame(
3878                    ctx,
3879                    &frame,
3880                    &target_module_id,
3881                    "removed",
3882                    error_codes::MODULE_REMOVED,
3883                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3884                )?]);
3885            }
3886            return Ok(vec![self.route_open_refusal_frame(
3887                ctx,
3888                &frame,
3889                &target_module_id,
3890                "not_registered",
3891                error_codes::UNKNOWN_MODULE,
3892                format!("module_id '{target_module_id}' is not registered"),
3893            )?]);
3894        };
3895
3896        // Best-effort only: registry readiness and forwarding reservation use
3897        // different locks, so a module can flip readiness between this read and
3898        // the relay. Modules must still tolerate an `on_bind` while not ready.
3899        if !registration.ready {
3900            self.counters
3901                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3902            info!(
3903                target: "control",
3904                code = error_codes::MODULE_WARMING,
3905                module_id = ?target_module_id,
3906                connection_id = ctx.connection_id.get(),
3907                reason = "declared_not_ready",
3908                "route.open refused"
3909            );
3910            // The module is registered but says it cannot take work, which is
3911            // an outage from the caller's side even though its process is up.
3912            self.route_outages
3913                .record_not_serving(&target_module_id, "declared_not_ready");
3914            return Ok(vec![control_error_body_frame(
3915                &frame,
3916                ErrorBody {
3917                    code: error_codes::MODULE_WARMING.to_string(),
3918                    message: format!(
3919                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3920                    ),
3921                    detail: Some(serde_json::json!({
3922                        "reason": "declared_not_ready"
3923                    })),
3924                },
3925            )?]);
3926        }
3927
3928        // Effective readiness, second half: a module that declares a capability
3929        // `need: required` is not routable while that capability has no
3930        // registered provider. It is enforced HERE, as a retryable routing
3931        // refusal, and deliberately not as spawn ordering or a boot block. The
3932        // module is still started and registered and can make its own calls;
3933        // spawn ordering is a promise that cannot be kept once a provider
3934        // crashes at runtime, and refusing to boot would stop the whole
3935        // machine, including the tools needed to fix its configuration.
3936        //
3937        // "Provided" is the evaluator's verdict, which counts a provider as
3938        // soon as it has REGISTERED, not once it is ready. Two modules that
3939        // require each other's capabilities are therefore both routable once
3940        // both register; counting readiness instead would deadlock them.
3941        //
3942        // Only new opens are refused. Routes already bound when a provider
3943        // goes away stay bound: nothing here tears them down, and the module
3944        // answers them as it can. Like the readiness read above this is
3945        // best-effort against a provider registering or leaving concurrently.
3946        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
3947            self.counters
3948                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3949            info!(
3950                target: "control",
3951                code = error_codes::MODULE_WARMING,
3952                module_id = ?target_module_id,
3953                connection_id = ctx.connection_id.get(),
3954                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3955                capability = %capability,
3956                "route.open refused"
3957            );
3958            return Ok(vec![control_error_body_frame(
3959                &frame,
3960                ErrorBody {
3961                    code: error_codes::MODULE_WARMING.to_string(),
3962                    message: format!(
3963                        "module_id '{target_module_id}' requires capability '{capability}', \
3964                         which no registered module provides; retry"
3965                    ),
3966                    detail: Some(serde_json::json!({
3967                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3968                        "capability": capability,
3969                    })),
3970                },
3971            )?]);
3972        }
3973
3974        if !target_has_required_role(&target, &registration.manifest.provides) {
3975            return Ok(vec![self.route_open_refusal_frame(
3976                ctx,
3977                &frame,
3978                &target_module_id,
3979                "role_not_provided",
3980                "target_unavailable",
3981                format!("module_id '{target_module_id}' does not provide the requested target"),
3982            )?]);
3983        }
3984
3985        if registration.state != ChannelState::Active {
3986            return Ok(vec![self.route_open_refusal_frame(
3987                ctx,
3988                &frame,
3989                &target_module_id,
3990                "registration_not_active",
3991                "target_unavailable",
3992                format!("module_id '{target_module_id}' is not active"),
3993            )?]);
3994        }
3995
3996        if self
3997            .forwarding
3998            .module_is_draining(&target_module_id)
3999            .map_err(RouterError::Forwarding)?
4000        {
4001            return Ok(vec![self.route_open_refusal_frame(
4002                ctx,
4003                &frame,
4004                &target_module_id,
4005                "reloading",
4006                "module_reloading",
4007                format!("module_id '{target_module_id}' is reloading"),
4008            )?]);
4009        }
4010
4011        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4012            process_liveness.process_live(&target_module_id) == Some(false)
4013        }) {
4014            // A module the supervisor is restarting or reloading can still hold
4015            // a registration: the old process before its connection closes, or
4016            // a new one that registered while the supervisor was draining. The
4017            // forwarding table does not see that as draining, but the consumer
4018            // should still be told to retry soon, exactly as for the drain
4019            // above, rather than that the target is unavailable.
4020            if process_liveness.process_replacing(&target_module_id) {
4021                return Ok(vec![self.route_open_refusal_frame(
4022                    ctx,
4023                    &frame,
4024                    &target_module_id,
4025                    "reloading",
4026                    "module_reloading",
4027                    format!("module_id '{target_module_id}' is reloading"),
4028                )?]);
4029            }
4030            return Ok(vec![self.route_open_refusal_frame(
4031                ctx,
4032                &frame,
4033                &target_module_id,
4034                "supervisor_not_live",
4035                "target_unavailable",
4036                format!("module_id '{target_module_id}' is not live"),
4037            )?]);
4038        }
4039
4040        if !self
4041            .forwarding
4042            .has_live_module_connection(&target_module_id)
4043            .map_err(RouterError::Forwarding)?
4044        {
4045            return Ok(vec![self.route_open_refusal_frame(
4046                ctx,
4047                &frame,
4048                &target_module_id,
4049                "no_forwarding_connection",
4050                "target_unavailable",
4051                format!("module_id '{target_module_id}' has no live forwarding connection"),
4052            )?]);
4053        }
4054
4055        if let Some(error) =
4056            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4057        {
4058            self.observe_route_open_refusal(
4059                ctx,
4060                &target_module_id,
4061                "op_not_allowed",
4062                "op_not_allowed",
4063            );
4064            return Ok(vec![error]);
4065        }
4066
4067        let principal = match self.route_open_principal(&frame, consumer_identity)? {
4068            Ok(principal) => principal,
4069            Err(error) => {
4070                self.observe_route_open_refusal(
4071                    ctx,
4072                    &target_module_id,
4073                    "bad_consumer_identity",
4074                    "bad_consumer_identity",
4075                );
4076                return Ok(vec![error]);
4077            }
4078        };
4079
4080        // This is attested, control-plane policy for supervised module origins.
4081        // Keep it before route reservation and out of the opaque forwarding hot
4082        // path: data frames must never acquire a per-frame capability check.
4083        if let Principal::Reserved {
4084            module_id: opening_module_id,
4085        } = &principal
4086        {
4087            if let Some(opening_registration) = self
4088                .registry
4089                .get_module(opening_module_id)
4090                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4091            {
4092                if let Some(capability) =
4093                    denied_capability(&opening_registration.manifest, &registration.manifest)
4094                {
4095                    warn!(
4096                        opening_module_id,
4097                        target_module_id,
4098                        capability,
4099                        "refusing route.open because an attested capability deny edge matches"
4100                    );
4101                    return Ok(vec![self.route_open_refusal_frame(
4102                        ctx,
4103                        &frame,
4104                        &target_module_id,
4105                        "capability_deny_edge",
4106                        "capability_forbidden",
4107                        format!(
4108                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4109                        ),
4110                    )?]);
4111                }
4112            }
4113        }
4114
4115        if admission_facts.is_some() {
4116            let carrier_matches = matches!(
4117                &principal,
4118                Principal::Reserved { module_id }
4119                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4120            );
4121            if !carrier_matches {
4122                return Ok(vec![self.route_open_refusal_frame(
4123                    ctx,
4124                    &frame,
4125                    &target_module_id,
4126                    "admission_facts_carrier_not_permitted",
4127                    "admission_facts_not_permitted",
4128                    "admission facts may only be carried by the configured reserved module",
4129                )?]);
4130            }
4131
4132            let target_allowed = self
4133                .admission_facts_targets
4134                .as_ref()
4135                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4136            if !target_allowed {
4137                return Ok(vec![self.route_open_refusal_frame(
4138                    ctx,
4139                    &frame,
4140                    &target_module_id,
4141                    "admission_facts_target_not_listed",
4142                    "admission_facts_target_not_allowed",
4143                    format!(
4144                        "admission facts are not permitted for target module_id '{target_module_id}'"
4145                    ),
4146                )?]);
4147            }
4148
4149            // Keep the value opaque to subc. The downstream admission validator owns
4150            // schema and semantic checks; this daemon only enforces carrier authority
4151            // and the configured destination allowlist.
4152        }
4153
4154        // Scope admission, on the attested principal above and never on the
4155        // request body. The tag read here travels with the pending bind and is
4156        // compared with the published one at commit, so a sync between here
4157        // and the module's ack refuses the open instead of binding a stamp
4158        // that is no longer true.
4159        let (bound_scope, mut scope_stamp) = match scope {
4160            None => (None, None),
4161            Some(selector) => {
4162                let owner_configured = match &selector.owner {
4163                    // A reserved owner counts as configured from before its
4164                    // process is spawned (see `SupervisorHandle::is_configured`).
4165                    // So an owner that has not synced its scopes yet is refused
4166                    // as retryable (`scope_not_synced`), not as one that will
4167                    // never sync.
4168                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4169                    _ => false,
4170                };
4171                let admitted = self
4172                    .scopes
4173                    .read()
4174                    .unwrap_or_else(|poisoned| poisoned.into_inner())
4175                    .admit(&principal, &target_module_id, &selector, owner_configured)
4176                    .and_then(|admission| {
4177                        crate::scopes::check_target_flow_support(
4178                            &admission.stamp,
4179                            &target_module_id,
4180                            registration.manifest.capabilities.as_ref(),
4181                        )?;
4182                        crate::scopes::check_target_agent_run_support(
4183                            &admission.stamp,
4184                            &target_module_id,
4185                            registration.manifest.capabilities.as_ref(),
4186                        )?;
4187                        Ok(admission)
4188                    });
4189                match admitted {
4190                    Ok(admission) => (
4191                        Some(BoundScope {
4192                            owner: admission.owner,
4193                            scope_ref: admission.stamp.scope_ref.clone(),
4194                            tag: admission.tag,
4195                        }),
4196                        Some(admission.stamp),
4197                    ),
4198                    Err(refusal) => {
4199                        return Ok(vec![self.route_open_refusal_frame(
4200                            ctx,
4201                            &frame,
4202                            &target_module_id,
4203                            refusal.code,
4204                            refusal.code,
4205                            refusal.message,
4206                        )?]);
4207                    }
4208                }
4209            }
4210        };
4211
4212        // Bind admits a root that no longer exists on disk, because refusing here
4213        // closes the only exit from a paused run: cancel needs a bound route, and a
4214        // renamed or reclaimed directory makes that route unopenable forever. The
4215        // run itself is intact and still addressable by its recorded identity.
4216        //
4217        // This does NOT relax the rule the strict constructor protects. That rule is
4218        // that no root is ever aliased into NEW durable state -- a missing component
4219        // can reappear as a symlink elsewhere, which would move the identity and
4220        // split a session's history across two of them. The engine now refuses the
4221        // two operations that create such state (send and import) at admission,
4222        // which is a narrower way to hold the same invariant: reads and terminations
4223        // are admitted, writes are not. That refusal had to ship before this line
4224        // changed, or there is an interval where a send commits under a provisional
4225        // identity -- the exact failure the original policy existed to prevent.
4226        //
4227        // Resolution follows realpath rather than lexical cleanup: the longest
4228        // existing ancestor is canonicalized and the missing tail re-appended, so a
4229        // live root is unchanged and a vanished leaf keeps the identity it was
4230        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
4231        // same caller the moment the directory vanished, which strands the run more
4232        // quietly than refusing it.
4233        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4234            Ok(project_root) => project_root,
4235            Err(err) => {
4236                return Ok(vec![control_error_frame(
4237                    &frame,
4238                    "invalid_project_root",
4239                    err.to_string(),
4240                )?])
4241            }
4242        };
4243        identity.project_root = project_root.as_path().to_path_buf();
4244
4245        // Last gate before any relay work, and deliberately after the cheap
4246        // registry and availability checks above: those name a more precise
4247        // condition (unknown, removed, reloading) and a caller is better served
4248        // by the precise code than by this one.
4249        //
4250        // Everything below this point costs an egress permit, a reserved handle
4251        // pair and, if the module does not answer, the whole relay budget. The
4252        // reader no longer waits for that budget, so cap each target explicitly;
4253        // serial dispatch used to provide the accidental cap of one relay per
4254        // connection. Admission is a mutex-protected count and never waits.
4255        let _concurrency_guard = match self
4256            .route_bind_concurrency
4257            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4258        {
4259            Ok(guard) => guard,
4260            Err(in_flight) => {
4261                return Ok(vec![self.route_open_target_capacity_refusal(
4262                    ctx,
4263                    &frame,
4264                    &target_module_id,
4265                    in_flight,
4266                )?]);
4267            }
4268        };
4269
4270        // A module that has already burned the whole budget `threshold` times
4271        // in a row does not get to charge it again until a probe says it recovered.
4272        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4273            RouteBindAdmission::Admitted { guard, probe } => {
4274                if probe {
4275                    info!(
4276                        module_id = %target_module_id,
4277                        connection_id = ctx.connection_id.get(),
4278                        "route.bind breaker half-open: admitting one probe"
4279                    );
4280                }
4281                guard
4282            }
4283            RouteBindAdmission::Refused {
4284                consecutive_timeouts,
4285                retry_in,
4286                probe_in_flight,
4287            } => {
4288                return Ok(vec![self.route_open_breaker_refusal_frame(
4289                    ctx,
4290                    &frame,
4291                    &target_module_id,
4292                    consecutive_timeouts,
4293                    retry_in,
4294                    probe_in_flight,
4295                )?]);
4296            }
4297        };
4298
4299        // Resolve the per-module budget here so the wait matches the operator's
4300        // intent for this specific target. A per-module override in
4301        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
4302        // daemons) wins over the daemon-wide default.
4303        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4304        let relay_deadline = Instant::now() + route_bind_relay_timeout;
4305        let pending = match self
4306            .forwarding
4307            .begin_route_bind_relay_for(
4308                ctx.connection_id,
4309                ctx.egress.clone(),
4310                response_version(&frame),
4311                frame.header.corr,
4312                &target_module_id,
4313                principal.clone(),
4314                bound_scope,
4315                Some(project_root),
4316                relay_deadline,
4317            )
4318            .await
4319        {
4320            Ok(pending) => pending,
4321            Err(err) => {
4322                return Ok(vec![self.route_open_refusal_frame(
4323                    ctx,
4324                    &frame,
4325                    &target_module_id,
4326                    "relay_reservation_failed",
4327                    forwarding_error_code(&err),
4328                    err.to_string(),
4329                )?])
4330            }
4331        };
4332        let crate::forwarding::PendingRouteBindRelay {
4333            endpoint,
4334            module_sink,
4335            negotiated_ver,
4336            client_channel,
4337            client_epoch,
4338            module_channel,
4339            module_epoch,
4340            corr: relay_corr,
4341            receiver,
4342        } = pending;
4343        let mut reservation =
4344            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4345
4346        // Reserving egress can wait while a module reconnects or a swap cuts
4347        // over. Check the relay's captured connection, not the earlier by-id
4348        // lookup: the original module's flow/run capabilities cannot authorize
4349        // its replacement. The captured sink stays bound to that connection.
4350        let relay_registration = self
4351            .registry
4352            .get_module_by_connection(endpoint.connection_id)
4353            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4354        let relay_capabilities = relay_registration
4355            .as_ref()
4356            .and_then(|registration| registration.manifest.capabilities.as_ref());
4357        if let Some(stamp) = scope_stamp
4358            .as_ref()
4359            .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4360        {
4361            if let Err(refusal) = crate::scopes::check_target_flow_support(
4362                stamp,
4363                &target_module_id,
4364                relay_capabilities,
4365            )
4366            .and_then(|()| {
4367                crate::scopes::check_target_agent_run_support(
4368                    stamp,
4369                    &target_module_id,
4370                    relay_capabilities,
4371                )
4372            }) {
4373                reservation.release_and_disarm();
4374                return Ok(vec![self.route_open_refusal_frame(
4375                    ctx,
4376                    &frame,
4377                    &target_module_id,
4378                    refusal.code,
4379                    refusal.code,
4380                    refusal.message,
4381                )?]);
4382            }
4383        }
4384
4385        // Older readers may reject unknown fields. Select both new fields from
4386        // the captured endpoint's declaration, including on unscoped binds.
4387        let call_mode = if relay_capabilities
4388            .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4389        {
4390            Some(derive_call_mode(scope_stamp.as_ref()))
4391        } else {
4392            if let Some(stamp) = scope_stamp.as_mut() {
4393                stamp.expires_at_ms = None;
4394            }
4395            None
4396        };
4397
4398        debug!(
4399            connection_id = ctx.connection_id.get(),
4400            client_channel,
4401            client_epoch,
4402            module_channel,
4403            module_epoch,
4404            "reserved route handle pair"
4405        );
4406        // Rendered BEFORE the move into the relay, because the accept arm below
4407        // is where it is logged and the principal is gone by then.
4408        let principal_label = principal_label(&principal);
4409        let relay = ModuleControlRequest::RouteBind {
4410            route_channel: module_channel,
4411            epoch: module_epoch,
4412            target,
4413            identity,
4414            principal: Some(principal),
4415            consumer_capabilities,
4416            role_versions,
4417            admission_facts,
4418            scope: scope_stamp,
4419            call_mode,
4420        };
4421        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4422            RouterError::backend(
4423                0,
4424                frame.header.corr,
4425                format!("failed to encode route.bind request: {err}"),
4426            )
4427        })?;
4428        let relay_frame = Frame::build_with_version(
4429            negotiated_ver,
4430            FrameType::Request,
4431            control_flags(),
4432            0,
4433            0,
4434            relay_corr,
4435            relay_body,
4436        )
4437        .map_err(RouterError::FrameBuild)?;
4438
4439        if let Err(err) = module_sink.send(relay_frame).await {
4440            reservation.release_and_disarm();
4441            return Ok(vec![self.route_open_refusal_frame(
4442                ctx,
4443                &frame,
4444                &target_module_id,
4445                "relay_send_failed",
4446                "target_unavailable",
4447                err.to_string(),
4448            )?]);
4449        }
4450
4451        #[cfg(test)]
4452        if let Some(hold) = &self.route_bind_relay_sent_hold {
4453            // First wait: tell the test the bind is queued. Second wait: stay
4454            // here until the test has driven whatever it wants to happen
4455            // between queueing the bind and marking it enqueued.
4456            hold.wait().await;
4457            hold.wait().await;
4458        }
4459
4460        // A missing relay is not by itself an abandoned bind: the module may
4461        // already have acked it and the route may be committed. Only a relay
4462        // dropped without any module reply needs this GOODBYE; one a reply
4463        // settled has already had its GOODBYE decided by the reply path.
4464        if self
4465            .forwarding
4466            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4467            .map_err(RouterError::Forwarding)?
4468            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4469        {
4470            self.send_abandoned_route_bind_goodbye(
4471                &module_sink,
4472                negotiated_ver,
4473                module_channel,
4474                module_epoch,
4475            );
4476        }
4477
4478        match timeout_at(relay_deadline, receiver).await {
4479            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4480                reservation.disarm();
4481                if breaker.record_accepted() {
4482                    info!(
4483                        module_id = %target_module_id,
4484                        "route.bind breaker closed: the probe was accepted"
4485                    );
4486                }
4487                self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4488                Ok(Vec::new())
4489            }
4490            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4491                reservation.release_and_disarm();
4492                // A module that says no in microseconds is healthy. Rejection
4493                // is a different condition with its own refusal and must not
4494                // move the breaker.
4495                breaker.record_inconclusive();
4496                // The daemon's own commit re-check refused the bind because the
4497                // scope ended or changed after admission. The module accepted;
4498                // counting it as a module rejection would blame the module.
4499                let scope_code = match body.code.as_str() {
4500                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4501                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4502                    _ => None,
4503                };
4504                if let Some(code) = scope_code {
4505                    self.observe_route_open_refusal(
4506                        ctx,
4507                        &target_module_id,
4508                        "scope_changed_before_commit",
4509                        code,
4510                    );
4511                    return Ok(vec![control_error_body_frame(&frame, body)?]);
4512                }
4513                self.counters
4514                    .increment_route_open_refused("module_rejected");
4515                info!(
4516                    target: "control",
4517                    code = "module_rejected",
4518                    module_code = ?body.code,
4519                    module_id = ?target_module_id,
4520                    connection_id = ctx.connection_id.get(),
4521                    "route.open refused"
4522                );
4523                Ok(vec![control_error_body_frame(&frame, body)?])
4524            }
4525            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4526                reservation.release_and_disarm();
4527                breaker.record_inconclusive();
4528                // Fires when the module's connection closes while a relayed
4529                // bind is pending -- typically a caller racing a module restart
4530                // whose bind was relayed BEFORE the drain mark went up. Logged
4531                // because the caller sees only its own error and the fleet has
4532                // already spent one diagnosis round unable to tell this arm
4533                // from a relay timeout without daemon-side evidence.
4534                tracing::warn!(
4535                    module_id = %target_module_id,
4536                    "route.bind relay abandoned: {message}"
4537                );
4538                Ok(vec![self.route_open_refusal_frame(
4539                    ctx,
4540                    &frame,
4541                    &target_module_id,
4542                    "relay_abandoned",
4543                    "target_unavailable",
4544                    message,
4545                )?])
4546            }
4547            Ok(Err(_)) => {
4548                reservation.release_and_disarm();
4549                breaker.record_inconclusive();
4550                Ok(vec![self.route_open_refusal_frame(
4551                    ctx,
4552                    &frame,
4553                    &target_module_id,
4554                    "relay_waiter_canceled",
4555                    "target_unavailable",
4556                    "route.bind relay waiter was canceled before the module responded",
4557                )?])
4558            }
4559            Err(_) => {
4560                reservation.release_and_disarm();
4561                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
4562                // answer at all is the one condition a fast refusal can
4563                // usefully stand in for; every other arm already answered.
4564                if let Some(opened) = breaker.record_timeout(
4565                    self.route_bind_breaker_threshold,
4566                    self.route_bind_breaker_cooldown,
4567                ) {
4568                    warn!(
4569                        module_id = %target_module_id,
4570                        consecutive_timeouts = opened.consecutive_timeouts,
4571                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4572                        reopened_after_probe = opened.reopened_after_probe,
4573                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4574                    );
4575                }
4576                // The generous budget just burned to no answer: the module is
4577                // registered and its connection is up, but its bind handler sat
4578                // on the ack for the full budget (warm-on-bind, cold configure,
4579                // or a wedged handler). Every earlier unavailability shape
4580                // fast-refuses BEFORE the relay, so this arm firing means the
4581                // slowness is module-side -- log it so the per-module timeline
4582                // is reconstructable without client audit rows.
4583                tracing::warn!(
4584                    module_id = %target_module_id,
4585                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4586                    "route.bind relay timed out: module did not ack within budget"
4587                );
4588                Ok(vec![self.route_open_refusal_frame(
4589                    ctx,
4590                    &frame,
4591                    &target_module_id,
4592                    "relay_timed_out",
4593                    "module_timeout",
4594                    format!(
4595                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
4596                        route_bind_relay_timeout
4597                    ),
4598                )?])
4599            }
4600        }
4601    }
4602
4603    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4604        let response = ClientControlResponse::SupervisorSpawnSnapshot {
4605            snapshot: self.supervisor.spawn_snapshot(),
4606        };
4607        Ok(vec![control_response_body_frame(
4608            &frame,
4609            &response,
4610            "ClientControlResponse::SupervisorSpawnSnapshot",
4611        )?])
4612    }
4613
4614    fn handle_supervisor_spawn_subscribe(
4615        &self,
4616        ctx: &RouteCtx,
4617        frame: Frame,
4618        since: Option<SpawnCursor>,
4619    ) -> Result<Vec<Frame>, RouterError> {
4620        match self.supervisor.subscribe_spawns(
4621            ctx.connection_id,
4622            frame.header.corr,
4623            response_version(&frame),
4624            since,
4625            ctx.egress.clone(),
4626        ) {
4627            Ok(()) => Ok(Vec::new()),
4628            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4629                Ok(vec![control_error_body_frame(
4630                    &frame,
4631                    ErrorBody {
4632                        code: "spawn_cursor_incarnation_mismatch".to_string(),
4633                        message: "spawn cursor belongs to a different daemon incarnation"
4634                            .to_string(),
4635                        detail: Some(serde_json::json!({
4636                            "current_daemon_incarnation": current
4637                        })),
4638                    },
4639                )?])
4640            }
4641            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4642                &frame,
4643                ErrorBody {
4644                    code: "spawn_cursor_too_old".to_string(),
4645                    message: "spawn cursor predates the retained event ring".to_string(),
4646                    detail: Some(serde_json::json!({
4647                        "oldest_retained_cursor": oldest
4648                    })),
4649                },
4650            )?]),
4651            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4652                0,
4653                frame.header.corr,
4654                format!("failed to open supervisor spawn subscription: {error}"),
4655            )),
4656        }
4657    }
4658
4659    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4660        let generation = self
4661            .registry
4662            .generation()
4663            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4664        let mut modules = Vec::new();
4665        for module in self.supervisor.list() {
4666            let status = module.status_for_control("list").map_err(|err| {
4667                RouterError::backend(
4668                    0,
4669                    frame.header.corr,
4670                    format!("failed to read supervisor status: {err}"),
4671                )
4672            })?;
4673            let (configured, _) = module.configuration().map_err(|err| {
4674                RouterError::backend(
4675                    0,
4676                    frame.header.corr,
4677                    format!("failed to read module configuration: {err}"),
4678                )
4679            })?;
4680            // Status and configuration snapshots release their locks before the image probe awaits.
4681            let image = module.running_image_agreement().await;
4682            // Read per request so the figure is current when the operator asks;
4683            // the daemon samples nothing in between.
4684            let resources = Some(module.child_resource_usage());
4685            let pending_reload = Some(reload_verdict(
4686                &configured.program,
4687                status.spawned_from.as_deref(),
4688                image,
4689            ));
4690            // Keep the retired policy field on the wire for one release so
4691            // existing status consumers still receive the platform policy.
4692            modules.push(
4693                SupervisorEntry::new(
4694                    status.module_id,
4695                    status.state.to_string(),
4696                    status.enabled,
4697                    status.live,
4698                    status.health.status,
4699                )
4700                .with_launch_nonce_env(Some(!cfg!(unix)))
4701                .with_launch_nonce_source(status.launch_nonce_source)
4702                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
4703                .with_protocol(status.protocol)
4704                .with_pending_reload(pending_reload)
4705                .with_last_probe_ms(status.health.last_probe_ms)
4706                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
4707                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
4708                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
4709                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
4710                .with_restart_count(Some(status.restart_count))
4711                .with_max_restarts(Some(status.max_restarts))
4712                .with_lifetime_restarts(Some(status.lifetime_restarts))
4713                .with_spawn_generation(Some(status.spawn_generation))
4714                .with_restart_window_secs(Some(status.restart_window.as_secs()))
4715                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
4716                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
4717                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
4718                .with_resources(resources),
4719            );
4720        }
4721        let response = ClientControlResponse::SupervisorList {
4722            generation,
4723            modules,
4724        };
4725        Ok(vec![control_response_body_frame(
4726            &frame,
4727            &response,
4728            "ClientControlResponse::SupervisorList",
4729        )?])
4730    }
4731
4732    fn handle_supervisor_stderr_tail(
4733        &self,
4734        frame: Frame,
4735        module_id: String,
4736        max_lines: Option<u32>,
4737        max_bytes: Option<u32>,
4738    ) -> Result<Vec<Frame>, RouterError> {
4739        let Some(module) = self.supervisor.get(&module_id) else {
4740            return Ok(vec![control_error_frame(
4741                &frame,
4742                "unknown_module",
4743                format!("module_id '{module_id}' is not supervised"),
4744            )?]);
4745        };
4746
4747        let snapshot = module.stderr_tail(
4748            max_lines.map(|value| value as usize),
4749            max_bytes.map(|value| value as usize),
4750        );
4751
4752        let response = ClientControlResponse::SupervisorStderrTail {
4753            module_id,
4754            tail: StderrTail {
4755                capture: match snapshot.capture {
4756                    CaptureState::Captured => StderrCaptureState::Captured,
4757                    CaptureState::Incomplete { reason } => {
4758                        StderrCaptureState::Incomplete { reason }
4759                    }
4760                    CaptureState::NotCaptured { reason } => {
4761                        StderrCaptureState::NotCaptured { reason }
4762                    }
4763                },
4764                entries: snapshot
4765                    .entries
4766                    .into_iter()
4767                    .map(|entry| match entry {
4768                        TailEntry::Line {
4769                            text,
4770                            truncated,
4771                            at_ms,
4772                        } => StderrTailEntry::Line {
4773                            text,
4774                            truncated,
4775                            at_ms,
4776                        },
4777                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4778                    })
4779                    .collect(),
4780                dropped_lines: snapshot.dropped_lines,
4781            },
4782        };
4783        Ok(vec![control_response_body_frame(
4784            &frame,
4785            &response,
4786            "ClientControlResponse::SupervisorStderrTail",
4787        )?])
4788    }
4789
4790    async fn handle_supervisor_terminals(
4791        &self,
4792        frame: Frame,
4793        module_id: String,
4794    ) -> Result<Vec<Frame>, RouterError> {
4795        let Some(module) = self.supervisor.get(&module_id) else {
4796            return Ok(vec![control_error_frame(
4797                &frame,
4798                "unknown_module",
4799                format!("module_id '{module_id}' is not supervised"),
4800            )?]);
4801        };
4802
4803        // The journal read runs on a blocking thread: it can be megabytes of
4804        // file I/O and must not occupy a runtime worker.
4805        let terminals = module
4806            .read_durable_terminal_history()
4807            .await
4808            .map_err(|error| {
4809                RouterError::backend(
4810                    0,
4811                    frame.header.corr,
4812                    format!("failed to read terminal history: {error}"),
4813                )
4814            })?;
4815        let response = ClientControlResponse::SupervisorTerminals {
4816            module_id,
4817            terminals,
4818        };
4819        Ok(vec![control_response_body_frame(
4820            &frame,
4821            &response,
4822            "ClientControlResponse::SupervisorTerminals",
4823        )?])
4824    }
4825
4826    fn handle_supervisor_routes(
4827        &self,
4828        frame: Frame,
4829        module_id: Option<String>,
4830    ) -> Result<Vec<Frame>, RouterError> {
4831        let modules = self
4832            .forwarding
4833            .route_census(module_id.as_deref())
4834            .map_err(RouterError::Forwarding)?
4835            .into_iter()
4836            .map(|(module_id, routes)| SupervisorRouteModule {
4837                module_id,
4838                routes: routes
4839                    .into_iter()
4840                    .map(|route| SupervisorRoute {
4841                        consumer: match route.principal {
4842                            Principal::Reserved { module_id } => {
4843                                SupervisorRouteConsumer::Reserved { module_id }
4844                            }
4845                            Principal::Direct | Principal::Unverified => {
4846                                SupervisorRouteConsumer::Direct {
4847                                    connection_id: route.goodbye_target.connection_id.get(),
4848                                }
4849                            }
4850                        },
4851                        age_ms: Instant::now()
4852                            .saturating_duration_since(route.bound_at)
4853                            .as_millis()
4854                            .try_into()
4855                            .unwrap_or(u64::MAX),
4856                        draining: route.draining,
4857                        drain_reason: route.drain_reason,
4858                    })
4859                    .collect(),
4860            })
4861            .collect();
4862        let response = ClientControlResponse::SupervisorRoutes { modules };
4863        Ok(vec![control_response_body_frame(
4864            &frame,
4865            &response,
4866            "ClientControlResponse::SupervisorRoutes",
4867        )?])
4868    }
4869
4870    async fn handle_supervisor_provenance(
4871        &self,
4872        frame: Frame,
4873        module_id: Option<String>,
4874    ) -> Result<Vec<Frame>, RouterError> {
4875        let mut selected = if let Some(module_id) = module_id {
4876            let Some(module) = self.supervisor.get(&module_id) else {
4877                return Ok(vec![control_error_frame(
4878                    &frame,
4879                    "unknown_module",
4880                    format!("module_id '{module_id}' is not supervised"),
4881                )?]);
4882            };
4883            vec![module]
4884        } else {
4885            self.supervisor.list()
4886        };
4887
4888        let mut modules = Vec::with_capacity(selected.len());
4889        for module in selected.drain(..) {
4890            let (status, observed_image) = module
4891                .status_and_running_image_agreement()
4892                .await
4893                .map_err(|err| {
4894                    RouterError::backend(
4895                        0,
4896                        frame.header.corr,
4897                        format!("failed to read supervisor status: {err}"),
4898                    )
4899                })?;
4900            let module_declared = self
4901                .registry
4902                .get_module(&status.module_id)
4903                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4904                .and_then(|registration| registration.manifest.provenance)
4905                .map(|build| ModuleDeclaredProvenance::Reported { build })
4906                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4907            #[cfg(test)]
4908            let running_image = match &self.provenance_probe_override {
4909                Some(result) => result.clone(),
4910                None => observed_image,
4911            };
4912            #[cfg(not(test))]
4913            let running_image = observed_image;
4914            modules.push(SupervisorModuleProvenance {
4915                module_id: status.module_id,
4916                module_declared,
4917                daemon_observed: SupervisorObservedProcess {
4918                    launch_nonce_source: status.launch_nonce_source,
4919                    pid: status.pid,
4920                    spawned_at_ms: status.spawned_at_ms,
4921                    spawned_from: status.spawned_from,
4922                    running_image,
4923                },
4924            });
4925        }
4926        let daemon = SupervisorDaemonProvenance {
4927            daemon_build: self.daemon_provenance.build.clone(),
4928            daemon_observed: DaemonObservedProcess {
4929                pid: self.daemon_provenance.pid,
4930                started_at_ms: self
4931                    .daemon_provenance
4932                    .start_clock
4933                    .map(|clock| clock.started_at_ms())
4934                    .or(self.daemon_provenance.started_at_ms),
4935                running_image: self
4936                    .daemon_provenance
4937                    .probe
4938                    .observe(
4939                        self.daemon_provenance.pid,
4940                        self.daemon_provenance.executable_path.as_deref(),
4941                        self.daemon_provenance.executable_identity.clone(),
4942                        self.daemon_provenance.process_start_time,
4943                    )
4944                    .await,
4945            },
4946        };
4947        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4948        Ok(vec![control_response_body_frame(
4949            &frame,
4950            &response,
4951            "ClientControlResponse::SupervisorProvenance",
4952        )?])
4953    }
4954
4955    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4956        self.refresh_capability_requirements();
4957        let generation = self
4958            .registry
4959            .generation()
4960            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4961        let modules = self
4962            .supervisor
4963            .list()
4964            .into_iter()
4965            .map(|module| {
4966                let status = module.status_for_control("health").map_err(|err| {
4967                    RouterError::backend(
4968                        0,
4969                        frame.header.corr,
4970                        format!("failed to read supervisor health: {err}"),
4971                    )
4972                })?;
4973                let module_id = status.module_id;
4974                let capability_detail = self
4975                    .capability_evaluator
4976                    .required_problem_detail(&module_id);
4977                Ok(SupervisorHealthEntry {
4978                    module_id,
4979                    status: status.health.status,
4980                    detail: append_capability_problem_detail(
4981                        status.health.detail,
4982                        capability_detail,
4983                    ),
4984                    metrics: status.health.metrics,
4985                    consecutive_failures: status.health.consecutive_failures,
4986                    late_answer_count: status.health.late_answer_count,
4987                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4988                    last_action: status.health.last_action,
4989                    last_action_ms: status.health.last_action_ms,
4990                    last_probe_ms: status.health.last_probe_ms,
4991                })
4992            })
4993            .collect::<Result<Vec<_>, RouterError>>()?;
4994        let response = ClientControlResponse::SupervisorHealth {
4995            generation,
4996            modules,
4997        };
4998        Ok(vec![control_response_body_frame(
4999            &frame,
5000            &response,
5001            "ClientControlResponse::SupervisorHealth",
5002        )?])
5003    }
5004
5005    async fn handle_supervisor_restart(
5006        &self,
5007        frame: Frame,
5008        module_id: String,
5009        drain_timeout_ms: Option<u64>,
5010        requester: Principal,
5011    ) -> Result<Vec<Frame>, RouterError> {
5012        let operation_lock = self.supervisor.operation_lock();
5013        let _operation_guard = operation_lock.lock().await;
5014        let Some(module) = self.supervisor.get(&module_id) else {
5015            return Ok(vec![control_error_frame(
5016                &frame,
5017                "unknown_module",
5018                format!("module_id '{module_id}' is not supervised"),
5019            )?]);
5020        };
5021
5022        self.route_outages.mark_operator_action(&module_id);
5023        if let Err(err) = module
5024            .restart_requested(drain_timeout_ms, Some(requester))
5025            .await
5026        {
5027            self.route_outages
5028                .operator_action_ended_unrefused(&module_id);
5029            let (code, message) = match err {
5030                crate::supervise::SuperviseError::Disabled { .. } => {
5031                    ("module_disabled", err.to_string())
5032                }
5033                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5034                    ("swap_in_progress", err.to_string())
5035                }
5036                _ => (
5037                    "target_unavailable",
5038                    format!("failed to restart module_id '{module_id}': {err}"),
5039                ),
5040            };
5041            return Ok(vec![control_error_frame(&frame, code, message)?]);
5042        }
5043
5044        let response = ClientControlResponse::SupervisorAck {
5045            module_id,
5046            applied: true,
5047        };
5048        Ok(vec![control_response_body_frame(
5049            &frame,
5050            &response,
5051            "ClientControlResponse::SupervisorAck",
5052        )?])
5053    }
5054
5055    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
5056    /// when the old process has finished draining: a caller whose own lane
5057    /// rides the old process must get its reply before that drain waits on it.
5058    async fn handle_supervisor_swap(
5059        &self,
5060        frame: Frame,
5061        module_id: String,
5062        ready_timeout_ms: Option<u64>,
5063        requester: Principal,
5064    ) -> Result<Vec<Frame>, RouterError> {
5065        // The daemon-wide operation lock is held only to resolve the handle,
5066        // not across the swap. The swap can take its whole readiness budget,
5067        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
5068        // holding it here would park an operator's stop behind the swap it is
5069        // meant to abort. A rescan or stop that reaches the module during the
5070        // swap is served by the swap itself (see `supervise_swap`).
5071        let module = {
5072            let operation_lock = self.supervisor.operation_lock();
5073            let _operation_guard = operation_lock.lock().await;
5074            self.supervisor.get(&module_id)
5075        };
5076        let Some(module) = module else {
5077            return Ok(vec![control_error_frame(
5078                &frame,
5079                "unknown_module",
5080                format!("module_id '{module_id}' is not supervised"),
5081            )?]);
5082        };
5083
5084        self.route_outages.mark_operator_action(&module_id);
5085        if let Err(err) = module
5086            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5087            .await
5088        {
5089            self.route_outages
5090                .operator_action_ended_unrefused(&module_id);
5091            use crate::supervise::SuperviseError;
5092            let message = err.to_string();
5093            let error = match err {
5094                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5095                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5096                    code: "swap_refused".to_string(),
5097                    message,
5098                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5099                },
5100                SuperviseError::SwapFailed {
5101                    arm,
5102                    candidate_exit,
5103                    ..
5104                } => ErrorBody {
5105                    code: "swap_failed".to_string(),
5106                    message,
5107                    detail: Some(serde_json::json!({
5108                        "arm": arm.as_str(),
5109                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5110                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5111                    })),
5112                },
5113                _ => ErrorBody::new(
5114                    "target_unavailable",
5115                    format!("failed to swap module_id '{module_id}': {message}"),
5116                ),
5117            };
5118            return Ok(vec![control_error_body_frame(&frame, error)?]);
5119        }
5120        // A completed swap kept the incumbent serving until cutover, so it
5121        // usually opened no outage; a mark left behind would make the next,
5122        // unrelated outage read as requested.
5123        self.route_outages
5124            .operator_action_ended_unrefused(&module_id);
5125
5126        let response = ClientControlResponse::SupervisorAck {
5127            module_id,
5128            applied: true,
5129        };
5130        Ok(vec![control_response_body_frame(
5131            &frame,
5132            &response,
5133            "ClientControlResponse::SupervisorAck",
5134        )?])
5135    }
5136
5137    async fn handle_supervisor_reload(
5138        &self,
5139        frame: Frame,
5140        module_id: String,
5141        requester: Principal,
5142    ) -> Result<Vec<Frame>, RouterError> {
5143        let operation_lock = self.supervisor.operation_lock();
5144        let _operation_guard = operation_lock.lock().await;
5145        let Some(module) = self.supervisor.get(&module_id) else {
5146            return Ok(vec![control_error_frame(
5147                &frame,
5148                "unknown_module",
5149                format!("module_id '{module_id}' is not supervised"),
5150            )?]);
5151        };
5152
5153        self.route_outages.mark_operator_action(&module_id);
5154        if let Err(err) = module.reload_requested(Some(requester)).await {
5155            self.route_outages
5156                .operator_action_ended_unrefused(&module_id);
5157            let (code, message) = match err {
5158                crate::supervise::SuperviseError::Disabled { .. } => {
5159                    ("module_disabled", err.to_string())
5160                }
5161                crate::supervise::SuperviseError::SwapInProgress { .. } => {
5162                    ("swap_in_progress", err.to_string())
5163                }
5164                _ => (
5165                    "reload_failed",
5166                    format!("failed to reload module_id '{module_id}': {err}"),
5167                ),
5168            };
5169            return Ok(vec![control_error_frame(&frame, code, message)?]);
5170        }
5171
5172        let response = ClientControlResponse::SupervisorAck {
5173            module_id,
5174            applied: true,
5175        };
5176        Ok(vec![control_response_body_frame(
5177            &frame,
5178            &response,
5179            "ClientControlResponse::SupervisorAck",
5180        )?])
5181    }
5182
5183    async fn handle_supervisor_rescan(
5184        &self,
5185        frame: Frame,
5186        preview: bool,
5187        requester: Principal,
5188    ) -> Result<Vec<Frame>, RouterError> {
5189        let Some(context) = self.rescan.clone() else {
5190            return Ok(vec![control_error_frame(
5191                &frame,
5192                "rescan_unavailable",
5193                "the daemon was not started with a reloadable config path".to_string(),
5194            )?]);
5195        };
5196
5197        let operation_lock = self.supervisor.operation_lock();
5198        let _operation_guard = operation_lock.lock().await;
5199        let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5200            Ok(config) => config,
5201            Err(err) => {
5202                return Ok(vec![control_error_frame(
5203                    &frame,
5204                    "invalid_daemon_config",
5205                    format!("supervisor rescan rejected daemon config: {err}"),
5206                )?])
5207            }
5208        };
5209        // `load` reports a missing file as Ok(None), which is correct at boot
5210        // (no config, nothing to supervise) and catastrophic here: rescan treats
5211        // "not in the config" as "remove it", so an absent file would read as an
5212        // empty module list and retire the entire running fleet. An editor
5213        // writing via write-new-then-rename, or a half-finished edit, is enough
5214        // to open that window. Refuse instead: a config that cannot be read
5215        // carries no instruction to remove anything.
5216        let Some((config, activity_config)) = loaded else {
5217            return Ok(vec![control_error_frame(
5218                &frame,
5219                "invalid_daemon_config",
5220                format!(
5221                    "daemon config not found at {}; refusing to rescan (an absent config would \
5222                     retire every supervised module)",
5223                    context.config_path.display()
5224                ),
5225            )?]);
5226        };
5227        let (
5228            configured_port,
5229            storage_config,
5230            admission_facts_carrier_module_id,
5231            admission_facts_targets,
5232            scope_authority_owners,
5233            modules,
5234            reserved_capabilities,
5235        ) = (
5236            config.port,
5237            config.storage,
5238            config.admission_facts_carrier_module_id,
5239            config.admission_facts_targets,
5240            config.scope_authority_owners,
5241            config.modules,
5242            config.reserved_capabilities,
5243        );
5244
5245        // Collect the sections rescan cannot apply, so the REPLY carries them.
5246        //
5247        // The warning below has always been correct and has always gone only to
5248        // the journal -- addressed to whoever reads logs, while the person who
5249        // just edited the config is looking at the CLI. Naming each section
5250        // individually rather than setting a flag: "something outside modules
5251        // changed" sends the operator back to diffing their own file, which is
5252        // the work this is meant to save.
5253        let mut restart_required = Vec::new();
5254        for section in RestartRequiredSection::ALL {
5255            let changed = match section {
5256                RestartRequiredSection::Port => configured_port != context.configured_port,
5257                RestartRequiredSection::Storage => storage_config != context.storage_config,
5258                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5259                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5260                }
5261                RestartRequiredSection::AdmissionFactsTargets => {
5262                    admission_facts_targets != context.admission_facts_targets
5263                }
5264                RestartRequiredSection::ScopeAuthorityOwners => {
5265                    scope_authority_owners != context.scope_authority_owners
5266                }
5267            };
5268            if changed {
5269                restart_required.push(section.label().to_string());
5270            }
5271        }
5272        if !restart_required.is_empty() {
5273            warn!(
5274                config_path = %context.config_path.display(),
5275                sections = %restart_required.join(", "),
5276                "daemon config changed outside the modules section; restart the daemon to apply those changes"
5277            );
5278        }
5279
5280        for configured in &modules {
5281            if let Err(err) = validate_spec(&configured.module_spec()) {
5282                return Ok(vec![control_error_frame(
5283                    &frame,
5284                    "invalid_daemon_config",
5285                    format!("supervisor rescan rejected daemon config: {err}"),
5286                )?]);
5287            }
5288        }
5289
5290        let configured_capabilities = modules
5291            .iter()
5292            .map(|module| (module.module_id.clone(), module.enabled))
5293            .collect::<Vec<_>>();
5294        let preview_capability_warnings = if preview {
5295            let (_, registrations) = self.runtime_capability_snapshot()?;
5296            let current_modules = self
5297                .supervisor
5298                .list()
5299                .into_iter()
5300                .map(|module| module.module_id().to_string())
5301                .collect::<BTreeSet<_>>();
5302            let resulting_modules = configured_capabilities.clone();
5303            let removed = current_modules
5304                .into_iter()
5305                .filter(|module_id| {
5306                    !resulting_modules
5307                        .iter()
5308                        .any(|(configured_id, _)| configured_id == module_id)
5309                })
5310                .collect::<Vec<_>>();
5311            self.capability_evaluator.preview_removal_warnings(
5312                resulting_modules,
5313                &removed,
5314                &registrations,
5315            )
5316        } else {
5317            Vec::new()
5318        };
5319        let result = match self
5320            .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5321            .await
5322        {
5323            Ok(result) => result,
5324            Err(message) => {
5325                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5326            }
5327        };
5328        if !preview {
5329            if let Some(ledger) = &self.activity {
5330                ledger.module_rescanned(&result.added, &result.removed, requester);
5331            }
5332            if let Err(code) = self.configure_activity(activity_config).await {
5333                warn!(
5334                    code,
5335                    "activity retention could not be applied during rescan"
5336                );
5337            }
5338            self.capability_evaluator
5339                .configure(configured_capabilities, reserved_capabilities);
5340            self.capability_evaluator.wake_deadline_loop();
5341            self.refresh_capability_requirements();
5342        }
5343        let mut result = result;
5344        result.restart_required = restart_required;
5345        result.capability_warnings = preview_capability_warnings;
5346        let response = ClientControlResponse::SupervisorRescan { result };
5347        Ok(vec![control_response_body_frame(
5348            &frame,
5349            &response,
5350            "ClientControlResponse::SupervisorRescan",
5351        )?])
5352    }
5353
5354    async fn handle_supervisor_release_reserved(
5355        &self,
5356        frame: Frame,
5357        module_id: String,
5358    ) -> Result<Vec<Frame>, RouterError> {
5359        let Some(context) = self.rescan.clone() else {
5360            return Ok(vec![control_error_frame(
5361                &frame,
5362                "release_unavailable",
5363                "reserved-id release requires a daemon started with a reloadable config path",
5364            )?]);
5365        };
5366        let operation_lock = self.supervisor.operation_lock();
5367        let _operation_guard = operation_lock.lock().await;
5368        let loaded = match crate::daemon_config::load(&context.config_path) {
5369            Ok(Some(config)) => config,
5370            Ok(None) => {
5371                return Ok(vec![control_error_frame(
5372                    &frame,
5373                    "invalid_daemon_config",
5374                    format!(
5375                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5376                        context.config_path.display()
5377                    ),
5378                )?])
5379            }
5380            Err(err) => {
5381                return Ok(vec![control_error_frame(
5382                    &frame,
5383                    "invalid_daemon_config",
5384                    format!("unable to verify reserved-id release against daemon config: {err}"),
5385                )?])
5386            }
5387        };
5388        if loaded
5389            .modules
5390            .iter()
5391            .any(|configured| configured.module_id == module_id)
5392        {
5393            return Ok(vec![control_error_frame(
5394                &frame,
5395                "reserved_module_configured",
5396                format!(
5397                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5398                ),
5399            )?]);
5400        }
5401        if !self.supervisor.release_retained_reserved_gate(&module_id) {
5402            return Ok(vec![control_error_frame(
5403                &frame,
5404                "reserved_gate_not_retained",
5405                format!(
5406                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5407                ),
5408            )?]);
5409        }
5410
5411        let response = ClientControlResponse::SupervisorAck {
5412            module_id,
5413            applied: true,
5414        };
5415        Ok(vec![control_response_body_frame(
5416            &frame,
5417            &response,
5418            "ClientControlResponse::SupervisorAck",
5419        )?])
5420    }
5421
5422    /// Reconcile the running module set against the configured one.
5423    ///
5424    /// With `preview` set, the diff is computed and returned WITHOUT applying any
5425    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
5426    /// deliberately shares this function with the executing path rather than
5427    /// computing the same diff somewhere else -- two implementations of one
5428    /// decision agree until they do not, and the whole value of a preview is that
5429    /// it describes the operation that will actually run.
5430    async fn reconcile_supervised_modules(
5431        &self,
5432        supervisor: &Supervisor,
5433        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5434        preview: bool,
5435        requester: Principal,
5436    ) -> Result<SupervisorRescanResult, String> {
5437        let mut current = BTreeMap::new();
5438        for module in self.supervisor.list() {
5439            let (spec, health) = module.configuration().map_err(|err| {
5440                format!(
5441                    "failed to read configuration for module_id '{}': {err}",
5442                    module.module_id()
5443                )
5444            })?;
5445            let enabled = module
5446                .status()
5447                .map_err(|err| {
5448                    format!(
5449                        "failed to read status for module_id '{}': {err}",
5450                        module.module_id()
5451                    )
5452                })?
5453                .enabled;
5454            current.insert(
5455                module.module_id().to_string(),
5456                (module, spec, health, enabled),
5457            );
5458        }
5459        let configured = configured_modules
5460            .into_iter()
5461            .map(|module| (module.module_id.clone(), module))
5462            .collect::<BTreeMap<_, _>>();
5463
5464        let added = configured
5465            .keys()
5466            .filter(|module_id| !current.contains_key(*module_id))
5467            .cloned()
5468            .collect::<Vec<_>>();
5469        let removed = current
5470            .keys()
5471            .filter(|module_id| !configured.contains_key(*module_id))
5472            .cloned()
5473            .collect::<Vec<_>>();
5474        let mut changed_pending_reload = Vec::new();
5475        let mut configuration_changes = BTreeSet::new();
5476        let mut enabled_changes = BTreeSet::new();
5477        let mut unchanged = 0_u32;
5478
5479        for (module_id, configured_module) in &configured {
5480            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5481            else {
5482                continue;
5483            };
5484            // Compare the whole launch spec so a future launch field cannot
5485            // accidentally become a live-only policy change. Health is stored
5486            // separately and applies live without replacing the process.
5487            let launch_changed = *current_spec != configured_module.module_spec();
5488            let configuration_changed =
5489                launch_changed || *current_health != configured_module.health;
5490            let enabled_changed = *current_enabled != configured_module.enabled;
5491            if configuration_changed {
5492                configuration_changes.insert(module_id.clone());
5493            }
5494            if launch_changed {
5495                changed_pending_reload.push(module_id.clone());
5496            }
5497            if enabled_changed {
5498                enabled_changes.insert(module_id.clone());
5499            }
5500            if !configuration_changed && !enabled_changed {
5501                unchanged = unchanged.saturating_add(1);
5502            }
5503        }
5504
5505        // Everything above this point is pure computation over two snapshots.
5506        // Everything below MUTATES. The preview returns here so the boundary is a
5507        // single early return rather than a condition repeated at each mutation
5508        // site, where one missed guard would apply part of a change the caller was
5509        // told would not happen.
5510        if preview {
5511            return Ok(SupervisorRescanResult {
5512                added,
5513                removed,
5514                changed_pending_reload,
5515                enabled_changes: enabled_changes.iter().cloned().collect(),
5516                unchanged,
5517                preview: true,
5518                // Filled by the caller on both paths, so the preview reports
5519                // restart-required sections identically to an executed rescan --
5520                // the preview is where an operator is most likely to be looking.
5521                restart_required: Vec::new(),
5522                capability_warnings: Vec::new(),
5523            });
5524        }
5525
5526        for module_id in &removed {
5527            let module = &current
5528                .get(module_id)
5529                .expect("removed module came from current supervisor state")
5530                .0;
5531            module
5532                .retire_requested(Some(requester.clone()))
5533                .await
5534                .map_err(|err| {
5535                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
5536                })?;
5537            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
5538            //
5539            // `handle_route_open` resolves an absent module in three steps:
5540            // registry, then supervisor status, then tombstone. Retiring first
5541            // opens a window where ALL THREE ARE ABSENT -- the registry entry
5542            // went with the teardown above, the supervisor entry went with
5543            // `retire`, and the tombstone does not exist yet -- so a route.open
5544            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
5545            // it") for a module that was deliberately removed and whose caller
5546            // should get `module_removed` (TERMINAL, carrying a removal age).
5547            //
5548            // Writing the tombstone first closes it: during the window the
5549            // supervisor entry still answers, so the caller gets
5550            // `target_unavailable` -- retryable, and TRUE, because the module
5551            // is mid-teardown. After both statements it is `module_removed`.
5552            // No instant remains where a removed module reads as one that
5553            // never existed.
5554            //
5555            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
5556            // the absence of a test beside a fix invites deletion: these are
5557            // two sync statements with no await between them, so reaching the
5558            // window needs a second worker thread to land exactly between them
5559            // and there is no hook to force it. MEASURED: the 25 daemon_config
5560            // tests pass identically with the old order and the new one, so
5561            // the existing suite cannot see this and a green run is not
5562            // evidence either way. What the suite does hold is the
5563            // post-condition -- a removed module answers `module_removed` --
5564            // which this preserves.
5565            //
5566            // Found by an Athena panel reading the shipped tree against a
5567            // design note (2026-09-19), as the one concrete instance of that
5568            // note's class that survived contact with source. Direction is
5569            // benign: retryable where terminal was intended, never the reverse.
5570            self.supervisor.record_rescan_removal(module_id);
5571            self.supervisor.retire(module_id);
5572            self.route_outages.forget(module_id);
5573        }
5574
5575        for module_id in configured.keys() {
5576            let Some((module, _, _, _)) = current.get(module_id) else {
5577                continue;
5578            };
5579            let configured_module = configured
5580                .get(module_id)
5581                .expect("configured module id came from configured map");
5582            if configuration_changes.contains(module_id) {
5583                module
5584                    .update_configuration(
5585                        configured_module.module_spec(),
5586                        configured_module.health.clone(),
5587                        configured_module.drain_timeout_ms,
5588                    )
5589                    .await
5590                    .map_err(|err| {
5591                        format!(
5592                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
5593                        )
5594                    })?;
5595            }
5596            if enabled_changes.contains(module_id) {
5597                // A rescan that starts or stops a module applies an operator's
5598                // edit to the config, so the resulting outage was asked for.
5599                self.route_outages.mark_operator_action(module_id);
5600                module
5601                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
5602                    .await
5603                    .map_err(|err| {
5604                        self.route_outages.operator_action_ended_unrefused(module_id);
5605                        format!(
5606                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5607                            configured_module.enabled
5608                        )
5609                    })?;
5610            }
5611        }
5612
5613        for module_id in &added {
5614            let configured_module = configured
5615                .get(module_id)
5616                .expect("added module id came from configured map");
5617            supervisor
5618                .supervise_configured_requested(
5619                    configured_module.module_spec(),
5620                    configured_module.enabled,
5621                    configured_module.health.clone(),
5622                    configured_module.drain_timeout_ms,
5623                    configured_module.restart,
5624                    Some(requester.clone()),
5625                )
5626                .map_err(|err| {
5627                    format!("failed to add module_id '{module_id}' during rescan: {err}")
5628                })?;
5629        }
5630
5631        Ok(SupervisorRescanResult {
5632            added,
5633            removed,
5634            changed_pending_reload,
5635            enabled_changes: enabled_changes.iter().cloned().collect(),
5636            unchanged,
5637            preview: false,
5638            // Filled by the caller, which is the only layer that can see the
5639            // previous config to diff against.
5640            restart_required: Vec::new(),
5641            capability_warnings: Vec::new(),
5642        })
5643    }
5644
5645    async fn handle_supervisor_set_enabled(
5646        &self,
5647        frame: Frame,
5648        module_id: String,
5649        enabled: bool,
5650        requester: Principal,
5651    ) -> Result<Vec<Frame>, RouterError> {
5652        let operation_lock = self.supervisor.operation_lock();
5653        let _operation_guard = operation_lock.lock().await;
5654        let Some(module) = self.supervisor.get(&module_id) else {
5655            return Ok(vec![control_error_frame(
5656                &frame,
5657                "unknown_module",
5658                format!("module_id '{module_id}' is not supervised"),
5659            )?]);
5660        };
5661
5662        // Enabling counts as well as disabling: a module an operator starts
5663        // is refused until it registers, and that wait was asked for.
5664        self.route_outages.mark_operator_action(&module_id);
5665        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
5666            Ok(applied) => applied,
5667            Err(err) => {
5668                self.route_outages
5669                    .operator_action_ended_unrefused(&module_id);
5670                return Ok(vec![control_error_frame(
5671                    &frame,
5672                    "target_unavailable",
5673                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5674                )?]);
5675            }
5676        };
5677        if !applied {
5678            // Already in the requested state: nothing was made unavailable,
5679            // so the mark must not outlive this request.
5680            self.route_outages
5681                .operator_action_ended_unrefused(&module_id);
5682        }
5683
5684        self.capability_evaluator.wake_deadline_loop();
5685        self.refresh_capability_requirements();
5686        let response = ClientControlResponse::SupervisorAck { module_id, applied };
5687        Ok(vec![control_response_body_frame(
5688            &frame,
5689            &response,
5690            "ClientControlResponse::SupervisorAck",
5691        )?])
5692    }
5693
5694    async fn handle_supervisor_health_probe(
5695        &self,
5696        frame: Frame,
5697        module_id: String,
5698    ) -> Result<Vec<Frame>, RouterError> {
5699        self.refresh_capability_requirements();
5700        let Some(registration) = self
5701            .registry
5702            .get_module(&module_id)
5703            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5704        else {
5705            return Ok(vec![control_error_frame(
5706                &frame,
5707                "unknown_module",
5708                format!("module_id '{module_id}' is not registered"),
5709            )?]);
5710        };
5711
5712        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
5713        // named for it. Making `module_registration_grants_op` return false
5714        // unconditionally reddens five tests, and every one is named for something
5715        // else -- capability relay, probe/bind demultiplexing, supervision-only
5716        // probing. They exercise a successful advertisement check on the way to their
5717        // own subject.
5718        //
5719        // Real protection, fragile in a specific way: narrowing any of those tests to
5720        // focus on its stated subject would silently remove coverage nobody knows
5721        // they are carrying. Recorded here rather than as a sixth test, because the
5722        // useful fact is WHICH tests hold the guard up -- a new test would add
5723        // coverage without telling the next person what the existing ones quietly do.
5724        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5725        {
5726            return Ok(vec![control_error_frame(
5727                &frame,
5728                "health_not_advertised",
5729                format!("module_id '{module_id}' did not advertise health.check"),
5730            )?]);
5731        }
5732
5733        let deadline = Instant::now() + self.health_probe_timeout;
5734        let pending = match self.forwarding.begin_module_control_rpc_for(
5735            &module_id,
5736            MODULE_CONTROL_OP_HEALTH_CHECK,
5737            deadline,
5738        ) {
5739            Ok(pending) => pending,
5740            Err(err) => {
5741                return Ok(vec![control_error_frame(
5742                    &frame,
5743                    forwarding_error_code(&err),
5744                    err.to_string(),
5745                )?])
5746            }
5747        };
5748
5749        let PendingModuleControlRpc {
5750            endpoint,
5751            module_sink,
5752            negotiated_ver,
5753            corr: probe_corr,
5754            receiver,
5755        } = pending;
5756        let mut guard =
5757            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5758        let probe_body =
5759            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5760                RouterError::backend(
5761                    0,
5762                    frame.header.corr,
5763                    format!("failed to encode health.check request: {err}"),
5764                )
5765            })?;
5766        let probe_frame = Frame::build_with_version(
5767            negotiated_ver,
5768            FrameType::Request,
5769            control_flags(),
5770            0,
5771            0,
5772            probe_corr,
5773            probe_body,
5774        )
5775        .map_err(RouterError::FrameBuild)?;
5776
5777        if let Err(err) = module_sink.send(probe_frame).await {
5778            return Ok(vec![control_error_frame(
5779                &frame,
5780                "target_unavailable",
5781                err.to_string(),
5782            )?]);
5783        }
5784
5785        match timeout_at(deadline, receiver).await {
5786            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5787                guard.disarm();
5788                let Some(report) = response.health_report() else {
5789                    return Ok(vec![control_error_frame(
5790                        &frame,
5791                        "invalid_control_body",
5792                        "health.check RPC returned a non-health response",
5793                    )?]);
5794                };
5795                // Metrics go out whole here. The supervisor's cached snapshot
5796                // caps this blob (see truncate_health_metrics), and this path
5797                // exists precisely to answer without that cap -- so applying it
5798                // here would leave no way to see what the cached view drops.
5799                let HealthReport {
5800                    status,
5801                    detail,
5802                    metrics,
5803                } = report;
5804                let capability_detail = self
5805                    .capability_evaluator
5806                    .required_problem_detail(&module_id);
5807                let response = ClientControlResponse::SupervisorHealthProbe {
5808                    module_id,
5809                    status,
5810                    detail: append_capability_problem_detail(detail, capability_detail),
5811                    metrics,
5812                };
5813                Ok(vec![control_response_body_frame(
5814                    &frame,
5815                    &response,
5816                    "ClientControlResponse::SupervisorHealthProbe",
5817                )?])
5818            }
5819            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5820                guard.disarm();
5821                Ok(vec![control_error_body_frame(&frame, body)?])
5822            }
5823            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5824                guard.disarm();
5825                Ok(vec![control_error_frame(
5826                    &frame,
5827                    "target_unavailable",
5828                    message,
5829                )?])
5830            }
5831            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5832                guard.disarm();
5833                Ok(vec![control_error_frame(
5834                    &frame,
5835                    "invalid_control_body",
5836                    message,
5837                )?])
5838            }
5839            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5840                guard.disarm();
5841                Ok(vec![control_error_frame(
5842                    &frame,
5843                    "invalid_control_body",
5844                    format!("expected module-control op '{expected}', got '{actual}'"),
5845                )?])
5846            }
5847            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5848                guard.disarm();
5849                Ok(vec![control_error_frame(
5850                    &frame,
5851                    "module_timeout",
5852                    format!(
5853                        "module_id '{module_id}' answered health.check after {:?}",
5854                        self.health_probe_timeout
5855                    ),
5856                )?])
5857            }
5858            Ok(Err(_)) => Ok(vec![control_error_frame(
5859                &frame,
5860                "target_unavailable",
5861                "health.check waiter was canceled before the module responded",
5862            )?]),
5863            Err(_) => Ok(vec![control_error_frame(
5864                &frame,
5865                "module_timeout",
5866                format!(
5867                    "module_id '{module_id}' did not answer health.check within {:?}",
5868                    self.health_probe_timeout
5869                ),
5870            )?]),
5871        }
5872    }
5873
5874    fn supervisor_status(
5875        &self,
5876        module_id: &str,
5877        corr: u64,
5878    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5879        self.supervisor
5880            .get(module_id)
5881            .map(|module| {
5882                let warming = module.is_warming_for_control("status").map_err(|err| {
5883                    RouterError::backend(
5884                        0,
5885                        corr,
5886                        format!(
5887                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
5888                        ),
5889                    )
5890                })?;
5891                module.status_for_control("status").map_err(|err| {
5892                    RouterError::backend(
5893                        0,
5894                        corr,
5895                        format!(
5896                            "failed to read supervisor status for module_id '{module_id}': {err}"
5897                        ),
5898                    )
5899                }).map(|status| (status, warming))
5900            })
5901            .transpose()
5902    }
5903
5904    fn guard_module_control_op(
5905        &self,
5906        frame: &Frame,
5907        module_id: &str,
5908        op: &str,
5909    ) -> Result<Option<Frame>, RouterError> {
5910        if self.module_grants_op(module_id, op, frame.header.corr)? {
5911            return Ok(None);
5912        }
5913
5914        Ok(Some(control_error_frame(
5915            frame,
5916            "op_not_allowed",
5917            format!("module_id '{module_id}' did not grant control op '{op}'"),
5918        )?))
5919    }
5920
5921    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5922        let Some(registration) = self
5923            .registry
5924            .get_module(module_id)
5925            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5926        else {
5927            return Ok(false);
5928        };
5929        Ok(module_registration_grants_op(&registration.control_ops, op))
5930    }
5931
5932    fn handle_status_update(
5933        &self,
5934        endpoint: ModuleEndpointId,
5935        frame: Frame,
5936    ) -> Result<Vec<Frame>, RouterError> {
5937        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5938            Ok(update) => update,
5939            Err(err) => {
5940                // Forward-compat: a newer module may push a channel-0 op this subc
5941                // version doesn't know. The control contract says unknown push ops
5942                // are IGNORED, never answered with an error. Only a malformed body
5943                // for an op we DO know is a real error worth surfacing.
5944                if is_known_module_push_op(&frame.body) {
5945                    return Ok(vec![control_error_frame(
5946                        &frame,
5947                        "invalid_control_body",
5948                        format!("malformed module control push body: {err}"),
5949                    )?]);
5950                }
5951                return Ok(Vec::new());
5952            }
5953        };
5954
5955        match update {
5956            ModuleControlPush::RouteStatus {
5957                route_channel,
5958                route_epoch,
5959                status,
5960            } => {
5961                self.forwarding
5962                    .cache_status(endpoint, route_channel, route_epoch, status)
5963                    .map_err(RouterError::Forwarding)?;
5964            }
5965        }
5966        Ok(Vec::new())
5967    }
5968
5969    fn handle_route_poll(
5970        &self,
5971        ctx: &RouteCtx,
5972        frame: Frame,
5973        route_channel: u16,
5974        route_epoch: u32,
5975        kind: PollKind,
5976    ) -> Result<Vec<Frame>, RouterError> {
5977        let snapshot = self
5978            .forwarding
5979            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5980            .map_err(RouterError::Forwarding)?;
5981        let response = match (kind, snapshot) {
5982            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5983                ClientControlResponse::RoutePoll {
5984                    route_channel,
5985                    route_epoch,
5986                    status,
5987                    live: None,
5988                }
5989            }
5990            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5991                route_channel,
5992                route_epoch,
5993                status: None,
5994                live: None,
5995            },
5996            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5997                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
5998                // is what makes reporting `true` correct rather than a
5999                // confident guess. `process_live` returns None only when the
6000                // module id has no supervisor snapshot at all -- an
6001                // externally-started module the daemon did not spawn -- and
6002                // for those the supervisor has no opinion to offer, ever. It
6003                // is never None for a supervised module in an unknown state:
6004                // a supervised module always has a snapshot, and the answer
6005                // comes from `state == Running && process_alive`.
6006                //
6007                // The route is Bound, so the module completed a HELLO on a
6008                // live connection; "the process this route points at is
6009                // running" is therefore attested by the binding rather than
6010                // assumed. Reporting `false` for an unsupervised module would
6011                // be the actual lie -- it would tell a client its healthy
6012                // route is dead because the daemon does not manage the
6013                // process.
6014                //
6015                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
6016                // module whose liveness is genuinely unknown, e.g. a snapshot
6017                // that has not been populated yet -- THIS DEFAULT BECOMES
6018                // WRONG and must split: unsupervised stays true, unknown
6019                // becomes null so the client can tell the two apart. The
6020                // response field is already `Option<bool>`, so the wire can
6021                // carry that distinction today.
6022                let live = self
6023                    .process_liveness
6024                    .as_ref()
6025                    .and_then(|source| source.process_live(&module_id))
6026                    .unwrap_or(true);
6027                ClientControlResponse::RoutePoll {
6028                    route_channel,
6029                    route_epoch,
6030                    status: None,
6031                    live: Some(live),
6032                }
6033            }
6034            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6035                route_channel,
6036                route_epoch,
6037                status: None,
6038                live: Some(false),
6039            },
6040        };
6041
6042        Ok(vec![control_response_body_frame(
6043            &frame,
6044            &response,
6045            "ClientControlResponse::RoutePoll",
6046        )?])
6047    }
6048
6049    pub(crate) fn observe_module_control_completion(
6050        &self,
6051        completion: ModuleControlRpcCompletion,
6052    ) -> bool {
6053        match completion {
6054            ModuleControlRpcCompletion::Unknown => false,
6055            ModuleControlRpcCompletion::Settled => true,
6056            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6057                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6058                info!(
6059                    module_id = %module_id,
6060                    latency_ms,
6061                    "late health.check answer proves the module is alive"
6062                );
6063                match self
6064                    .supervisor
6065                    .record_late_health_answer(&module_id, latency_ms)
6066                {
6067                    Ok(true) => {}
6068                    Ok(false) => debug!(
6069                        module_id = %module_id,
6070                        latency_ms,
6071                        "late health.check answer has no active supervisor snapshot"
6072                    ),
6073                    Err(err) => warn!(
6074                        module_id = %module_id,
6075                        latency_ms,
6076                        error = %err,
6077                        "failed to record late health.check answer"
6078                    ),
6079                }
6080                true
6081            }
6082        }
6083    }
6084
6085    /// Decide whether a failure while settling a relayed `route.bind` belongs to
6086    /// the module connection whose frame is being handled, or to the client that
6087    /// relay was opened for.
6088    ///
6089    /// This runs on the MODULE connection's frame handler, where returning `Err`
6090    /// ends that connection -- and a module connection carries every client's
6091    /// routes to that module, so ending it costs the whole fleet its tools.
6092    /// `ConnectionClosing` carries the id of the connection that is closing, and
6093    /// when that id is a CLIENT's, the condition is entirely about that one
6094    /// client's route.open. A client-scoped condition has no authority over a
6095    /// shared module connection, so it is logged and the single relay is dropped:
6096    /// the client is going away, and `complete_pending_relay` already removed the
6097    /// relay before failing, so there is nothing left to settle. Anything that
6098    /// relay still reserved is released by that client's own connection teardown,
6099    /// which is already under way -- that is what "closing" means.
6100    ///
6101    /// Every other failure is a statement about THIS connection and stays fatal:
6102    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
6103    /// id in `ConnectionClosing` all mean this connection cannot keep serving
6104    /// frames correctly.
6105    fn refuse_to_end_module_connection_for_a_client(
6106        &self,
6107        module_connection_id: ConnectionId,
6108        corr: u64,
6109        err: ForwardingError,
6110    ) -> Result<(), RouterError> {
6111        if let ForwardingError::ConnectionClosing { connection_id } = err {
6112            if connection_id != module_connection_id {
6113                warn!(
6114                    module_connection_id = module_connection_id.get(),
6115                    client_connection_id = connection_id.get(),
6116                    corr,
6117                    "dropping a route.bind response for a closing client; the module connection keeps serving"
6118                );
6119                return Ok(());
6120            }
6121        }
6122        Err(RouterError::Forwarding(err))
6123    }
6124
6125    fn handle_module_relay_response(
6126        &self,
6127        connection_id: ConnectionId,
6128        frame: Frame,
6129    ) -> Result<Vec<Frame>, RouterError> {
6130        let mut secondary_error = None;
6131        let outcome = match frame.header.ty {
6132            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6133                Ok(probe) if probe.op == "route.bind" => {
6134                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6135                        Ok(ModuleControlResponse::RouteBindAck {}) => {
6136                            RouteBindRelayOutcome::Accepted
6137                        }
6138                        Ok(other) => {
6139                            let message =
6140                                format!("route.bind response carried unexpected body: {other:?}");
6141                            secondary_error = Some(control_error_frame(
6142                                &frame,
6143                                "invalid_control_body",
6144                                message.clone(),
6145                            )?);
6146                            RouteBindRelayOutcome::ModuleGone(message)
6147                        }
6148                        Err(err) => {
6149                            let message = format!("malformed route.bind response body: {err}");
6150                            secondary_error = Some(control_error_frame(
6151                                &frame,
6152                                "invalid_control_body",
6153                                message.clone(),
6154                            )?);
6155                            RouteBindRelayOutcome::ModuleGone(message)
6156                        }
6157                    }
6158                }
6159                Ok(probe) => {
6160                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6161                    {
6162                        Ok(response) => ModuleControlRpcOutcome::Response(response),
6163                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6164                            "malformed {} response body: {err}",
6165                            probe.op
6166                        )),
6167                    };
6168                    let completion = self
6169                        .forwarding
6170                        .complete_module_control_rpc(
6171                            connection_id,
6172                            frame.header.corr,
6173                            Some(&probe.op),
6174                            outcome,
6175                        )
6176                        .map_err(RouterError::Forwarding)?;
6177                    if !self.observe_module_control_completion(completion) {
6178                        debug!(
6179                            connection_id = connection_id.get(),
6180                            corr = frame.header.corr,
6181                            op = %probe.op,
6182                            "dropping late or unknown module-control RPC response"
6183                        );
6184                    }
6185                    return Ok(Vec::new());
6186                }
6187                Err(err) => {
6188                    if let Some(expected_op) = self
6189                        .forwarding
6190                        .pending_module_control_op(connection_id, frame.header.corr)
6191                        .map_err(RouterError::Forwarding)?
6192                    {
6193                        let completion = self
6194                            .forwarding
6195                            .complete_module_control_rpc(
6196                                connection_id,
6197                                frame.header.corr,
6198                                None,
6199                                ModuleControlRpcOutcome::MalformedResponse(format!(
6200                                    "malformed {expected_op} response body: {err}"
6201                                )),
6202                            )
6203                            .map_err(RouterError::Forwarding)?;
6204                        if !self.observe_module_control_completion(completion) {
6205                            debug!(
6206                                connection_id = connection_id.get(),
6207                                corr = frame.header.corr,
6208                                "dropping late malformed module-control RPC response"
6209                            );
6210                        }
6211                        return Ok(Vec::new());
6212                    }
6213                    let message = format!("malformed route.bind response body: {err}");
6214                    secondary_error = Some(control_error_frame(
6215                        &frame,
6216                        "invalid_control_body",
6217                        message.clone(),
6218                    )?);
6219                    RouteBindRelayOutcome::ModuleGone(message)
6220                }
6221            },
6222            FrameType::Error => {
6223                if self
6224                    .forwarding
6225                    .pending_module_control_op(connection_id, frame.header.corr)
6226                    .map_err(RouterError::Forwarding)?
6227                    .is_some()
6228                {
6229                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6230                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6231                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6232                            "malformed module-control ERROR body: {err}"
6233                        )),
6234                    };
6235                    let completion = self
6236                        .forwarding
6237                        .complete_module_control_rpc(
6238                            connection_id,
6239                            frame.header.corr,
6240                            None,
6241                            outcome,
6242                        )
6243                        .map_err(RouterError::Forwarding)?;
6244                    if !self.observe_module_control_completion(completion) {
6245                        debug!(
6246                            connection_id = connection_id.get(),
6247                            corr = frame.header.corr,
6248                            "dropping late or unknown module-control RPC error"
6249                        );
6250                    }
6251                    return Ok(Vec::new());
6252                }
6253                match serde_json::from_slice::<ErrorBody>(&frame.body) {
6254                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
6255                    Err(err) => {
6256                        let message = format!("malformed route.bind ERROR body: {err}");
6257                        secondary_error = Some(control_error_frame(
6258                            &frame,
6259                            "invalid_control_body",
6260                            message.clone(),
6261                        )?);
6262                        RouteBindRelayOutcome::ModuleGone(message)
6263                    }
6264                }
6265            }
6266            ty => {
6267                return Ok(vec![control_error_frame(
6268                    &frame,
6269                    "unsupported_control_frame",
6270                    format!("unsupported module channel-0 frame {ty:?}"),
6271                )?])
6272            }
6273        };
6274
6275        let settled =
6276            self.forwarding
6277                .complete_pending_relay(connection_id, frame.header.corr, outcome);
6278        let completion = match settled {
6279            Ok(completion) => completion,
6280            Err(err) => {
6281                self.refuse_to_end_module_connection_for_a_client(
6282                    connection_id,
6283                    frame.header.corr,
6284                    err,
6285                )?;
6286                return Ok(secondary_error.into_iter().collect());
6287            }
6288        };
6289        if let Some(target) = completion.abandoned.as_ref() {
6290            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6291        }
6292        if !completion.settled {
6293            debug!(
6294                connection_id = connection_id.get(),
6295                corr = frame.header.corr,
6296                frame_type = ?frame.header.ty,
6297                "dropping late or unknown route.bind relay response"
6298            );
6299        }
6300        Ok(secondary_error.into_iter().collect())
6301    }
6302
6303    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6304        debug!(connection_id = connection_id.get(), "handling GOODBYE");
6305        // GOODBYE ends the connection's logical session even when its socket
6306        // stays open. Use disconnect teardown so verdicts, client notices and
6307        // scope authority are released at the same lifecycle boundary.
6308        self.cleanup_connection_with_end_reason(
6309            connection_id,
6310            RegistrationEndReason::ExplicitGoodbye,
6311        )
6312        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6313        Ok(Vec::new())
6314    }
6315}
6316
6317impl Default for ControlHandler {
6318    fn default() -> Self {
6319        Self::new(Arc::new(Registry::default()))
6320    }
6321}
6322
6323impl crate::supervise::SwapPromotionObserver for ControlHandler {
6324    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6325        self.apply_registration_capabilities(registration);
6326    }
6327}
6328
6329fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6330    CapabilityRequirementStatus {
6331        consumer: status.consumer,
6332        capability: status.capability,
6333        need: match status.need {
6334            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6335            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6336        },
6337        verdict: status.verdict.as_str().to_string(),
6338        episode_seq: status.episode_seq,
6339        config_satisfiable: status.config_satisfiable,
6340        runtime_available: status.runtime_available,
6341        detail: status.detail,
6342    }
6343}
6344
6345fn append_capability_problem_detail(
6346    detail: Option<String>,
6347    capability_detail: Option<String>,
6348) -> Option<String> {
6349    match (detail, capability_detail) {
6350        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6351        (Some(detail), None) => Some(detail),
6352        (None, Some(capability_detail)) => Some(capability_detail),
6353        (None, None) => None,
6354    }
6355}
6356
6357fn subc_ops() -> Vec<String> {
6358    SUBC_CONTROL_OPS
6359        .iter()
6360        .map(|op| (*op).to_string())
6361        .collect()
6362}
6363
6364fn module_subc_ops() -> Vec<String> {
6365    SUBC_CONTROL_OPS
6366        .iter()
6367        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6368        .map(|op| (*op).to_string())
6369        .collect()
6370}
6371
6372#[cfg(test)]
6373fn module_baseline_control_ops() -> Vec<String> {
6374    MODULE_BASELINE_CONTROL_OPS
6375        .iter()
6376        .map(|op| (*op).to_string())
6377        .collect()
6378}
6379
6380fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6381    let mut seen = HashSet::new();
6382    let mut effective = Vec::new();
6383    for op in MODULE_BASELINE_CONTROL_OPS {
6384        if seen.insert((*op).to_string()) {
6385            effective.push((*op).to_string());
6386        }
6387    }
6388    for op in declared.unwrap_or_default() {
6389        if seen.insert(op.clone()) {
6390            effective.push(op);
6391        }
6392    }
6393    effective
6394}
6395
6396fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6397    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6398}
6399
6400fn target_module_id(target: &RouteTarget) -> &str {
6401    match target {
6402        RouteTarget::ToolProvider { module_id }
6403        | RouteTarget::ManagementSurface { module_id }
6404        | RouteTarget::InternalService { module_id, .. } => module_id,
6405    }
6406}
6407
6408fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6409    roles.iter().any(|role| match (target, role) {
6410        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6411        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6412        (
6413            RouteTarget::InternalService { service_id, .. },
6414            ProviderRole::InternalService {
6415                service_id: provided,
6416                ..
6417            },
6418        ) => service_id == provided,
6419        _ => false,
6420    })
6421}
6422
6423fn is_routable_role(role: &ProviderRole) -> bool {
6424    matches!(
6425        role,
6426        ProviderRole::ToolProvider { .. }
6427            | ProviderRole::ManagementSurface { .. }
6428            | ProviderRole::InternalService { .. }
6429    )
6430}
6431
6432#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6433enum ControlRequestBodyError {
6434    UnknownOp,
6435    InvalidBody,
6436}
6437
6438#[derive(Debug, Deserialize)]
6439struct ControlOpProbe {
6440    op: String,
6441}
6442
6443/// Channel-0 push ops this subc version understands. A push whose `op` is not in
6444/// this set is treated as a forward-compat unknown and ignored rather than errored.
6445const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6446
6447fn is_known_module_push_op(body: &[u8]) -> bool {
6448    serde_json::from_slice::<ControlOpProbe>(body)
6449        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6450        .unwrap_or(false)
6451}
6452
6453fn is_known_module_request_op(body: &[u8]) -> bool {
6454    serde_json::from_slice::<ControlOpProbe>(body)
6455        .map(|probe| is_module_to_subc_op(&probe.op))
6456        .unwrap_or(false)
6457}
6458
6459fn is_module_to_subc_op(op: &str) -> bool {
6460    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6461}
6462
6463fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6464    debug!(
6465        op = %op,
6466        connection_id = connection_id.get(),
6467        corr,
6468        "control dispatch"
6469    );
6470}
6471
6472fn log_slow_control_dispatch(
6473    dispatch_started_at: Option<StdInstant>,
6474    op: &'static str,
6475    connection_id: ConnectionId,
6476    corr: u64,
6477) {
6478    let Some(dispatch_started_at) = dispatch_started_at else {
6479        return;
6480    };
6481    let elapsed = dispatch_started_at.elapsed();
6482    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6483        warn!(
6484            op = %op,
6485            connection_id = connection_id.get(),
6486            corr,
6487            elapsed_ms = elapsed.as_millis() as u64,
6488            "slow control dispatch"
6489        );
6490    }
6491}
6492
6493fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6494    match request {
6495        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6496        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6497        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6498        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6499        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6500        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6501        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6502        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6503        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6504        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6505        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6506        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6507        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6508        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6509        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6510        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6511        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6512        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6513        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6514    }
6515}
6516
6517fn principal_label(principal: &Principal) -> String {
6518    match principal {
6519        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6520        Principal::Direct => "direct".to_string(),
6521        other => format!("{other:?}"),
6522    }
6523}
6524
6525fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6526    match request {
6527        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6528        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6529        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6530        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6531        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6532    }
6533}
6534
6535fn parse_client_control_request(
6536    body: &[u8],
6537) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6538    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6539        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6540            Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6541                ControlRequestBodyError::InvalidBody
6542            }
6543            Ok(_) => ControlRequestBodyError::UnknownOp,
6544            Err(_) => ControlRequestBodyError::InvalidBody,
6545        };
6546        (err, classification)
6547    })
6548}
6549
6550fn parse_module_control_request_from_module(
6551    body: &[u8],
6552) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6553    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6554        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6555            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6556            Ok(_) => ControlRequestBodyError::UnknownOp,
6557            Err(_) => ControlRequestBodyError::InvalidBody,
6558        };
6559        (err, classification)
6560    })
6561}
6562
6563#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6564enum ProviderRoleKind {
6565    ToolProvider,
6566    PipelineStage,
6567    ManagementSurface,
6568    InternalService,
6569}
6570
6571fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6572    match role {
6573        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6574        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6575        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6576        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6577    }
6578}
6579
6580fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6581    roles.iter().map(provider_role_kind).collect()
6582}
6583
6584/// Most refused scope records named individually in the log per sync; the
6585/// `refused` count on the accepted line is always complete.
6586const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6587
6588/// Per-outcome counts of one accepted `scope.sync`, for its log line.
6589#[derive(Debug, Default, PartialEq, Eq)]
6590struct ScopeOutcomeCounts {
6591    created: usize,
6592    replaced: usize,
6593    updated: usize,
6594    unchanged: usize,
6595    refused: usize,
6596}
6597
6598impl ScopeOutcomeCounts {
6599    fn of(results: &[ScopeRecordResult]) -> Self {
6600        let mut counts = Self::default();
6601        for result in results {
6602            let slot = match result.outcome {
6603                ScopeRecordOutcome::Created => &mut counts.created,
6604                ScopeRecordOutcome::Replaced => &mut counts.replaced,
6605                ScopeRecordOutcome::Updated => &mut counts.updated,
6606                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6607                ScopeRecordOutcome::Refused => &mut counts.refused,
6608            };
6609            *slot += 1;
6610        }
6611        counts
6612    }
6613}
6614
6615#[cfg(test)]
6616mod scope_outcome_count_tests {
6617    use super::*;
6618
6619    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6620        ScopeRecordResult {
6621            scope_ref: "r".to_string(),
6622            scope_epoch: 1,
6623            outcome,
6624            code: None,
6625            message: None,
6626            version: None,
6627            parent_state: None,
6628        }
6629    }
6630
6631    /// Each outcome lands in its own count, so a refused record can never be
6632    /// hidden inside the total the log already printed.
6633    #[test]
6634    fn every_outcome_is_counted_in_its_own_field() {
6635        let results = [
6636            result(ScopeRecordOutcome::Created),
6637            result(ScopeRecordOutcome::Created),
6638            result(ScopeRecordOutcome::Replaced),
6639            result(ScopeRecordOutcome::Updated),
6640            result(ScopeRecordOutcome::Unchanged),
6641            result(ScopeRecordOutcome::Refused),
6642            result(ScopeRecordOutcome::Refused),
6643            result(ScopeRecordOutcome::Refused),
6644        ];
6645        assert_eq!(
6646            ScopeOutcomeCounts::of(&results),
6647            ScopeOutcomeCounts {
6648                created: 2,
6649                replaced: 1,
6650                updated: 1,
6651                unchanged: 1,
6652                refused: 3,
6653            }
6654        );
6655    }
6656}
6657
6658/// Return whether a catalog change can create a newly violating live route.
6659/// Removing an attested claim is intentionally excluded: it makes fewer routes
6660/// forbidden and therefore must leave the existing route census untouched.
6661fn capability_census_trigger(
6662    old: Option<&CapabilityDeclarations>,
6663    new: Option<&CapabilityDeclarations>,
6664) -> bool {
6665    let old_provides = old
6666        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6667        .unwrap_or_default();
6668    let old_denies = old
6669        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6670        .unwrap_or_default();
6671    let new = new.cloned().unwrap_or(CapabilityDeclarations {
6672        provides: Vec::new(),
6673        requires: Vec::new(),
6674        must_never_reach: Vec::new(),
6675    });
6676
6677    new.provides
6678        .iter()
6679        .any(|capability| !old_provides.contains(capability))
6680        || new
6681            .must_never_reach
6682            .iter()
6683            .any(|capability| !old_denies.contains(capability))
6684}
6685
6686/// Find the first capability an attested opener denies that an attested target
6687/// claims. Both manifests are live registry records, never cached or client data.
6688fn denied_capability<'a>(
6689    opening_manifest: &'a ModuleManifest,
6690    target_manifest: &ModuleManifest,
6691) -> Option<&'a str> {
6692    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6693    let target_capabilities = target_manifest.capabilities.as_ref()?;
6694    opening_capabilities
6695        .must_never_reach
6696        .iter()
6697        .find(|denied| {
6698            target_capabilities
6699                .provides
6700                .iter()
6701                .any(|provided| provided == *denied)
6702        })
6703        .map(String::as_str)
6704}
6705
6706fn catalog_update_frozen_field_message(
6707    registered: &ModuleManifest,
6708    provides: &[ProviderRole],
6709) -> Option<String> {
6710    let old_has_provides = !registered.provides.is_empty();
6711    let new_has_provides = !provides.is_empty();
6712    if old_has_provides != new_has_provides {
6713        return Some(format!(
6714            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6715            registered.module_id
6716        ));
6717    }
6718
6719    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
6720        return Some(format!(
6721            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6722            registered.module_id
6723        ));
6724    }
6725
6726    let registered_concurrency = manifest_concurrency(registered);
6727    let mut candidate = registered.clone();
6728    candidate.provides = provides.to_vec();
6729    let candidate_concurrency = manifest_concurrency(&candidate);
6730    if candidate_concurrency != registered_concurrency {
6731        return Some(format!(
6732            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
6733            registered.module_id, registered_concurrency, candidate_concurrency
6734        ));
6735    }
6736
6737    // control_ops live beside the manifest in the HELLO body, not inside
6738    // ModuleManifest, so a provides-only catalog.update cannot change them.
6739    None
6740}
6741
6742fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
6743    manifest.provides.iter().any(is_routable_role)
6744}
6745
6746/// Returns the routable-provider concurrency subc should enforce for this manifest.
6747///
6748/// ToolProvider and ManagementSurface store their delivery concurrency directly.
6749/// InternalService has no role-specific concurrency field, so it retains the
6750/// existing ModuleManaged default for backward compatibility.
6751fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
6752    manifest
6753        .provides
6754        .iter()
6755        .find_map(|provider| match provider {
6756            ProviderRole::ToolProvider { concurrency, .. }
6757            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
6758            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
6759        })
6760        .unwrap_or(Concurrency::ModuleManaged)
6761}
6762
6763/// True when the manifest carries a ManagementSurface role whose concurrency
6764/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
6765/// bytes because the typed manifest deliberately erases that distinction: the
6766/// default exists for wire compatibility, and this probe exists so the default
6767/// stays observable. Any parse irregularity returns false -- the caller only
6768/// logs, and a malformed body already failed registration upstream.
6769fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
6770    let has_management_surface = manifest
6771        .provides
6772        .iter()
6773        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
6774    if !has_management_surface {
6775        return false;
6776    }
6777    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
6778        return false;
6779    };
6780    let Some(provides) = raw
6781        .get("manifest")
6782        .and_then(|manifest| manifest.get("provides"))
6783        .and_then(serde_json::Value::as_array)
6784    else {
6785        return false;
6786    };
6787    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
6788    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
6789    // against the management_surface_manifest_without_concurrency golden, not
6790    // recalled (the externally-tagged guess was this function's first bug).
6791    provides.iter().any(|role| {
6792        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
6793            && role.get("concurrency").is_none()
6794    })
6795}
6796
6797fn negotiate_version(peer_version: u8) -> Result<u8, String> {
6798    if peer_version != PROTOCOL_VERSION {
6799        return Err(format!(
6800            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
6801        ));
6802    }
6803    Ok(PROTOCOL_VERSION)
6804}
6805
6806fn pong(frame: &Frame) -> Result<Frame, RouterError> {
6807    Frame::build_with_version(
6808        response_version(frame),
6809        FrameType::Pong,
6810        frame.header.flags,
6811        0,
6812        0,
6813        frame.header.corr,
6814        Vec::new(),
6815    )
6816    .map_err(RouterError::FrameBuild)
6817}
6818
6819pub(crate) fn control_error_frame(
6820    frame: &Frame,
6821    code: &'static str,
6822    message: impl Into<String>,
6823) -> Result<Frame, RouterError> {
6824    control_error_body_frame(
6825        frame,
6826        ErrorBody {
6827            code: code.to_string(),
6828            message: message.into(),
6829            detail: None,
6830        },
6831    )
6832}
6833
6834fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6835    let body = serde_json::to_vec(&error).map_err(|err| {
6836        RouterError::backend(
6837            0,
6838            frame.header.corr,
6839            format!("failed to encode control ERROR: {err}"),
6840        )
6841    })?;
6842
6843    Frame::build_with_version(
6844        response_version(frame),
6845        FrameType::Error,
6846        control_flags(),
6847        0,
6848        0,
6849        frame.header.corr,
6850        body,
6851    )
6852    .map_err(RouterError::FrameBuild)
6853}
6854
6855fn control_response_body_frame<T: Serialize>(
6856    frame: &Frame,
6857    reply: &T,
6858    label: &'static str,
6859) -> Result<Frame, RouterError> {
6860    let body = serde_json::to_vec(reply).map_err(|err| {
6861        RouterError::backend(
6862            0,
6863            frame.header.corr,
6864            format!("failed to encode {label}: {err}"),
6865        )
6866    })?;
6867
6868    Frame::build_with_version(
6869        response_version(frame),
6870        FrameType::Response,
6871        control_flags(),
6872        0,
6873        0,
6874        frame.header.corr,
6875        body,
6876    )
6877    .map_err(RouterError::FrameBuild)
6878}
6879
6880/// Map a forwarding failure to the wire code a client sees.
6881///
6882/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
6883/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
6884/// chosen here decides whether a caller retries or gives up.
6885///
6886/// That makes attribution the load-bearing property, not merely having a code. A
6887/// permanent fault published as a retryable one produces a fleet-wide retry storm
6888/// against something that can never recover; a transient fault published as
6889/// permanent gives up on work that would have succeeded. Both look correct in a
6890/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
6891/// the mapping per variant rather than merely asserting that some code exists.
6892///
6893/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
6894/// two codes on the same side of the boundary passes it. Measured rather than
6895/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
6896/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
6897/// a test named for something else that happens to assert the string.
6898///
6899/// That accidental coverage is deliberately left alone rather than promoted to a
6900/// named test, because it guards a property this function does not promise.
6901/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
6902/// specific code within a class, so identity is free to change and only the
6903/// partition is a contract. Splitting it out would assert a guarantee nothing
6904/// depends on — and a suite that promises more than the code does is the harder
6905/// thing to correct later, because the next reader cannot tell which assertions
6906/// are load-bearing.
6907///
6908/// Pin identity here the moment a consumer branches on a specific code.
6909fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6910    match err {
6911        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6912        ForwardingError::NoModuleConnection => "target_unavailable",
6913        ForwardingError::ModuleReloading { .. } => "module_reloading",
6914        ForwardingError::ClientRouteChannelExhausted { .. }
6915        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6916        ForwardingError::StaleModuleEndpoint
6917        | ForwardingError::UnknownReservation { .. }
6918        | ForwardingError::ConnectionClosing { .. }
6919        | ForwardingError::ClientEgressClosed { .. }
6920        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6921        // Only a swap candidate's registration can produce this, and it means
6922        // exactly what a second active HELLO for a live id means.
6923        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6924        ForwardingError::RelayCorrelationExhausted
6925        | ForwardingError::RouteOpenBuild(_)
6926        | ForwardingError::Poisoned => "forwarding_error",
6927    }
6928}
6929
6930fn response_version(frame: &Frame) -> u8 {
6931    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6932        frame.header.ver
6933    } else {
6934        PROTOCOL_VERSION
6935    }
6936}
6937
6938fn control_flags() -> Flags {
6939    Flags::new(false, Priority::Passive, false)
6940}
6941
6942/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
6943/// channel. The target is the module (a client never saw the route), so this
6944/// takes the module path: delivered late rather than dropped when the module's
6945/// queue is momentarily full, and never closing its connection.
6946fn send_goodbye_target_best_effort(
6947    counters: &DaemonCounters,
6948    target: &GoodbyeTarget,
6949    context: &'static str,
6950) {
6951    let Ok(frame) = Frame::build_with_version(
6952        target.negotiated_ver,
6953        FrameType::Goodbye,
6954        control_flags(),
6955        target.channel,
6956        target.epoch,
6957        0,
6958        Vec::new(),
6959    ) else {
6960        return;
6961    };
6962    crate::forwarding::send_module_route_goodbye(
6963        counters,
6964        &target.sink,
6965        frame,
6966        target.module_id.as_deref(),
6967        context,
6968    );
6969}
6970
6971pub(crate) fn send_route_control_pushes(
6972    forwarding: &ForwardingTable,
6973    routes: Vec<EndpointRoute>,
6974    push: ClientControlPush,
6975) {
6976    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6977    for route in routes {
6978        let target = route.goodbye_target;
6979        if let Some((existing, channels)) = targets
6980            .iter_mut()
6981            .find(|(existing, _)| existing.connection_id == target.connection_id)
6982        {
6983            debug_assert_eq!(
6984                existing.negotiated_ver, target.negotiated_ver,
6985                "one connection cannot negotiate multiple frame versions"
6986            );
6987            if !channels.contains(&target.channel) {
6988                channels.push(target.channel);
6989            }
6990            continue;
6991        }
6992        let channel = target.channel;
6993        targets.push((target, vec![channel]));
6994    }
6995    for (target, mut channels) in targets {
6996        channels.sort_unstable();
6997        let mut push = push.clone();
6998        match &mut push {
6999            ClientControlPush::RouteClosing {
7000                channels: covered, ..
7001            }
7002            | ClientControlPush::RouteClosed {
7003                channels: covered, ..
7004            } => *covered = channels,
7005        }
7006        let body = match serde_json::to_vec(&push) {
7007            Ok(body) => body,
7008            Err(err) => {
7009                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7010                continue;
7011            }
7012        };
7013        let frame = match Frame::build_with_version(
7014            target.negotiated_ver,
7015            FrameType::Push,
7016            control_flags(),
7017            0,
7018            0,
7019            0,
7020            body.clone(),
7021        ) {
7022            Ok(frame) => frame,
7023            Err(err) => {
7024                warn!(
7025                    route_channel = target.channel,
7026                    error = %err,
7027                    "failed to build route lifecycle control PUSH frame"
7028                );
7029                continue;
7030            }
7031        };
7032        if let Err(err) = target.sink.try_send(frame) {
7033            if target.close_on_delivery_failure() {
7034                warn!(
7035                    target_connection_id = target.connection_id.get(),
7036                    route_channel = target.channel,
7037                    error = %err,
7038                    "route lifecycle control PUSH was not delivered to client; closing target connection"
7039                );
7040                let _ = forwarding.escalate_client_delivery_failure(
7041                    target.connection_id,
7042                    target.channel,
7043                    target.epoch,
7044                    CloseReason::new(
7045                        "route_lifecycle_push_delivery_failed",
7046                        format!(
7047                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7048                            target.channel
7049                        ),
7050                    ),
7051                    crate::forwarding::UndeliveredFrame {
7052                        module_id: target.module_id.as_deref(),
7053                        sink: &target.sink,
7054                    },
7055                );
7056            }
7057        }
7058    }
7059}
7060
7061#[cfg(test)]
7062mod tests {
7063    mod activity_read {
7064        include!("activity_read_tests.rs");
7065    }
7066
7067    mod lifecycle_activity {
7068        include!("control_lifecycle_tests.rs");
7069    }
7070    #[cfg(unix)]
7071    #[tokio::test]
7072    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7073        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7074        let path = dir.join("subc.jsonc");
7075        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7076            "program":"/bin/sleep","args":["60"],"protocol":"none",
7077            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7078        }}}).to_string()).unwrap();
7079        let mut configured = crate::daemon_config::load(&path)
7080            .unwrap()
7081            .unwrap()
7082            .modules
7083            .pop()
7084            .unwrap();
7085        let registry = std::sync::Arc::new(crate::Registry::default());
7086        let handle = crate::SupervisorHandle::new();
7087        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7088            .with_handle(handle.clone());
7089        let module = supervisor
7090            .supervise_configured_with_health(
7091                configured.module_spec(),
7092                true,
7093                configured.health.clone(),
7094                None,
7095                configured.restart,
7096            )
7097            .unwrap();
7098        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7099        let before = module.status().unwrap().pid;
7100        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7101        configured.health.cadence = std::time::Duration::from_secs(3600);
7102        let health_only = handler
7103            .reconcile_supervised_modules(
7104                &supervisor,
7105                vec![configured.clone()],
7106                false,
7107                Principal::Direct,
7108            )
7109            .await
7110            .unwrap();
7111        assert!(
7112            health_only.changed_pending_reload.is_empty(),
7113            "health policy is already applied live"
7114        );
7115        assert_eq!(module.status().unwrap().pid, before);
7116        assert_eq!(
7117            module.configuration().unwrap().1.http,
7118            configured.health.http
7119        );
7120        configured.args = vec!["61".into()];
7121        let launch = handler
7122            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7123            .await
7124            .unwrap();
7125        assert_eq!(launch.changed_pending_reload, ["stock"]);
7126        assert_eq!(
7127            module.status().unwrap().pid,
7128            before,
7129            "a launch edit is stored until reload"
7130        );
7131        module.drain().await.unwrap();
7132    }
7133    use cortexkit_test_support::ScratchDir;
7134    use std::{
7135        collections::BTreeMap,
7136        fmt,
7137        path::PathBuf,
7138        sync::{Arc, Mutex},
7139        time::Duration,
7140    };
7141
7142    use serde_json::{json, Value};
7143    use subc_protocol::{
7144        manifest::{
7145            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7146            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7147        },
7148        session::HealthStatus,
7149        FrameType,
7150    };
7151
7152    use super::*;
7153    use crate::{
7154        forwarding::{DataRoute, DataRouteState},
7155        registry::ChannelState,
7156        router::FrameSink,
7157        stderr_tail::DEFAULT_MAX_LINE_BYTES,
7158        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7159        RouteCtx, Router,
7160    };
7161    use tokio::{
7162        sync::mpsc,
7163        time::{sleep, Instant},
7164    };
7165    use tracing::{
7166        field::{Field, Visit},
7167        Event, Subscriber,
7168    };
7169    use tracing_subscriber::{layer::Context, prelude::*, Layer};
7170
7171    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
7172    ///
7173    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
7174    /// is only populated for `tests/*.rs` integration test binaries -- this file
7175    /// compiles as part of the library target, which gets neither. This test's
7176    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
7177    /// and the sibling binary lives two directories up at
7178    /// `<target-dir>/<profile>/fake-aft-stub`.
7179    ///
7180    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
7181    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
7182    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
7183    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
7184    /// `NotFound`, which reads as a broken test rather than an unbuilt
7185    /// dependency -- so state the cause and the remedy instead. Deliberately a
7186    /// panic and not a silent skip: a test that quietly passes when it could not
7187    /// run is worse than one that fails, because it reports health it never
7188    /// verified.
7189    fn fake_aft_stub_path() -> PathBuf {
7190        let mut path = std::env::current_exe().expect("current_exe available in tests");
7191        path.pop(); // .../deps/
7192        path.pop(); // .../<profile>/
7193        path.push(if cfg!(windows) {
7194            "fake-aft-stub.exe"
7195        } else {
7196            "fake-aft-stub"
7197        });
7198        assert!(
7199            path.exists(),
7200            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7201             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7202            path.display()
7203        );
7204        path
7205    }
7206
7207    /// Whether clients retry `code` in place: the predicate itself, never a copy
7208    /// of its set. A copied list breaks silently when a code is added to or
7209    /// removed from the real one, and a stale copy here would let exactly the
7210    /// failure this test exists to catch pass.
7211    fn client_retries(code: &str) -> bool {
7212        subc_protocol::error_codes::is_retryable_route_open(code)
7213    }
7214
7215    /// A code is not a label — clients branch on it, so publishing the wrong KIND
7216    /// of failure is worse than publishing none. A permanent fault dressed as
7217    /// retryable makes every client in the fleet retry forever against something
7218    /// that cannot recover; a transient fault dressed as permanent abandons work
7219    /// that would have succeeded.
7220    ///
7221    /// Asserting "a code exists" cannot catch either, because the string is free
7222    /// to say anything. This enumerates every variant and pins which side of the
7223    /// retry boundary it lands on, so a new variant must be classified here
7224    /// deliberately rather than inheriting whichever arm it was appended to.
7225    #[test]
7226    fn retryability_of_forwarding_codes_matches_the_failure() {
7227        // Transient by nature: the target is booting, reloading, or its endpoint
7228        // was swapped mid-flight. Retrying is how these resolve.
7229        let transient = [
7230            ForwardingError::NoModuleConnection,
7231            ForwardingError::ModuleReloading {
7232                module_id: "m".into(),
7233            },
7234            ForwardingError::StaleModuleEndpoint,
7235            ForwardingError::UnknownReservation {
7236                client_channel: 1,
7237                module_channel: 1,
7238            },
7239            ForwardingError::ConnectionClosing {
7240                connection_id: ConnectionId::new(1),
7241            },
7242            ForwardingError::ClientEgressClosed {
7243                connection_id: ConnectionId::new(1),
7244            },
7245            ForwardingError::ModuleEgressUnavailable {
7246                connection_id: ConnectionId::new(1),
7247            },
7248        ];
7249        for err in transient {
7250            let code = forwarding_error_code(&err);
7251            assert!(
7252                client_retries(code),
7253                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7254            );
7255        }
7256
7257        // Not fixed by retrying. Channel and correlation exhaustion need the
7258        // caller to close routes, and a poisoned lock is a daemon that cannot
7259        // recover at all — the worst thing to advertise as retryable, since every
7260        // client would storm a daemon that will never answer.
7261        let permanent = [
7262            ForwardingError::ConnectionRoleConflict {
7263                connection_id: ConnectionId::new(1),
7264            },
7265            ForwardingError::ClientRouteChannelExhausted {
7266                connection_id: ConnectionId::new(1),
7267            },
7268            ForwardingError::ModuleRouteChannelExhausted {
7269                endpoint: ModuleEndpointId {
7270                    connection_id: ConnectionId::new(1),
7271                    generation: 1,
7272                },
7273            },
7274            ForwardingError::RelayCorrelationExhausted,
7275            ForwardingError::RouteOpenBuild("x".into()),
7276            ForwardingError::Poisoned,
7277        ];
7278        for err in permanent {
7279            let code = forwarding_error_code(&err);
7280            assert!(
7281                !client_retries(code),
7282                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7283            );
7284        }
7285    }
7286
7287    /// The principal is the daemon's answer to "who is calling", and modules
7288    /// branch on it: aft gates bash on it, cerebellum gates browser control,
7289    /// plexus gates connector invocation. So a stamp is an authorization input in
7290    /// another process, not a label — and both possible answers SUCCEED, which is
7291    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
7292    /// first-party capability to something that never proved it; a supervised one
7293    /// stamped `Direct` silently strips a module of capability it is entitled to.
7294    ///
7295    /// Neither shows up in a test that only checks the bind succeeded. Before this
7296    /// test the only coverage was accidental —
7297    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
7298    /// stamped principal on its way past, so narrowing that wire-shape test to its
7299    /// stated subject would have deleted the last assertion on this value. It
7300    /// still asserts the stamp, which is now redundancy rather than the only
7301    /// guard: both fail under the same mutation, and this one names the reason.
7302    /// SCOPE: this handler's supervisor has spawned nothing, so
7303    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
7304    /// is unreachable here. Both assertions below are refusals, and a mutant that
7305    /// refuses everything would satisfy them.
7306    ///
7307    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
7308    /// spawns a supervised consumer, reads its live nonce, and asserts the module
7309    /// observed `principal.kind == "reserved"` carrying that module_id — verified
7310    /// at source rather than assumed, since a citation is a claim about another
7311    /// file and ages like one. Recorded because a harness that structurally
7312    /// cannot reach an arm reports "none" for that arm identically to one that
7313    /// covers it and found nothing.
7314    #[tokio::test]
7315    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7316        let handler = ControlHandler::default();
7317        let frame =
7318            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7319
7320        // Absent consumer_identity is the ordinary case: a human at a terminal, or
7321        // any process holding the connection file. Nothing was proved, so nothing
7322        // may be granted beyond the unattested floor.
7323        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7324        assert_eq!(
7325            stamped,
7326            Principal::Direct,
7327            "a caller that proved nothing must not be stamped as a supervised module"
7328        );
7329
7330        // A claimed module_id with a nonce no supervised child was given is a
7331        // forgery attempt, not a weaker caller: it must be REFUSED rather than
7332        // quietly demoted to Direct, or an impersonation attempt looks identical
7333        // to an ordinary unattested connection.
7334        let forged = handler
7335            .route_open_principal(
7336                &frame,
7337                Some(ConsumerIdentity {
7338                    module_id: "aft".to_string(),
7339                    launch_nonce: "not-a-real-nonce".to_string(),
7340                }),
7341            )
7342            .unwrap();
7343        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7344        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7345    }
7346
7347    /// The test above hands `route_open_principal` an identity it built itself,
7348    /// which proves the stamping rule and nothing about where the identity comes
7349    /// from. The real producer is a wire body, and the two are joined by a serde
7350    /// field name that nothing else asserts.
7351    ///
7352    /// That join fails quietly in one specific way: an unrecognised key is simply
7353    /// absent after parsing, so a renamed or misspelled `consumer_identity`
7354    /// yields `None` and every supervised module silently drops to `Direct`.
7355    /// Capability-wise that is the safe direction, but it surfaces far from its
7356    /// cause — as a module mysteriously refused bash — and it would pass every
7357    /// test that builds its own input.
7358    ///
7359    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
7360    /// would break every client the moment the daemon gains a field, trading a
7361    /// quiet demotion for a hard refusal on additive change. Asserting the join
7362    /// instead means a rename breaks a test here rather than the fleet.
7363    #[test]
7364    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7365        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7366        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7367        let ClientControlRequest::RouteOpen {
7368            consumer_identity, ..
7369        } = parsed
7370        else {
7371            panic!("route.open body must parse as RouteOpen");
7372        };
7373        assert_eq!(
7374            consumer_identity,
7375            Some(ConsumerIdentity {
7376                module_id: "aft".to_string(),
7377                launch_nonce: "n".to_string(),
7378            }),
7379            "the wire field name must reach the value route_open_principal reads"
7380        );
7381    }
7382
7383    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7384        ModuleManifest::builder(module_id, "0.1.0")
7385            .protocol_ver(protocol_ver)
7386            .provides(vec![ProviderRole::ToolProvider {
7387                tools: vec![Tool {
7388                    name: "read".to_string(),
7389                    description: None,
7390                    execution_mode: ExecutionMode::Pure,
7391                    schema: json!({"type": "object"}),
7392                }],
7393                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7394                concurrency: Concurrency::ModuleManaged,
7395                emits_push: true,
7396                sub_supervises: true,
7397            }])
7398            .build()
7399    }
7400
7401    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7402        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7403    }
7404
7405    fn hello_frame_with_control_ops(
7406        module_id: &str,
7407        protocol_ver: u8,
7408        corr: u64,
7409        control_ops: Option<Vec<String>>,
7410    ) -> Frame {
7411        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7412    }
7413
7414    fn hello_frame_with_nonce(
7415        module_id: &str,
7416        protocol_ver: u8,
7417        corr: u64,
7418        launch_nonce: Option<&str>,
7419    ) -> Frame {
7420        hello_frame_full(
7421            module_id,
7422            protocol_ver,
7423            corr,
7424            None,
7425            launch_nonce.map(ToOwned::to_owned),
7426        )
7427    }
7428
7429    fn hello_frame_full(
7430        module_id: &str,
7431        protocol_ver: u8,
7432        corr: u64,
7433        control_ops: Option<Vec<String>>,
7434        launch_nonce: Option<String>,
7435    ) -> Frame {
7436        let body = serde_json::to_vec(&ModuleHelloBody {
7437            manifest: manifest(module_id, protocol_ver),
7438            protocol_ver,
7439            control_ops,
7440            launch_nonce,
7441        })
7442        .unwrap();
7443        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7444    }
7445
7446    fn non_routable_hello_frame_with_control_ops(
7447        module_id: &str,
7448        corr: u64,
7449        control_ops: Option<Vec<String>>,
7450    ) -> Frame {
7451        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7452        manifest.provides.clear();
7453        let body = serde_json::to_vec(&ModuleHelloBody {
7454            manifest,
7455            protocol_ver: PROTOCOL_VERSION,
7456            control_ops,
7457            launch_nonce: None,
7458        })
7459        .unwrap();
7460        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7461    }
7462
7463    fn capability_grammar_hello_frame(
7464        capabilities: Value,
7465        runtime_computed: Option<Value>,
7466        corr: u64,
7467    ) -> Frame {
7468        let mut body = serde_json::to_value(ModuleHelloBody {
7469            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7470            protocol_ver: PROTOCOL_VERSION,
7471            control_ops: None,
7472            launch_nonce: None,
7473        })
7474        .expect("HELLO body serializes");
7475        body["manifest"]["capabilities"] = capabilities;
7476        if let Some(runtime_computed) = runtime_computed {
7477            body["runtime_computed"] = runtime_computed;
7478        }
7479        Frame::build(
7480            FrameType::Hello,
7481            control_flags(),
7482            0,
7483            0,
7484            corr,
7485            serde_json::to_vec(&body).expect("HELLO body reserializes"),
7486        )
7487        .expect("HELLO frame builds")
7488    }
7489
7490    fn channel_request(channel: u16, corr: u64) -> Frame {
7491        Frame::build(
7492            FrameType::Request,
7493            Flags::new(true, Priority::Interactive, false),
7494            channel,
7495            0,
7496            corr,
7497            b"opaque".to_vec(),
7498        )
7499        .unwrap()
7500    }
7501
7502    fn route_ctx(
7503        connection_id: ConnectionId,
7504    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7505        let (tx, rx) = mpsc::channel(8);
7506        (
7507            RouteCtx {
7508                connection_id,
7509                egress: FrameSink::new(tx),
7510            },
7511            rx,
7512        )
7513    }
7514
7515    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7516        serde_json::from_slice(&frame.body).unwrap()
7517    }
7518
7519    /// Register a module over a connection that has a sink and return the
7520    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
7521    /// module's own sink rather than returning it as a reply, so the ack is
7522    /// taken off `rx` here and whatever the test reads next is what followed it.
7523    async fn hello_via_sink(
7524        handler: &ControlHandler,
7525        ctx: &RouteCtx,
7526        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7527        hello: Frame,
7528    ) -> Frame {
7529        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7530        assert!(
7531            replies.is_empty(),
7532            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7533        );
7534        let ack = rx
7535            .try_recv()
7536            .expect("HELLO_ACK is queued on the module sink")
7537            .frame;
7538        assert_eq!(ack.header.ty, FrameType::HelloAck);
7539        ack
7540    }
7541
7542    fn parse_error(frame: &Frame) -> Value {
7543        serde_json::from_slice(&frame.body).unwrap()
7544    }
7545
7546    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7547        serde_json::from_slice(&frame.body).unwrap()
7548    }
7549
7550    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7551        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7552            route_channel,
7553            route_epoch: 0,
7554            kind,
7555        })
7556        .unwrap();
7557        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7558    }
7559
7560    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7561        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7562            module_id: module_id.to_string(),
7563        })
7564        .unwrap();
7565        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7566    }
7567
7568    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7569        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7570    }
7571
7572    fn route_open_frame_with_consumer_capabilities(
7573        corr: u64,
7574        module_id: &str,
7575        project_root: ScratchDir,
7576        consumer_capabilities: Option<Vec<String>>,
7577    ) -> Frame {
7578        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7579            target: RouteTarget::ToolProvider {
7580                module_id: module_id.to_string(),
7581            },
7582            identity: BindIdentity::new(
7583                project_root.path().to_path_buf(),
7584                "unit".to_string(),
7585                "session".to_string(),
7586            ),
7587            consumer_identity: None,
7588            consumer_capabilities,
7589            role_versions: None,
7590            admission_facts: None,
7591            scope: None,
7592        })
7593        .unwrap();
7594        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7595    }
7596
7597    fn route_open_frame_with_role_versions(
7598        corr: u64,
7599        module_id: &str,
7600        project_root: ScratchDir,
7601        role_versions: Option<BTreeMap<String, String>>,
7602    ) -> Frame {
7603        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7604            target: RouteTarget::ToolProvider {
7605                module_id: module_id.to_string(),
7606            },
7607            identity: BindIdentity::new(
7608                project_root.path().to_path_buf(),
7609                "unit".to_string(),
7610                format!("session-{corr}"),
7611            ),
7612            consumer_identity: None,
7613            consumer_capabilities: None,
7614            role_versions,
7615            admission_facts: None,
7616            scope: None,
7617        })
7618        .unwrap();
7619        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7620    }
7621
7622    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
7623        entries
7624            .iter()
7625            .map(|(role, version)| (role.to_string(), version.to_string()))
7626            .collect()
7627    }
7628
7629    fn route_open_frame_with_admission_facts(
7630        corr: u64,
7631        module_id: &str,
7632        project_root: ScratchDir,
7633        consumer_identity: Option<subc_control::ConsumerIdentity>,
7634        facts: Option<Value>,
7635    ) -> Frame {
7636        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7637            target: RouteTarget::ToolProvider {
7638                module_id: module_id.to_string(),
7639            },
7640            identity: BindIdentity::new(
7641                project_root.path().to_path_buf(),
7642                "unit".to_string(),
7643                format!("session-{corr}"),
7644            ),
7645            consumer_identity,
7646            consumer_capabilities: None,
7647            role_versions: None,
7648            admission_facts: facts,
7649            scope: None,
7650        })
7651        .unwrap();
7652        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7653    }
7654
7655    #[derive(Clone, Default)]
7656    struct EventCapture {
7657        events: Arc<Mutex<Vec<CapturedEvent>>>,
7658    }
7659
7660    #[derive(Clone, Debug)]
7661    struct CapturedEvent {
7662        target: String,
7663        level: tracing::Level,
7664        fields: BTreeMap<String, String>,
7665    }
7666
7667    impl EventCapture {
7668        fn events(&self) -> Vec<CapturedEvent> {
7669            self.events.lock().unwrap().clone()
7670        }
7671    }
7672
7673    impl<S> Layer<S> for EventCapture
7674    where
7675        S: Subscriber,
7676    {
7677        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7678            let mut visitor = EventFieldVisitor::default();
7679            event.record(&mut visitor);
7680            self.events.lock().unwrap().push(CapturedEvent {
7681                target: event.metadata().target().to_string(),
7682                level: *event.metadata().level(),
7683                fields: visitor.fields,
7684            });
7685        }
7686    }
7687
7688    #[derive(Default)]
7689    struct EventFieldVisitor {
7690        fields: BTreeMap<String, String>,
7691    }
7692
7693    impl Visit for EventFieldVisitor {
7694        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7695            self.fields
7696                .insert(field.name().to_string(), format!("{value:?}"));
7697        }
7698    }
7699
7700    fn health_response(corr: u64, status: HealthStatus) -> Frame {
7701        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7702            status,
7703            detail: Some("warming".to_string()),
7704            metrics: Some(json!({"queue_depth": 3})),
7705        })
7706        .unwrap();
7707        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7708    }
7709
7710    fn route_bind_ack(corr: u64) -> Frame {
7711        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7712        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7713    }
7714
7715    fn unique_project_root(label: &str) -> ScratchDir {
7716        ScratchDir::new(label)
7717    }
7718
7719    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7720        match parse_route_poll(frame) {
7721            ClientControlResponse::RoutePoll {
7722                status: None,
7723                live: Some(live),
7724                ..
7725            } => assert_eq!(live, expected_live),
7726            other => panic!("unexpected route.poll response: {other:?}"),
7727        }
7728    }
7729
7730    fn bind_liveness_route(
7731        registry: &Registry,
7732        forwarding: &ForwardingTable,
7733        module_id: &str,
7734    ) -> (RouteCtx, u16, u32) {
7735        let module_connection = ConnectionId::new(101);
7736        let client_connection = ConnectionId::new(202);
7737        let registration = registry
7738            .register_with_control_ops(
7739                manifest(module_id, PROTOCOL_VERSION),
7740                PROTOCOL_VERSION,
7741                module_connection,
7742                module_baseline_control_ops(),
7743            )
7744            .unwrap();
7745        let (module_tx, _module_rx) = mpsc::channel(8);
7746        let endpoint = forwarding
7747            .register_module_connection(
7748                module_connection,
7749                module_id.to_string(),
7750                PROTOCOL_VERSION,
7751                manifest_concurrency(&registration.manifest),
7752                FrameSink::new(module_tx),
7753            )
7754            .unwrap();
7755        let (client_ctx, _client_rx) = route_ctx(client_connection);
7756        let pending = forwarding
7757            .begin_route_bind_relay_for_test(
7758                client_connection,
7759                client_ctx.egress.clone(),
7760                1,
7761                module_id,
7762            )
7763            .unwrap();
7764        assert_eq!(pending.endpoint, endpoint);
7765        let route_channel = pending.client_channel;
7766        let route_epoch = pending.client_epoch;
7767        forwarding
7768            .complete_pending_relay(
7769                module_connection,
7770                pending.corr,
7771                RouteBindRelayOutcome::Accepted,
7772            )
7773            .unwrap();
7774        (client_ctx, route_channel, route_epoch)
7775    }
7776
7777    struct FakeProcessLiveness {
7778        live: Option<bool>,
7779    }
7780
7781    impl ModuleProcessLiveness for FakeProcessLiveness {
7782        fn process_live(&self, _module_id: &str) -> Option<bool> {
7783            self.live
7784        }
7785    }
7786
7787    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7788    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
7789    {
7790        let registry = Arc::new(Registry::default());
7791        let supervisor_handle = SupervisorHandle::new();
7792        let supervisor = Supervisor::new_for_test(
7793            Arc::clone(&registry),
7794            RestartPolicy::new(1, Duration::from_millis(10)),
7795        )
7796        .with_handle(supervisor_handle.clone());
7797        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
7798        let module = supervisor
7799            .spawn(ModuleSpec {
7800                module_id: "stderr-tail-wire".to_string(),
7801                program: fake_aft_stub_path(),
7802                args: Vec::new(),
7803                env: vec![
7804                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
7805                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
7806                ],
7807                reserved: false,
7808                reserved_prefixes: Vec::new(),
7809                protocol: ModuleProtocol::Subc,
7810                overlap: Default::default(),
7811            })
7812            .unwrap();
7813
7814        let deadline = Instant::now() + Duration::from_secs(5);
7815        loop {
7816            let tail = module.stderr_tail(None, None);
7817            if tail
7818                .entries
7819                .iter()
7820                .any(|entry| matches!(entry, TailEntry::ProcessStart))
7821                && tail.entries.iter().any(|entry| {
7822                    matches!(
7823                        entry,
7824                        TailEntry::Line {
7825                            truncated: true,
7826                            ..
7827                        }
7828                    )
7829                })
7830            {
7831                break;
7832            }
7833            assert!(
7834                Instant::now() < deadline,
7835                "module did not produce a truncated line and restart boundary: {tail:?}"
7836            );
7837            sleep(Duration::from_millis(10)).await;
7838        }
7839
7840        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
7841        let request = ClientControlRequest::SupervisorStderrTail {
7842            module_id: "stderr-tail-wire".to_string(),
7843            max_lines: None,
7844            max_bytes: None,
7845        };
7846        let frame = Frame::build(
7847            FrameType::Request,
7848            control_flags(),
7849            0,
7850            0,
7851            1,
7852            serde_json::to_vec(&request).unwrap(),
7853        )
7854        .unwrap();
7855        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7856        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7857        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7858            serde_json::from_slice(&responses[0].body).unwrap()
7859        else {
7860            panic!("expected supervisor.stderr_tail response");
7861        };
7862
7863        assert!(
7864            tail.entries
7865                .iter()
7866                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7867            "the control response lost the restart boundary"
7868        );
7869        let Some(StderrTailEntry::Line {
7870            text,
7871            truncated,
7872            at_ms,
7873        }) = tail.entries.iter().find(|entry| {
7874            matches!(
7875                entry,
7876                StderrTailEntry::Line {
7877                    truncated: true,
7878                    ..
7879                }
7880            )
7881        })
7882        else {
7883            panic!("the control response lost the truncated line");
7884        };
7885        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7886        assert!(*truncated);
7887        assert!(
7888            at_ms.is_some(),
7889            "the control response lost the line's capture time"
7890        );
7891    }
7892
7893    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
7894    /// read done on the worker thread would stall every other task until it
7895    /// finished; the read must run off the worker so this test's own task keeps
7896    /// running while the read is paused.
7897    #[tokio::test(flavor = "current_thread")]
7898    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7899        let dir = ScratchDir::new("terminals-off-worker");
7900        let journal_path = dir.join("terminals.jsonl");
7901        let registry = Arc::new(Registry::default());
7902        let supervisor_handle = SupervisorHandle::new();
7903        let supervisor =
7904            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
7905                .with_handle(supervisor_handle.clone())
7906                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7907        let module = supervisor
7908            .spawn(ModuleSpec {
7909                module_id: "terminal-off-worker".to_string(),
7910                program: fake_aft_stub_path(),
7911                args: Vec::new(),
7912                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7913                reserved: false,
7914                reserved_prefixes: Vec::new(),
7915                protocol: ModuleProtocol::Subc,
7916                overlap: Default::default(),
7917            })
7918            .unwrap();
7919        let deadline = Instant::now() + Duration::from_secs(5);
7920        while module.terminal_history().entries.len() != 2 {
7921            assert!(Instant::now() < deadline, "module did not record two exits");
7922            sleep(Duration::from_millis(10)).await;
7923        }
7924
7925        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7926        let handler =
7927            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
7928        let frame = Frame::build(
7929            FrameType::Request,
7930            control_flags(),
7931            0,
7932            0,
7933            1,
7934            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7935                module_id: "terminal-off-worker".to_string(),
7936            })
7937            .unwrap(),
7938        )
7939        .unwrap();
7940        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7941        let spawned_at = std::time::Instant::now();
7942        let read = tokio::spawn({
7943            let handler = Arc::clone(&handler);
7944            async move { handler.handle_control_frame(&ctx, frame).await }
7945        });
7946        // Waiting for the pause from a blocking thread keeps this task pending,
7947        // so the runtime's single worker is free to run the read task.
7948        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7949            .await
7950            .unwrap()
7951            .expect("the history read reached its pause");
7952        let elapsed = spawned_at.elapsed();
7953        assert!(
7954            elapsed < Duration::from_secs(2) && !read.is_finished(),
7955            "this task could not run while the history read was paused \
7956             (resumed after {elapsed:?}, read finished: {})",
7957            read.is_finished()
7958        );
7959
7960        drop(release);
7961        let responses = read.await.unwrap().unwrap();
7962        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7963        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7964            panic!("expected supervisor.terminals response");
7965        };
7966        assert_eq!(terminals.entries.len(), 2);
7967        assert_eq!(terminals.journal_skipped_lines, 0);
7968        assert_eq!(terminals.journal_read_errors, 0);
7969    }
7970
7971    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7972    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7973        let registry = Arc::new(Registry::default());
7974        let supervisor_handle = SupervisorHandle::new();
7975        let supervisor =
7976            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
7977                .with_handle(supervisor_handle.clone());
7978        let module = supervisor
7979            .spawn(ModuleSpec {
7980                module_id: "terminal-golden".to_string(),
7981                program: fake_aft_stub_path(),
7982                args: Vec::new(),
7983                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7984                reserved: false,
7985                reserved_prefixes: Vec::new(),
7986                protocol: ModuleProtocol::Subc,
7987                overlap: Default::default(),
7988            })
7989            .unwrap();
7990
7991        let deadline = Instant::now() + Duration::from_secs(5);
7992        while module.terminal_history().entries.len() != 2 {
7993            assert!(
7994                Instant::now() < deadline,
7995                "module did not retain two terminal exits: {:?}",
7996                module.terminal_history()
7997            );
7998            sleep(Duration::from_millis(10)).await;
7999        }
8000
8001        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8002        let request = ClientControlRequest::SupervisorTerminals {
8003            module_id: "terminal-golden".to_string(),
8004        };
8005        let frame = Frame::build(
8006            FrameType::Request,
8007            control_flags(),
8008            0,
8009            0,
8010            1,
8011            serde_json::to_vec(&request).unwrap(),
8012        )
8013        .unwrap();
8014        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8015        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8016        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8017        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8018            panic!("expected supervisor.terminals response");
8019        };
8020        assert_eq!(terminals.entries.len(), 2);
8021        assert_eq!(terminals.dropped, 0);
8022
8023        let mut rendered = serde_json::to_value(response).unwrap();
8024        // Wall-clock fields are the observation contract, but not stable fixture
8025        // bytes; normalize only them after the real handler has shaped the response.
8026        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8027        for (index, entry) in rendered["entries"]
8028            .as_array_mut()
8029            .expect("terminal response entries array")
8030            .iter_mut()
8031            .enumerate()
8032        {
8033            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8034        }
8035
8036        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8037            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8038        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8039        if std::env::var_os("UPDATE_GOLDEN").is_some() {
8040            std::fs::write(&golden_path, &serialized).unwrap();
8041        }
8042        let expected: Value =
8043            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8044        assert_eq!(rendered, expected);
8045    }
8046
8047    #[test]
8048    fn hello_registers_manifest_and_returns_ack() {
8049        let registry = Arc::new(Registry::default());
8050        let handler = ControlHandler::new(Arc::clone(&registry));
8051        let conn = ConnectionId::new(1);
8052
8053        let responses = handler
8054            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8055            .unwrap();
8056
8057        assert_eq!(responses.len(), 1);
8058        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8059        assert_eq!(responses[0].header.channel, 0);
8060        assert_eq!(responses[0].header.corr, 7);
8061        let ack = parse_ack(&responses[0]);
8062        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8063        assert!(ack
8064            .subc_capabilities
8065            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8066        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8067        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8068        assert!(ack
8069            .subc_ops
8070            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8071        assert!(ack
8072            .subc_ops
8073            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8074
8075        let registration = registry.get_module("aft").unwrap().unwrap();
8076        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8077        assert_eq!(registration.state, ChannelState::Active);
8078        assert_eq!(registration.connection_id, conn);
8079        assert_eq!(registration.control_ops, module_baseline_control_ops());
8080    }
8081
8082    #[test]
8083    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8084        let invalid_identifiers = [
8085            ("case_change", "credentials-Provider/v1"),
8086            ("leading_zero", "credentials-provider/v01"),
8087            ("trailing_hyphen", "credentials-provider-/v1"),
8088            ("consecutive_hyphens", "credentials--provider/v1"),
8089            ("uppercase", "Credentials-provider/v1"),
8090            ("missing_v", "credentials-provider/1"),
8091            ("whitespace", "credentials provider/v1"),
8092            ("zero_version", "credentials-provider/v0"),
8093            ("out_of_range_version", "credentials-provider/v4294967296"),
8094            (
8095                "overlength_name",
8096                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8097            ),
8098        ];
8099        let mut cases = invalid_identifiers
8100            .into_iter()
8101            .map(|(name, identifier)| {
8102                (
8103                    format!("identifier_{name}"),
8104                    "capabilities.provides[0]".to_string(),
8105                    identifier.to_string(),
8106                    json!({ "provides": [identifier] }),
8107                    None,
8108                )
8109            })
8110            .collect::<Vec<_>>();
8111        cases.extend([
8112            (
8113                "unknown_need".to_string(),
8114                "capabilities.requires[0].need".to_string(),
8115                "deferred".to_string(),
8116                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8117                None,
8118            ),
8119            (
8120                "duplicate_provides".to_string(),
8121                "capabilities.provides[1]".to_string(),
8122                "credentials-provider/v1".to_string(),
8123                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8124                None,
8125            ),
8126            (
8127                "duplicate_must_never_reach".to_string(),
8128                "capabilities.must_never_reach[1]".to_string(),
8129                "credentials-provider/v1".to_string(),
8130                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8131                None,
8132            ),
8133            (
8134                "duplicate_requires_same_need".to_string(),
8135                "capabilities.requires[1]".to_string(),
8136                "credentials-provider/v1".to_string(),
8137                json!({ "requires": [
8138                    { "capability": "credentials-provider/v1", "need": "required" },
8139                    { "capability": "credentials-provider/v1", "need": "required" }
8140                ] }),
8141                None,
8142            ),
8143            (
8144                "duplicate_requires_conflicting_need".to_string(),
8145                "capabilities.requires[1]".to_string(),
8146                "credentials-provider/v1".to_string(),
8147                json!({ "requires": [
8148                    { "capability": "credentials-provider/v1", "need": "required" },
8149                    { "capability": "credentials-provider/v1", "need": "optional" }
8150                ] }),
8151                None,
8152            ),
8153            (
8154                "capabilities_root_pointer".to_string(),
8155                "runtime_computed[0]".to_string(),
8156                "/capabilities".to_string(),
8157                json!({}),
8158                Some(json!(["/capabilities"])),
8159            ),
8160            (
8161                "capabilities_descendant_pointer".to_string(),
8162                "runtime_computed[0]".to_string(),
8163                "/capabilities/provides".to_string(),
8164                json!({}),
8165                Some(json!(["/capabilities/provides"])),
8166            ),
8167            (
8168                "malformed_pointer_without_leading_slash".to_string(),
8169                "runtime_computed[0]".to_string(),
8170                "capabilities".to_string(),
8171                json!({}),
8172                Some(json!(["capabilities"])),
8173            ),
8174            (
8175                "malformed_pointer_escape".to_string(),
8176                "runtime_computed[0]".to_string(),
8177                "/roles/~2/tools".to_string(),
8178                json!({}),
8179                Some(json!(["/roles/~2/tools"])),
8180            ),
8181            (
8182                "unknown_capabilities_field".to_string(),
8183                "capabilities.future".to_string(),
8184                "<array>".to_string(),
8185                json!({ "future": [] }),
8186                None,
8187            ),
8188        ]);
8189
8190        for (index, (name, field, value, capabilities, runtime_computed)) in
8191            cases.into_iter().enumerate()
8192        {
8193            let registry = Arc::new(Registry::default());
8194            let handler = ControlHandler::new(Arc::clone(&registry));
8195            let response = handler
8196                .handle_control(
8197                    ConnectionId::new((index + 1) as u64),
8198                    capability_grammar_hello_frame(
8199                        capabilities,
8200                        runtime_computed,
8201                        index as u64 + 1,
8202                    ),
8203                )
8204                .expect("invalid HELLO returns a refusal");
8205
8206            assert_eq!(response.len(), 1, "{name} must emit one refusal");
8207            let error = parse_error(&response[0]);
8208            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8209            let message = error["message"]
8210                .as_str()
8211                .expect("error message is a string");
8212            assert!(
8213                message.contains(&field),
8214                "{name}: field missing from {message}"
8215            );
8216            assert!(
8217                message.contains(&value),
8218                "{name}: value missing from {message}"
8219            );
8220            assert_eq!(
8221                registry
8222                    .active_registration_count()
8223                    .expect("registry reads"),
8224                0,
8225                "{name}: refused HELLO must not create a catalog entry"
8226            );
8227        }
8228    }
8229
8230    #[test]
8231    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8232        let registry = Arc::new(Registry::default());
8233        let handler = ControlHandler::new(Arc::clone(&registry));
8234        let capabilities = json!({
8235            "provides": ["credentials-provider/v1"],
8236            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8237            "must_never_reach": ["federation-transport/v1"]
8238        });
8239        let response = handler
8240            .handle_control(
8241                ConnectionId::new(99),
8242                capability_grammar_hello_frame(
8243                    capabilities.clone(),
8244                    Some(json!(["/roles/0/tools"])),
8245                    99,
8246                ),
8247            )
8248            .expect("valid HELLO registers");
8249        assert_eq!(response[0].header.ty, FrameType::HelloAck);
8250
8251        let request = Frame::build(
8252            FrameType::Request,
8253            control_flags(),
8254            0,
8255            0,
8256            100,
8257            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8258                .expect("catalog request serializes"),
8259        )
8260        .expect("catalog request frame builds");
8261        let response = handler
8262            .handle_catalog_list(request, None)
8263            .expect("catalog list succeeds");
8264        let ClientControlResponse::CatalogList { modules, .. } =
8265            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8266        else {
8267            panic!("catalog request must return catalog.list");
8268        };
8269        assert_eq!(modules.len(), 1);
8270        assert_eq!(
8271            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8272            capabilities
8273        );
8274    }
8275
8276    #[test]
8277    fn catalog_list_mirrors_management_operation_description() {
8278        let registry = Arc::new(Registry::default());
8279        let handler = ControlHandler::new(Arc::clone(&registry));
8280        let description = "List managed records and return their identifiers and metadata.";
8281        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8282        manifest.provides = vec![ProviderRole::ManagementSurface {
8283            operations: vec![ManagementOperation {
8284                name: "records.list".to_string(),
8285                kind: ManagementOperationKind::Query,
8286                description: Some(description.to_string()),
8287            }],
8288            config_schema: json!({"type": "object"}),
8289            observability: vec![ObservabilitySurface {
8290                name: "records.stats".to_string(),
8291                kind: ObservabilityKind::Snapshot,
8292            }],
8293            identity_scope: vec![IdentityScope::Project],
8294            concurrency: Concurrency::ModuleManaged,
8295        }];
8296        registry
8297            .register_with_control_ops(
8298                manifest,
8299                PROTOCOL_VERSION,
8300                ConnectionId::new(99),
8301                Vec::new(),
8302            )
8303            .expect("described management manifest registers");
8304
8305        let request = Frame::build(
8306            FrameType::Request,
8307            control_flags(),
8308            0,
8309            0,
8310            100,
8311            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8312                .expect("catalog request serializes"),
8313        )
8314        .expect("catalog request frame builds");
8315        let response = handler
8316            .handle_catalog_list(request, None)
8317            .expect("catalog list succeeds");
8318        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8319        assert_eq!(
8320            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8321            "catalog.list must preserve the declared operation description verbatim"
8322        );
8323    }
8324
8325    #[test]
8326    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8327        let registry = Arc::new(Registry::default());
8328        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
8329            [("vault".to_string(), true), ("squatter".to_string(), true)],
8330            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8331        );
8332        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8333        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8334            provides: vec!["credentials-provider/v1".to_string()],
8335            requires: Vec::new(),
8336            must_never_reach: Vec::new(),
8337        });
8338        let frame = Frame::build(
8339            FrameType::Hello,
8340            control_flags(),
8341            0,
8342            0,
8343            77,
8344            serde_json::to_vec(&ModuleHelloBody {
8345                manifest: squatter,
8346                protocol_ver: PROTOCOL_VERSION,
8347                control_ops: None,
8348                launch_nonce: None,
8349            })
8350            .expect("HELLO serializes"),
8351        )
8352        .expect("HELLO frame builds");
8353        let response = handler
8354            .handle_control(ConnectionId::new(77), frame)
8355            .expect("reserved claim receives a typed refusal");
8356        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8357        assert_eq!(
8358            registry
8359                .active_registration_count()
8360                .expect("registry reads"),
8361            0,
8362            "a reserved capability refusal must not leave a catalog entry"
8363        );
8364    }
8365
8366    #[test]
8367    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8368        for settlement in ["timeout", "inconclusive", "drop"] {
8369            let breakers = RouteBindBreakers::default();
8370            let RouteBindAdmission::Admitted {
8371                guard: mut old,
8372                probe: false,
8373            } = breakers.admit("prov")
8374            else {
8375                panic!("ordinary relay admitted")
8376            };
8377            let RouteBindAdmission::Admitted {
8378                guard: mut opener, ..
8379            } = breakers.admit("prov")
8380            else {
8381                panic!("second relay admitted")
8382            };
8383            assert!(
8384                !opener
8385                    .record_timeout(1, Duration::ZERO)
8386                    .unwrap()
8387                    .reopened_after_probe
8388            );
8389            let RouteBindAdmission::Admitted {
8390                guard: mut probe,
8391                probe: true,
8392            } = breakers.admit("prov")
8393            else {
8394                panic!("one cooldown probe admitted")
8395            };
8396            match settlement {
8397                "timeout" => assert!(
8398                    !old.record_timeout(1, Duration::ZERO)
8399                        .unwrap()
8400                        .reopened_after_probe
8401                ),
8402                "inconclusive" => old.record_inconclusive(),
8403                "drop" => drop(old),
8404                _ => unreachable!(),
8405            }
8406            assert!(
8407                matches!(
8408                    breakers.admit("prov"),
8409                    RouteBindAdmission::Refused {
8410                        probe_in_flight: true,
8411                        ..
8412                    }
8413                ),
8414                "{settlement} of a pre-open relay cannot release the real probe"
8415            );
8416            assert!(
8417                probe
8418                    .record_timeout(1, Duration::ZERO)
8419                    .unwrap()
8420                    .reopened_after_probe
8421            );
8422            assert!(matches!(
8423                breakers.admit("prov"),
8424                RouteBindAdmission::Admitted { probe: true, .. }
8425            ));
8426        }
8427        let breakers = RouteBindBreakers::default();
8428        let admit = || match breakers.admit("prov") {
8429            RouteBindAdmission::Admitted { guard, .. } => guard,
8430            _ => panic!("relay admitted"),
8431        };
8432        admit().record_timeout(1, Duration::ZERO);
8433        let mut old_probe = admit();
8434        breakers.reset_for_new_module_connection("prov");
8435        admit().record_timeout(1, Duration::ZERO);
8436        let _new_probe = admit();
8437        old_probe.record_inconclusive();
8438        assert!(matches!(
8439            breakers.admit("prov"),
8440            RouteBindAdmission::Refused {
8441                probe_in_flight: true,
8442                ..
8443            }
8444        ));
8445    }
8446
8447    #[tokio::test]
8448    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8449        for candidate in [false, true] {
8450            let registry = Arc::new(Registry::default());
8451            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
8452                [("vault".to_string(), true), ("squatter".to_string(), true)],
8453                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8454            );
8455            let conn = ConnectionId::new(77);
8456            let (ctx, mut rx) = route_ctx(conn);
8457            let initial = capability_manifest("squatter", &[], &[]);
8458            if candidate {
8459                registry
8460                    .register_candidate_with_control_ops(
8461                        initial.clone(),
8462                        PROTOCOL_VERSION,
8463                        conn,
8464                        module_baseline_control_ops(),
8465                    )
8466                    .unwrap();
8467                handler
8468                    .forwarding
8469                    .register_candidate_module_connection(
8470                        conn,
8471                        "squatter".to_string(),
8472                        PROTOCOL_VERSION,
8473                        manifest_concurrency(&initial),
8474                        ctx.egress.clone(),
8475                    )
8476                    .unwrap();
8477            } else {
8478                hello_via_sink(
8479                    &handler,
8480                    &ctx,
8481                    &mut rx,
8482                    hello_frame_with_manifest(initial.clone(), 1),
8483                )
8484                .await;
8485            }
8486            let response = handler
8487                .handle_control_frame(
8488                    &ctx,
8489                    catalog_update_with_capabilities_frame(
8490                        2,
8491                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
8492                            .capabilities
8493                            .unwrap(),
8494                    ),
8495                )
8496                .await
8497                .unwrap();
8498            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8499            assert_eq!(
8500                registry
8501                    .get_module_by_connection(conn)
8502                    .unwrap()
8503                    .unwrap()
8504                    .manifest,
8505                initial
8506            );
8507        }
8508    }
8509
8510    #[test]
8511    fn server_describe_surfaces_required_capability_verdict_fields() {
8512        let registry = Arc::new(Registry::default());
8513        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
8514            [
8515                ("consumer".to_string(), true),
8516                ("provider".to_string(), false),
8517            ],
8518            BTreeMap::new(),
8519        );
8520        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8521        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8522            provides: Vec::new(),
8523            requires: vec![subc_protocol::manifest::CapabilityRequirement {
8524                capability: "credentials-provider/v1".to_string(),
8525                need: subc_protocol::manifest::CapabilityNeed::Required,
8526            }],
8527            must_never_reach: Vec::new(),
8528        });
8529        let hello = Frame::build(
8530            FrameType::Hello,
8531            control_flags(),
8532            0,
8533            0,
8534            78,
8535            serde_json::to_vec(&ModuleHelloBody {
8536                manifest: consumer,
8537                protocol_ver: PROTOCOL_VERSION,
8538                control_ops: None,
8539                launch_nonce: None,
8540            })
8541            .expect("HELLO serializes"),
8542        )
8543        .expect("HELLO frame builds");
8544        handler
8545            .handle_control(ConnectionId::new(78), hello)
8546            .expect("consumer registers");
8547        let describe = Frame::build(
8548            FrameType::Request,
8549            control_flags(),
8550            0,
8551            0,
8552            79,
8553            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8554                .expect("request serializes"),
8555        )
8556        .expect("describe frame builds");
8557        let response = handler
8558            .handle_server_describe(describe)
8559            .expect("server.describe succeeds");
8560        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8561        let requirement = &rendered["capability_requirements"][0];
8562        assert_eq!(requirement["consumer"], "consumer");
8563        assert_eq!(requirement["verdict"], "never_provided");
8564        assert_eq!(requirement["episode_seq"], 1);
8565        assert_eq!(requirement["config_satisfiable"], false);
8566        assert_eq!(requirement["runtime_available"], false);
8567        assert!(requirement["detail"]
8568            .as_str()
8569            .expect("detail string")
8570            .contains("credentials-provider/v1"));
8571    }
8572
8573    #[test]
8574    fn catalog_list_omits_capabilities_for_legacy_manifest() {
8575        let registry = Arc::new(Registry::default());
8576        let handler = ControlHandler::new(Arc::clone(&registry));
8577        let hello = handler
8578            .handle_control(
8579                ConnectionId::new(101),
8580                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8581            )
8582            .expect("legacy HELLO registers");
8583        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8584
8585        let request = Frame::build(
8586            FrameType::Request,
8587            control_flags(),
8588            0,
8589            0,
8590            102,
8591            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8592                .expect("catalog request serializes"),
8593        )
8594        .expect("catalog request frame builds");
8595        let response = handler
8596            .handle_catalog_list(request, None)
8597            .expect("catalog list succeeds");
8598        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8599        assert!(
8600            body["modules"][0].get("capabilities").is_none(),
8601            "legacy manifest must retain an absent capabilities field on catalog.list"
8602        );
8603    }
8604
8605    #[test]
8606    fn hello_ack_omits_storage_when_no_storage_config() {
8607        let registry = Arc::new(Registry::default());
8608        let handler = ControlHandler::new(Arc::clone(&registry));
8609        let responses = handler
8610            .handle_control(
8611                ConnectionId::new(1),
8612                hello_frame("aft", PROTOCOL_VERSION, 7),
8613            )
8614            .unwrap();
8615        let ack = parse_ack(&responses[0]);
8616        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8617        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8618    }
8619
8620    #[tokio::test]
8621    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8622        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
8623        let registry = Arc::new(Registry::default());
8624        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
8625        let responses = handler
8626            .handle_control(
8627                ConnectionId::new(1),
8628                hello_frame("aft", PROTOCOL_VERSION, 7),
8629            )
8630            .unwrap();
8631        let ack = parse_ack(&responses[0]);
8632        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
8633
8634        let described = handler
8635            .handle_control_frame(
8636                &route_ctx(ConnectionId::new(2)).0,
8637                Frame::build(
8638                    FrameType::Request,
8639                    control_flags(),
8640                    0,
8641                    0,
8642                    9,
8643                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8644                )
8645                .unwrap(),
8646            )
8647            .await
8648            .unwrap();
8649        let ClientControlResponse::ServerDescribe { machine_id, .. } =
8650            serde_json::from_slice(&described[0].body).unwrap()
8651        else {
8652            panic!("server.describe answered with another shape");
8653        };
8654        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
8655    }
8656
8657    #[test]
8658    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
8659        // With a central sqlite storage policy, each registering module gets its
8660        // own resolved descriptor in HELLO_ACK, keyed by its module id.
8661        let registry = Arc::new(Registry::default());
8662        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
8663            crate::daemon_config::StorageConfig::Sqlite {
8664                data_home: std::path::PathBuf::from("/data"),
8665            },
8666        ));
8667
8668        let responses = handler
8669            .handle_control(
8670                ConnectionId::new(1),
8671                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8672            )
8673            .unwrap();
8674        let ack = parse_ack(&responses[0]);
8675        assert_eq!(
8676            ack.storage,
8677            Some(serde_json::json!({
8678                "module_id": "alfonso-routing",
8679                "storage_namespace": "default",
8680                "isolation": { "kind": "module" },
8681                "backend": {
8682                    "backend": "sqlite",
8683                    "path": "/data/cortexkit/alfonso-routing/store.db"
8684                }
8685            })),
8686            "the delivered descriptor is the module's own sqlite store path"
8687        );
8688    }
8689
8690    #[test]
8691    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8692        let registry = Arc::new(Registry::default());
8693        let handler = ControlHandler::new(Arc::clone(&registry));
8694        let conn = ConnectionId::new(1);
8695        let responses = handler
8696            .handle_control(
8697                conn,
8698                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8699            )
8700            .unwrap();
8701        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8702        let registration = registry.get_module("aft").unwrap().unwrap();
8703        assert_eq!(registration.control_ops, module_baseline_control_ops());
8704
8705        let frame =
8706            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8707        assert!(handler
8708            .guard_module_control_op(&frame, "aft", "route.bind")
8709            .unwrap()
8710            .is_none());
8711        let error = handler
8712            .guard_module_control_op(&frame, "aft", "test.synthetic")
8713            .unwrap()
8714            .expect("synthetic ungranted op should be rejected");
8715        assert_eq!(error.header.ty, FrameType::Error);
8716        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8717    }
8718
8719    #[test]
8720    fn hello_control_ops_some_adds_optional_grants() {
8721        let registry = Arc::new(Registry::default());
8722        let handler = ControlHandler::new(Arc::clone(&registry));
8723        handler
8724            .handle_control(
8725                ConnectionId::new(1),
8726                hello_frame_with_control_ops(
8727                    "aft",
8728                    PROTOCOL_VERSION,
8729                    7,
8730                    Some(vec![
8731                        "future.synthetic".to_string(),
8732                        "route.bind".to_string(),
8733                    ]),
8734                ),
8735            )
8736            .unwrap();
8737        let registration = registry.get_module("aft").unwrap().unwrap();
8738        assert_eq!(
8739            registration.control_ops,
8740            vec![
8741                "route.bind".to_string(),
8742                "route.status".to_string(),
8743                "future.synthetic".to_string(),
8744            ]
8745        );
8746        let frame =
8747            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
8748        assert!(handler
8749            .guard_module_control_op(&frame, "aft", "future.synthetic")
8750            .unwrap()
8751            .is_none());
8752    }
8753
8754    #[tokio::test]
8755    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
8756        let registry = Arc::new(Registry::default());
8757        let forwarding = Arc::new(ForwardingTable::default());
8758        let handler =
8759            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8760        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
8761        hello_via_sink(
8762            &handler,
8763            &module_ctx,
8764            &mut module_rx,
8765            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8766        )
8767        .await;
8768
8769        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
8770        let responses = handler
8771            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
8772            .await
8773            .unwrap();
8774        assert_eq!(responses.len(), 1);
8775        assert_eq!(responses[0].header.ty, FrameType::Error);
8776        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
8777        assert!(module_rx.try_recv().is_err());
8778    }
8779
8780    #[tokio::test]
8781    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
8782        let registry = Arc::new(Registry::default());
8783        let forwarding = Arc::new(ForwardingTable::default());
8784        let handler =
8785            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8786        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
8787        hello_via_sink(
8788            &handler,
8789            &module_ctx,
8790            &mut module_rx,
8791            hello_frame_with_control_ops(
8792                "aft",
8793                PROTOCOL_VERSION,
8794                7,
8795                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8796            ),
8797        )
8798        .await;
8799
8800        let project_root = unique_project_root("demux");
8801        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
8802        let route_handler = handler.clone();
8803        let route_task = tokio::spawn(async move {
8804            route_handler
8805                .handle_control_frame(
8806                    &route_client_ctx,
8807                    route_open_frame(100, "aft", project_root),
8808                )
8809                .await
8810                .unwrap()
8811        });
8812        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8813            .await
8814            .unwrap()
8815            .unwrap();
8816        assert!(matches!(
8817            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
8818            ModuleControlRequest::RouteBind { .. }
8819        ));
8820
8821        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
8822        let health_handler = handler.clone();
8823        let health_task = tokio::spawn(async move {
8824            health_handler
8825                .handle_control_frame(
8826                    &health_client_ctx,
8827                    supervisor_health_probe_frame(101, "aft"),
8828                )
8829                .await
8830                .unwrap()
8831        });
8832        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8833            .await
8834            .unwrap()
8835            .unwrap();
8836        assert_eq!(
8837            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8838            ModuleControlRequest::HealthCheck {}
8839        );
8840
8841        handler
8842            .handle_control_frame(
8843                &module_ctx,
8844                health_response(health_frame.header.corr, HealthStatus::Degraded),
8845            )
8846            .await
8847            .unwrap();
8848        let health_response = health_task.await.unwrap();
8849        assert_eq!(health_response.len(), 1);
8850        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8851            ClientControlResponse::SupervisorHealthProbe {
8852                module_id,
8853                status,
8854                detail,
8855                metrics,
8856            } => {
8857                assert_eq!(module_id, "aft");
8858                assert_eq!(status, HealthStatus::Degraded);
8859                assert_eq!(detail.as_deref(), Some("warming"));
8860                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8861            }
8862            other => panic!("unexpected health response: {other:?}"),
8863        }
8864
8865        handler
8866            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8867            .await
8868            .unwrap();
8869        let route_response = route_task.await.unwrap();
8870        assert!(route_response.is_empty());
8871        let published = route_client_rx.recv().await.unwrap();
8872        assert!(matches!(
8873            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8874            ClientControlResponse::RouteOpen { .. }
8875        ));
8876    }
8877
8878    /// Start one `route.open` on `client_connection` and return its still-running
8879    /// handler task together with the `route.bind` the module received for it.
8880    /// The handler blocks until the module answers, so it has to run as a task
8881    /// while the test drives the module side.
8882    async fn relay_route_open(
8883        handler: &ControlHandler,
8884        client_connection: ConnectionId,
8885        client_egress: &FrameSink,
8886        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8887        corr: u64,
8888        module_id: &str,
8889        project_root_label: &str,
8890    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8891        let ctx = RouteCtx {
8892            connection_id: client_connection,
8893            egress: client_egress.clone(),
8894        };
8895        let handler = handler.clone();
8896        let project_root = unique_project_root(project_root_label);
8897        let module_id = module_id.to_string();
8898        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8899        let task = tokio::spawn(async move {
8900            let _guard = tracing::dispatcher::set_default(&dispatch);
8901            handler
8902                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8903                .await
8904                .unwrap()
8905        });
8906        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8907            .await
8908            .expect("module receives the relayed route.bind")
8909            .expect("module egress is open");
8910        (task, bind.frame)
8911    }
8912
8913    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8914        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8915            ModuleControlRequest::RouteBind {
8916                route_channel,
8917                epoch,
8918                ..
8919            } => (route_channel, epoch),
8920            other => panic!("expected a route.bind request, got {other:?}"),
8921        }
8922    }
8923
8924    fn published_route(frame: &Frame) -> (u16, u32) {
8925        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8926            ClientControlResponse::RouteOpen {
8927                route_channel,
8928                route_epoch,
8929            } => (route_channel, route_epoch),
8930            other => panic!("expected a route.open response, got {other:?}"),
8931        }
8932    }
8933
8934    /// Reproduction of a production outage. A client had `route.open`s in
8935    /// flight to a module and was already marked closing -- its egress had refused a
8936    /// module frame, so the daemon asked its connection to end -- while its sink
8937    /// was still open. When the module acked those binds, the daemon refused to
8938    /// commit a route for a closing client, and that refusal was returned from
8939    /// the MODULE connection's frame handler, where a router error that has no
8940    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
8941    /// the supervisor correctly did not respawn a clean exit, and every seat lost
8942    /// its tools for hours -- one client's teardown took down a connection
8943    /// carrying ~170 other routes.
8944    ///
8945    /// The window is opened here by calling the production path that opens it
8946    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
8947    /// state that matters is "in `closing_connections`, sink still open, relay
8948    /// still pending", and it lasts only from the close request until the
8949    /// connection loop reacts to it; a socket-level test can flood a client into
8950    /// that escalation but cannot pin the module's ack inside the window. Closing
8951    /// the socket instead takes the other path entirely -- connection teardown
8952    /// removes the pending relay under the same lock, so the ack finds nothing.
8953    #[tokio::test]
8954    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8955        let registry = Arc::new(Registry::default());
8956        let forwarding = Arc::new(ForwardingTable::default());
8957        let handler =
8958            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8959
8960        let module_connection = ConnectionId::new(30);
8961        let (module_ctx, mut module_rx) = route_ctx(module_connection);
8962        hello_via_sink(
8963            &handler,
8964            &module_ctx,
8965            &mut module_rx,
8966            hello_frame("aft", PROTOCOL_VERSION, 7),
8967        )
8968        .await;
8969
8970        let dying_client = ConnectionId::new(31);
8971        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8972
8973        // A published route on the dying client. The escalation below only marks
8974        // a connection closing for a route it has already published.
8975        let (first_task, first_bind) = relay_route_open(
8976            &handler,
8977            dying_client,
8978            &dying_ctx.egress,
8979            &mut module_rx,
8980            100,
8981            "aft",
8982            "closing-first",
8983        )
8984        .await;
8985        handler
8986            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8987            .await
8988            .unwrap();
8989        assert!(first_task.await.unwrap().is_empty());
8990        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8991
8992        // A second route.open from the same client, relayed and awaiting its ack.
8993        let (second_task, second_bind) = relay_route_open(
8994            &handler,
8995            dying_client,
8996            &dying_ctx.egress,
8997            &mut module_rx,
8998            101,
8999            "aft",
9000            "closing-second",
9001        )
9002        .await;
9003        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9004
9005        // The window: the client is closing, its sink is still open, and its
9006        // second bind is still pending.
9007        assert!(forwarding
9008            .escalate_client_delivery_failure(
9009                dying_client,
9010                first_channel,
9011                first_epoch,
9012                CloseReason::new(
9013                    "module_to_client_delivery_failed",
9014                    "client egress refused a module frame",
9015                ),
9016                crate::forwarding::UndeliveredFrame {
9017                    module_id: None,
9018                    sink: &dying_ctx.egress,
9019                },
9020            )
9021            .unwrap());
9022        assert!(!dying_ctx.egress.is_closed());
9023
9024        // The frame that used to end the module connection.
9025        let ack = handler
9026            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9027            .await;
9028        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9029        if module_loop_error.is_some() {
9030            // What the server's connection loop does with a router error that has
9031            // no ERROR-frame translation: end the connection, which releases the
9032            // module's registration and every route on it.
9033            handler.cleanup_connection(module_connection).unwrap();
9034        }
9035        // Read the module's next frame before opening the co-tenant's route, so
9036        // the GOODBYE assertion below is about THIS ack and not about later
9037        // traffic. `None` means the module was told nothing.
9038        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9039            .await
9040            .ok()
9041            .flatten();
9042
9043        // 1. The module connection is still registered.
9044        assert!(
9045            registry
9046                .get_module_by_connection(module_connection)
9047                .unwrap()
9048                .is_some(),
9049            "one client's closing connection ended the shared module connection: \
9050             {module_loop_error:?}"
9051        );
9052        // ...and still serving: another client can open and use a route on it.
9053        let cotenant = ConnectionId::new(32);
9054        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9055        let (cotenant_task, cotenant_bind) = relay_route_open(
9056            &handler,
9057            cotenant,
9058            &cotenant_ctx.egress,
9059            &mut module_rx,
9060            102,
9061            "aft",
9062            "closing-cotenant",
9063        )
9064        .await;
9065        handler
9066            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9067            .await
9068            .unwrap();
9069        assert!(cotenant_task.await.unwrap().is_empty());
9070        let (cotenant_channel, cotenant_epoch) =
9071            published_route(&cotenant_rx.recv().await.unwrap());
9072        assert!(matches!(
9073            forwarding
9074                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9075                .unwrap(),
9076            DataRoute::Client(DataRouteState::Bound(_))
9077        ));
9078
9079        // 2. The module was told to drop the binding it created for the route
9080        //    that will never be published.
9081        let goodbye = post_ack_module_frame
9082            .expect("module receives a GOODBYE for the abandoned route channel");
9083        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9084        assert_eq!(goodbye.header.channel, abandoned_channel);
9085        assert_eq!(goodbye.header.epoch, abandoned_epoch);
9086
9087        // 3. The dying client received nothing: no route was ever published to
9088        //    it. Its route.open is answered as unavailable, which the connection
9089        //    loop would write to a socket that is already going away.
9090        assert!(dying_rx.try_recv().is_err());
9091        let second_response = second_task.await.unwrap();
9092        assert_eq!(second_response.len(), 1);
9093        assert_eq!(
9094            parse_error(&second_response[0])["code"],
9095            "target_unavailable"
9096        );
9097    }
9098
9099    /// A handler whose `route.open` parks after queueing the route.bind and
9100    /// before marking it enqueued, with module "aft" registered on connection
9101    /// 40. Returns the two-party barrier that parks and releases it.
9102    async fn route_open_held_after_bind_send() -> (
9103        ControlHandler,
9104        Arc<tokio::sync::Barrier>,
9105        RouteCtx,
9106        mpsc::Receiver<crate::router::OutboundFrame>,
9107    ) {
9108        let hold = Arc::new(tokio::sync::Barrier::new(2));
9109        let handler = ControlHandler::with_forwarding(
9110            Arc::new(Registry::default()),
9111            Arc::new(ForwardingTable::default()),
9112        )
9113        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
9114        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9115        hello_via_sink(
9116            &handler,
9117            &module_ctx,
9118            &mut module_rx,
9119            hello_frame("aft", PROTOCOL_VERSION, 7),
9120        )
9121        .await;
9122        (handler, hold, module_ctx, module_rx)
9123    }
9124
9125    /// A fast module acks the route.bind, and the reader commits the route,
9126    /// before `route.open` has marked the relay enqueued. The daemon used to
9127    /// read the missing relay as an abandoned bind and send the module a
9128    /// GOODBYE for the route it had just committed: the client was told its
9129    /// route was open, the module unbound the channel, and the client's first
9130    /// request was dropped on the module side as unbound.
9131    #[tokio::test]
9132    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
9133        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9134        let client = ConnectionId::new(41);
9135        let (client_ctx, mut client_rx) = route_ctx(client);
9136
9137        let (open_task, bind) = relay_route_open(
9138            &handler,
9139            client,
9140            &client_ctx.egress,
9141            &mut module_rx,
9142            200,
9143            "aft",
9144            "acked-before-mark",
9145        )
9146        .await;
9147        let (module_channel, module_epoch) = route_bind_channel(&bind);
9148        // route.open is now parked between queueing the bind and marking it.
9149        hold.wait().await;
9150
9151        // The module answers inside that window, and the route commits.
9152        assert!(handler
9153            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9154            .await
9155            .unwrap()
9156            .is_empty());
9157        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
9158
9159        hold.wait().await;
9160        assert!(
9161            open_task.await.unwrap().is_empty(),
9162            "route.open answers accepted"
9163        );
9164
9165        // Every frame route.open could have sent is already queued, because it
9166        // has returned; the module must have been sent nothing.
9167        match module_rx.try_recv() {
9168            Err(mpsc::error::TryRecvError::Empty) => {}
9169            Ok(outbound) => panic!(
9170                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
9171                outbound.frame.header.ty,
9172                outbound.frame.header.channel,
9173                outbound.frame.header.epoch
9174            ),
9175            Err(err) => panic!("module egress closed: {err}"),
9176        }
9177
9178        // The client's first request on the route is the next thing the module
9179        // reads, on the binding it just created.
9180        let router = Router::with_control_handler(Arc::new(handler.clone()));
9181        let request = Frame::build(
9182            FrameType::Request,
9183            Flags::new(true, Priority::Interactive, false),
9184            client_channel,
9185            client_epoch,
9186            201,
9187            b"first request".to_vec(),
9188        )
9189        .unwrap();
9190        router
9191            .route_for_connection(&client_ctx, request)
9192            .await
9193            .unwrap();
9194        let delivered = module_rx.try_recv().unwrap().frame;
9195        assert_eq!(delivered.header.ty, FrameType::Request);
9196        assert_eq!(
9197            (delivered.header.channel, delivered.header.epoch),
9198            (module_channel, module_epoch)
9199        );
9200        assert_eq!(delivered.body, b"first request");
9201    }
9202
9203    /// The client goes away while `route.open` is still queueing the bind, so
9204    /// the relay is dropped with no module reply. The teardown cannot send the
9205    /// GOODBYE itself (it could reach the module ahead of the bind), so
9206    /// `route.open` must send it once the bind is queued; otherwise the module
9207    /// keeps a binding the daemon will never commit.
9208    #[tokio::test]
9209    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
9210        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9211        let client = ConnectionId::new(42);
9212        let (client_ctx, _client_rx) = route_ctx(client);
9213
9214        let (open_task, bind) = relay_route_open(
9215            &handler,
9216            client,
9217            &client_ctx.egress,
9218            &mut module_rx,
9219            300,
9220            "aft",
9221            "released-before-mark",
9222        )
9223        .await;
9224        let (module_channel, module_epoch) = route_bind_channel(&bind);
9225        hold.wait().await;
9226
9227        handler.cleanup_connection(client).unwrap();
9228        assert!(
9229            module_rx.try_recv().is_err(),
9230            "client teardown must not GOODBYE a bind route.open is still queueing"
9231        );
9232
9233        hold.wait().await;
9234        let response = open_task.await.unwrap();
9235        assert_eq!(response.len(), 1);
9236        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9237
9238        let goodbye = module_rx
9239            .try_recv()
9240            .expect("module is told to drop the binding the daemon abandoned")
9241            .frame;
9242        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9243        assert_eq!(
9244            (goodbye.header.channel, goodbye.header.epoch),
9245            (module_channel, module_epoch)
9246        );
9247        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9248    }
9249
9250    /// The module acks inside the same window, but the client's egress closed
9251    /// first, so the reader refuses to commit. The ack proves the module holds
9252    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
9253    /// send a second one.
9254    #[tokio::test]
9255    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
9256        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9257        let client = ConnectionId::new(43);
9258        let (client_ctx, client_rx) = route_ctx(client);
9259
9260        let (open_task, bind) = relay_route_open(
9261            &handler,
9262            client,
9263            &client_ctx.egress,
9264            &mut module_rx,
9265            400,
9266            "aft",
9267            "gone-client-before-mark",
9268        )
9269        .await;
9270        let (module_channel, module_epoch) = route_bind_channel(&bind);
9271        hold.wait().await;
9272
9273        drop(client_rx);
9274        assert!(client_ctx.egress.is_closed());
9275        handler
9276            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9277            .await
9278            .unwrap();
9279
9280        hold.wait().await;
9281        let response = open_task.await.unwrap();
9282        assert_eq!(response.len(), 1);
9283        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9284
9285        let goodbye = module_rx
9286            .try_recv()
9287            .expect("module is told to drop the binding it acked")
9288            .frame;
9289        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9290        assert_eq!(
9291            (goodbye.header.channel, goodbye.header.epoch),
9292            (module_channel, module_epoch)
9293        );
9294        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9295        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
9296    }
9297
9298    /// The fence at the module-loop boundary, stated as its own contract: which
9299    /// forwarding failures are allowed to end the module connection that is being
9300    /// served. A `ConnectionClosing` naming some client is about that client, and
9301    /// a module connection is shared; the same error naming the module's own
9302    /// connection is about this connection and must stay fatal, as must failures
9303    /// that are about the forwarding table itself.
9304    #[test]
9305    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9306        let handler = ControlHandler::default();
9307        let module_connection = ConnectionId::new(30);
9308        let client_connection = ConnectionId::new(31);
9309
9310        handler
9311            .refuse_to_end_module_connection_for_a_client(
9312                module_connection,
9313                77,
9314                ForwardingError::ConnectionClosing {
9315                    connection_id: client_connection,
9316                },
9317            )
9318            .expect("a closing client must never end the module connection");
9319
9320        assert!(matches!(
9321            handler.refuse_to_end_module_connection_for_a_client(
9322                module_connection,
9323                78,
9324                ForwardingError::ConnectionClosing {
9325                    connection_id: module_connection,
9326                },
9327            ),
9328            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9329                connection_id
9330            })) if connection_id == module_connection
9331        ));
9332        assert!(matches!(
9333            handler.refuse_to_end_module_connection_for_a_client(
9334                module_connection,
9335                79,
9336                ForwardingError::Poisoned,
9337            ),
9338            Err(RouterError::Forwarding(ForwardingError::Poisoned))
9339        ));
9340        assert!(matches!(
9341            handler.refuse_to_end_module_connection_for_a_client(
9342                module_connection,
9343                80,
9344                ForwardingError::StaleModuleEndpoint,
9345            ),
9346            Err(RouterError::Forwarding(
9347                ForwardingError::StaleModuleEndpoint
9348            ))
9349        ));
9350    }
9351
9352    /// The spawn-attestation guard is what stops a connected module from claiming
9353    /// another module's identity and being stamped `Reserved` for it. Every other
9354    /// test that supplies a consumer_identity supplies a CORRECT one, because a
9355    /// correct one is what the rest of the flow needs -- so the guard's rejection
9356    /// branch was never the subject of an assertion, only its acceptance branch.
9357    ///
9358    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
9359    /// whole subc-core library suite green; only the forwarding integration tests
9360    /// notice, and they notice for unrelated reasons. This test exists so the
9361    /// refusal itself is asserted where the guard lives: it fails if the identity
9362    /// check stops refusing, which is the direction that matters, since a guard
9363    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
9364    #[tokio::test]
9365    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9366        let registry = Arc::new(Registry::default());
9367        let forwarding = Arc::new(ForwardingTable::default());
9368        let supervisor = SupervisorHandle::new();
9369        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9370        let handler =
9371            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9372                .with_supervisor(supervisor);
9373
9374        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9375        hello_via_sink(
9376            &handler,
9377            &target_ctx,
9378            &mut target_rx,
9379            hello_frame("target", PROTOCOL_VERSION, 1),
9380        )
9381        .await;
9382
9383        // A real supervised module id presenting the wrong nonce. This is the
9384        // impersonation case: the attacker knows a privileged module_id, which is
9385        // public, and guesses at the nonce, which is not.
9386        let wrong_nonce = handler
9387            .handle_control_frame(
9388                &route_ctx(ConnectionId::new(91)).0,
9389                route_open_frame_with_admission_facts(
9390                    20,
9391                    "target",
9392                    unique_project_root("admission-facts"),
9393                    Some(subc_control::ConsumerIdentity {
9394                        module_id: "fed".to_string(),
9395                        launch_nonce: "not-the-real-nonce".to_string(),
9396                    }),
9397                    None,
9398                ),
9399            )
9400            .await
9401            .unwrap();
9402        assert_eq!(
9403            parse_error(&wrong_nonce[0])["code"],
9404            "bad_consumer_identity",
9405            "a mismatched launch nonce must be refused, not stamped Reserved"
9406        );
9407
9408        // A module id the supervisor never spawned at all, so no nonce exists to
9409        // compare against. An implementation that treats "no record" as "nothing
9410        // to check" fails open here while passing the case above.
9411        let never_spawned = handler
9412            .handle_control_frame(
9413                &route_ctx(ConnectionId::new(92)).0,
9414                route_open_frame_with_admission_facts(
9415                    21,
9416                    "target",
9417                    unique_project_root("admission-facts"),
9418                    Some(subc_control::ConsumerIdentity {
9419                        module_id: "never-spawned".to_string(),
9420                        launch_nonce: "any-nonce".to_string(),
9421                    }),
9422                    None,
9423                ),
9424            )
9425            .await
9426            .unwrap();
9427        assert_eq!(
9428            parse_error(&never_spawned[0])["code"],
9429            "bad_consumer_identity",
9430            "an unspawned module_id must be refused rather than accepted for lack of a record"
9431        );
9432    }
9433
9434    /// The refusal test above proves the guard says NO. Nothing proved it can say
9435    /// YES, and the difference is not academic: replacing the whole authorization
9436    /// with `false` -- admitting no consumer identity at all, revoking Reserved
9437    /// standing for every supervised module in the fleet -- leaves 110 of the 111
9438    /// library tests GREEN. The one that notices does so by HANGING, because it
9439    /// waits for a bind that can no longer happen.
9440    ///
9441    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
9442    /// or flaky test, invites a RETRY rather than an investigation, and the retry
9443    /// hangs too and gets blamed on the runner. So a total revocation of the
9444    /// daemon's trust grant would have shipped behind a symptom nobody attributes
9445    /// to code.
9446    ///
9447    /// The bias is structural rather than accidental. A REFUSAL looks like a
9448    /// failure someone writes a test for; a GRANT looks like the happy path. Every
9449    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
9450    /// suite for that reason, and this one is the purest case in the daemon.
9451    ///
9452    /// This test asserts the EFFECT rather than the absence of an error: the module
9453    /// receives a RouteBind and it carries `Reserved` naming the attested module.
9454    /// A guard that admitted nobody would produce no bind at all; one that admitted
9455    /// everybody would stamp the wrong principal, which the refusal test catches.
9456    #[tokio::test]
9457    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
9458        let registry = Arc::new(Registry::default());
9459        let forwarding = Arc::new(ForwardingTable::default());
9460        let supervisor = SupervisorHandle::new();
9461        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9462        let handler =
9463            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9464                .with_supervisor(supervisor);
9465
9466        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
9467        hello_via_sink(
9468            &handler,
9469            &target_ctx,
9470            &mut target_rx,
9471            hello_frame("target", PROTOCOL_VERSION, 1),
9472        )
9473        .await;
9474
9475        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
9476        let route_handler = handler.clone();
9477        let route_task = tokio::spawn(async move {
9478            route_handler
9479                .handle_control_frame(
9480                    &client_ctx,
9481                    route_open_frame_with_admission_facts(
9482                        30,
9483                        "target",
9484                        unique_project_root("admission-facts"),
9485                        Some(subc_control::ConsumerIdentity {
9486                            module_id: "fed".to_string(),
9487                            launch_nonce: "fed-nonce".to_string(),
9488                        }),
9489                        None,
9490                    ),
9491                )
9492                .await
9493                .unwrap()
9494        });
9495
9496        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
9497        // the very mutation it exists to catch -- a guard that admits nobody -- no
9498        // bind is ever sent, so it HUNG rather than failing. That reproduces the
9499        // exact defect being fixed: a total revocation detected only as a stalled
9500        // suite, which reads as flakiness and invites a retry. An acceptance test
9501        // that waits for an effect must bound the wait, or a red becomes a hang.
9502        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9503            .await
9504            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
9505            .expect("module control channel closed before route.bind");
9506        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9507        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9508            panic!("expected route.bind")
9509        };
9510        assert_eq!(
9511            principal,
9512            Some(Principal::Reserved {
9513                module_id: "fed".to_string()
9514            }),
9515            "a correctly attested consumer must be stamped Reserved for its own id"
9516        );
9517
9518        handler
9519            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9520            .await
9521            .unwrap();
9522        assert!(route_task.await.unwrap().is_empty());
9523        assert!(
9524            matches!(
9525                serde_json::from_slice::<ClientControlResponse>(
9526                    &client_rx.recv().await.unwrap().body
9527                )
9528                .unwrap(),
9529                ClientControlResponse::RouteOpen { .. }
9530            ),
9531            "the route must actually open, not merely avoid an error"
9532        );
9533    }
9534
9535    #[tokio::test(start_paused = true)]
9536    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
9537        let registry = Arc::new(Registry::default());
9538        let forwarding = Arc::new(ForwardingTable::default());
9539        let supervisor = SupervisorHandle::new();
9540        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9541        let handler =
9542            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9543                .with_supervisor(supervisor);
9544
9545        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
9546        hello_via_sink(
9547            &handler,
9548            &target_ctx,
9549            &mut target_rx,
9550            hello_frame("target", PROTOCOL_VERSION, 1),
9551        )
9552        .await;
9553
9554        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
9555        let direct_handler = handler.clone();
9556        let direct_open = tokio::spawn(async move {
9557            direct_handler
9558                .handle_control_frame(
9559                    &direct_ctx,
9560                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
9561                )
9562                .await
9563                .unwrap()
9564        });
9565        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9566            .await
9567            .expect("no direct route.bind within 5s")
9568            .expect("target control channel closed before direct route.bind");
9569        handler
9570            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
9571            .await
9572            .unwrap();
9573        assert!(direct_open.await.unwrap().is_empty());
9574        let _ = direct_rx.recv().await.unwrap();
9575
9576        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
9577        let reserved_handler = handler.clone();
9578        let reserved_open = tokio::spawn(async move {
9579            reserved_handler
9580                .handle_control_frame(
9581                    &reserved_ctx,
9582                    route_open_frame_with_admission_facts(
9583                        3,
9584                        "target",
9585                        unique_project_root("admission-facts"),
9586                        Some(ConsumerIdentity {
9587                            module_id: "fed".to_string(),
9588                            launch_nonce: "fed-nonce".to_string(),
9589                        }),
9590                        None,
9591                    ),
9592                )
9593                .await
9594                .unwrap()
9595        });
9596        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9597            .await
9598            .expect("no reserved route.bind within 5s")
9599            .expect("target control channel closed before reserved route.bind");
9600        handler
9601            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
9602            .await
9603            .unwrap();
9604        assert!(reserved_open.await.unwrap().is_empty());
9605        let _ = reserved_rx.recv().await.unwrap();
9606
9607        forwarding
9608            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
9609            .unwrap();
9610        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
9611        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
9612            module_id: Some("target".to_string()),
9613        })
9614        .unwrap();
9615        let census_frame =
9616            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
9617        let response = handler
9618            .handle_control_frame(&census_ctx, census_frame)
9619            .await
9620            .unwrap()
9621            .pop()
9622            .unwrap();
9623        let actual: Value = serde_json::from_slice(&response.body).unwrap();
9624        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
9625        assert!(matches!(
9626            decoded,
9627            ClientControlResponse::SupervisorRoutes { .. }
9628        ));
9629        let routes = actual["modules"][0]["routes"].as_array().unwrap();
9630        assert_eq!(routes.len(), 2);
9631        assert!(routes.iter().all(|route| route["draining"] == true));
9632        // The census carries WHY: the reason the drain was begun with, in the
9633        // route.closing vocabulary, on every draining route this drain marked.
9634        assert!(
9635            routes.iter().all(|route| route["drain_reason"] == "reload"),
9636            "draining routes must name the drain's reason: {routes:?}"
9637        );
9638        assert!(routes.iter().any(|route| {
9639            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
9640        }));
9641        assert!(routes.iter().any(|route| {
9642            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
9643        }));
9644
9645        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9646            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
9647        if std::env::var_os("UPDATE_GOLDEN").is_some() {
9648            std::fs::write(
9649                &golden_path,
9650                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
9651            )
9652            .unwrap();
9653        }
9654        let expected: Value =
9655            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
9656        assert_eq!(actual, expected);
9657    }
9658
9659    async fn query_live_roots(
9660        handler: &ControlHandler,
9661        module_ctx: &RouteCtx,
9662    ) -> ModuleControlResponseToModule {
9663        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
9664        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
9665        let response = handler
9666            .handle_control_frame(module_ctx, frame)
9667            .await
9668            .unwrap()
9669            .pop()
9670            .unwrap();
9671        serde_json::from_slice(&response.body).unwrap()
9672    }
9673
9674    #[tokio::test(start_paused = true)]
9675    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
9676        let registry = Arc::new(Registry::default());
9677        let forwarding = Arc::new(ForwardingTable::default());
9678        let handler = ControlHandler::with_forwarding(registry, forwarding);
9679        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
9680        hello_via_sink(
9681            &handler,
9682            &target_ctx,
9683            &mut target_rx,
9684            hello_frame("target", PROTOCOL_VERSION, 1),
9685        )
9686        .await;
9687        let root = unique_project_root("live-roots-known");
9688        let path = ProjectRootId::from_path_allowing_missing(root.path())
9689            .unwrap()
9690            .as_path()
9691            .to_path_buf();
9692        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
9693        let open_handler = handler.clone();
9694        let opened = tokio::spawn(async move {
9695            open_handler
9696                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
9697                .await
9698                .unwrap()
9699        });
9700        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9701            .await
9702            .unwrap()
9703            .unwrap();
9704        handler
9705            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
9706            .await
9707            .unwrap();
9708        assert!(opened.await.unwrap().is_empty());
9709        let _ = client_rx.recv().await.unwrap();
9710
9711        let root = unique_project_root("live-roots-pending");
9712        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
9713            .unwrap()
9714            .as_path()
9715            .to_path_buf();
9716        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
9717        let open_handler = handler.clone();
9718        let pending = tokio::spawn(async move {
9719            open_handler
9720                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
9721                .await
9722                .unwrap()
9723        });
9724        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9725            .await
9726            .unwrap()
9727            .unwrap();
9728        let actual = query_live_roots(&handler, &target_ctx).await;
9729        let ModuleControlResponseToModule::LiveRoots {
9730            roots,
9731            unknown_root_bindings,
9732            total_bindings,
9733        } = actual
9734        else {
9735            panic!("expected live roots")
9736        };
9737        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
9738        assert_eq!(unknown_root_bindings, 0);
9739        assert_eq!(
9740            roots.len(),
9741            2,
9742            "root-known arm must retain each canonical root"
9743        );
9744        assert_eq!(
9745            total_bindings,
9746            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9747        );
9748        let counts = roots
9749            .iter()
9750            .map(|root| (root.project_root.clone(), root.bound, root.pending))
9751            .collect::<Vec<_>>();
9752        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
9753        expected.sort_by(|a, b| a.0.cmp(&b.0));
9754        assert_eq!(
9755            counts, expected,
9756            "roots must sort by path and count pending separately"
9757        );
9758        handler
9759            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
9760            .await
9761            .unwrap();
9762        assert!(pending.await.unwrap().is_empty());
9763    }
9764
9765    #[tokio::test(start_paused = true)]
9766    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
9767        let forwarding = Arc::new(ForwardingTable::default());
9768        let handler =
9769            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9770        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
9771        hello_via_sink(
9772            &handler,
9773            &target_ctx,
9774            &mut target_rx,
9775            hello_frame("target", PROTOCOL_VERSION, 1),
9776        )
9777        .await;
9778        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
9779        let pending = forwarding
9780            .begin_route_bind_relay_for_test(
9781                client_ctx.connection_id,
9782                client_ctx.egress.clone(),
9783                2,
9784                "target",
9785            )
9786            .unwrap();
9787        forwarding
9788            .complete_pending_relay(
9789                target_ctx.connection_id,
9790                pending.corr,
9791                RouteBindRelayOutcome::Accepted,
9792            )
9793            .unwrap();
9794        let actual = query_live_roots(&handler, &target_ctx).await;
9795        let ModuleControlResponseToModule::LiveRoots {
9796            roots,
9797            unknown_root_bindings,
9798            total_bindings,
9799        } = actual
9800        else {
9801            panic!("expected live roots")
9802        };
9803        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
9804        assert_eq!(
9805            unknown_root_bindings, 1,
9806            "unknown-root arm must not read as no bindings"
9807        );
9808        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
9809        assert_eq!(
9810            total_bindings,
9811            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9812        );
9813    }
9814
9815    /// A module reads HELLO_ACK as its first frame and exits on anything else,
9816    /// so the ack has to be on its outbound queue before the module is
9817    /// routable. The connection loop writes a handler's replies only after the
9818    /// handler returns; this test stops in exactly that gap, runs a real
9819    /// route.open from another connection, and only then writes whatever the
9820    /// HELLO handler returned, the way the loop would. If the ack were still a
9821    /// reply, the route.bind request would reach the module first.
9822    #[tokio::test(start_paused = true)]
9823    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
9824        let forwarding = Arc::new(ForwardingTable::default());
9825        let handler =
9826            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9827        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
9828        let replies = handler
9829            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
9830            .await
9831            .unwrap();
9832        let queued_by_hello = module_rx.len();
9833
9834        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
9835        let open_handler = handler.clone();
9836        let open = tokio::spawn(async move {
9837            open_handler
9838                .handle_control_frame(
9839                    &client_ctx,
9840                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
9841                )
9842                .await
9843                .unwrap()
9844        });
9845        // Let the route.open run until its route.bind is on the module's queue.
9846        let mut spins = 0;
9847        while module_rx.len() == queued_by_hello {
9848            spins += 1;
9849            assert!(spins < 10_000, "route.open never queued a route.bind");
9850            tokio::task::yield_now().await;
9851        }
9852
9853        // Now the connection loop's half: write the HELLO handler's replies.
9854        for reply in replies {
9855            module_ctx.egress.send(reply).await.unwrap();
9856        }
9857
9858        let first = module_rx.recv().await.unwrap().frame;
9859        assert_eq!(
9860            first.header.ty,
9861            FrameType::HelloAck,
9862            "the first frame a registering module reads must be its HELLO_ACK"
9863        );
9864        assert_eq!(first.header.corr, 7);
9865        let second = module_rx.recv().await.unwrap().frame;
9866        assert_eq!(second.header.ty, FrameType::Request);
9867        assert!(
9868            matches!(
9869                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
9870                ModuleControlRequest::RouteBind { .. }
9871            ),
9872            "the route.bind follows the ack"
9873        );
9874        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
9875
9876        handler
9877            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
9878            .await
9879            .unwrap();
9880        assert!(open.await.unwrap().is_empty());
9881        let _ = client_rx.recv().await.unwrap();
9882    }
9883
9884    #[tokio::test(start_paused = true)]
9885    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
9886        let handler = ControlHandler::with_forwarding(
9887            Arc::new(Registry::default()),
9888            Arc::new(ForwardingTable::default()),
9889        );
9890        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
9891        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
9892        hello_via_sink(
9893            &handler,
9894            &first_ctx,
9895            &mut first_rx,
9896            hello_frame("first", PROTOCOL_VERSION, 1),
9897        )
9898        .await;
9899        hello_via_sink(
9900            &handler,
9901            &second_ctx,
9902            &mut second_rx,
9903            hello_frame("second", PROTOCOL_VERSION, 2),
9904        )
9905        .await;
9906        let root = unique_project_root("second-only");
9907        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
9908        let cloned = handler.clone();
9909        let open = tokio::spawn(async move {
9910            cloned
9911                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
9912                .await
9913                .unwrap()
9914        });
9915        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
9916            .await
9917            .unwrap()
9918            .unwrap();
9919        let first = query_live_roots(&handler, &first_ctx).await;
9920        let second = query_live_roots(&handler, &second_ctx).await;
9921        assert!(
9922            matches!(
9923                first,
9924                ModuleControlResponseToModule::LiveRoots {
9925                    total_bindings: 0,
9926                    ..
9927                }
9928            ),
9929            "cross-module scope must not expose another module's roots"
9930        );
9931        assert!(
9932            matches!(
9933                second,
9934                ModuleControlResponseToModule::LiveRoots {
9935                    total_bindings: 1,
9936                    ..
9937                }
9938            ),
9939            "second module must see its pending route"
9940        );
9941        handler
9942            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
9943            .await
9944            .unwrap();
9945        assert!(open.await.unwrap().is_empty());
9946    }
9947
9948    #[tokio::test(start_paused = true)]
9949    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
9950        let handler = ControlHandler::with_forwarding(
9951            Arc::new(Registry::default()),
9952            Arc::new(ForwardingTable::default()),
9953        );
9954        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
9955        hello_via_sink(
9956            &handler,
9957            &target_ctx,
9958            &mut target_rx,
9959            hello_frame("target", PROTOCOL_VERSION, 1),
9960        )
9961        .await;
9962        let actual = query_live_roots(&handler, &target_ctx).await;
9963        let ModuleControlResponseToModule::LiveRoots {
9964            roots,
9965            unknown_root_bindings,
9966            total_bindings,
9967        } = actual
9968        else {
9969            panic!("expected live roots")
9970        };
9971        assert!(roots.is_empty());
9972        assert_eq!(unknown_root_bindings, 0);
9973        assert_eq!(total_bindings, 0);
9974        assert_eq!(
9975            total_bindings,
9976            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9977        );
9978    }
9979
9980    /// Read the vendored fed corpus rather than hand-building a package.
9981    ///
9982    /// A hand-built object encodes what the test author believed the carrier
9983    /// emits. These vectors are what it actually emits, and one of them exists
9984    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
9985    /// ignores additive unknown fields at the traversal emit terminus."
9986    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
9987        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9988            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
9989        let text = std::fs::read_to_string(&path)
9990            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
9991        let vectors: Vec<(String, Value)> = text
9992            .lines()
9993            .filter(|line| !line.trim().is_empty())
9994            .map(|line| {
9995                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
9996                let id = entry["corpus_id"]
9997                    .as_str()
9998                    .expect("every vector carries a corpus_id")
9999                    .to_string();
10000                (id, entry["package"].clone())
10001            })
10002            .collect();
10003        // Pin the count: a corpus that silently shrinks would take its coverage
10004        // with it, and a suite reading N-1 vectors reports the same clean pass
10005        // as one reading N.
10006        assert_eq!(
10007            vectors.len(),
10008            3,
10009            "vendored fed corpus changed size; re-sync from subc-federation"
10010        );
10011
10012        // Pin what makes the corpus DISCRIMINATING, not just present.
10013        //
10014        // The relay test below takes its expected value from the corpus, so the
10015        // corpus supplies the test's power to detect a lossy relay rather than
10016        // its correctness. A relay that dropped unrecognised fields would still
10017        // be caught -- but only by a package carrying fields it does not know.
10018        // Shrink every package to the handful of keys any implementation would
10019        // recognise and the test keeps passing over an input that can no longer
10020        // fail, which is the same clean green as a corpus that shrank away.
10021        //
10022        // So assert the precondition rather than duplicating the packages here:
10023        // at least one vector must carry a field beyond the small common set.
10024        // That is one claim to maintain instead of nine, and it fails loudly if
10025        // a re-sync ever flattens the corpus.
10026        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10027        let richest = vectors
10028            .iter()
10029            .filter_map(|(_, package)| package.as_object())
10030            .map(|object| {
10031                object
10032                    .keys()
10033                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10034                    .count()
10035            })
10036            .max()
10037            .unwrap_or(0);
10038        assert!(
10039            richest >= 2,
10040            "vendored corpus no longer carries a package with unmodelled fields, \
10041             so the relay test can no longer distinguish a verbatim relay from a lossy one"
10042        );
10043
10044        vectors
10045    }
10046
10047    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
10048    ///
10049    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
10050    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
10051    /// three-key object, so a relay that quietly dropped fields it did not
10052    /// recognise would satisfy it. These vectors carry nine keys including ones
10053    /// this crate has no type for, so a typed relay fails here and only here.
10054    #[tokio::test]
10055    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10056        for (corpus_id, package) in fed_admission_facts_vectors() {
10057            let registry = Arc::new(Registry::default());
10058            let forwarding = Arc::new(ForwardingTable::default());
10059            let supervisor = SupervisorHandle::new();
10060            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10061            let handler =
10062                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10063                    .with_supervisor(supervisor)
10064                    .with_admission_facts_config(
10065                        Some("fed".to_string()),
10066                        Some(vec!["target".to_string()]),
10067                    );
10068
10069            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10070            hello_via_sink(
10071                &handler,
10072                &target_ctx,
10073                &mut target_rx,
10074                hello_frame("target", PROTOCOL_VERSION, 1),
10075            )
10076            .await;
10077
10078            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
10079            let route_handler = handler.clone();
10080            let expected = package.clone();
10081            let route_task = tokio::spawn(async move {
10082                route_handler
10083                    .handle_control_frame(
10084                        &client_ctx,
10085                        route_open_frame_with_admission_facts(
10086                            20,
10087                            "target",
10088                            unique_project_root("admission-facts"),
10089                            Some(subc_control::ConsumerIdentity {
10090                                module_id: "fed".to_string(),
10091                                launch_nonce: "fed-nonce".to_string(),
10092                            }),
10093                            Some(package),
10094                        ),
10095                    )
10096                    .await
10097                    .unwrap()
10098            });
10099
10100            let bind_frame = target_rx.recv().await.unwrap();
10101            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10102            let ModuleControlRequest::RouteBind {
10103                admission_facts, ..
10104            } = bind
10105            else {
10106                panic!("{corpus_id}: expected route.bind")
10107            };
10108            assert_eq!(
10109                admission_facts,
10110                Some(expected),
10111                "{corpus_id}: relay must not add, drop or reshape any field"
10112            );
10113
10114            handler
10115                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10116                .await
10117                .unwrap();
10118            route_task.await.unwrap();
10119        }
10120    }
10121
10122    #[tokio::test]
10123    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
10124        let registry = Arc::new(Registry::default());
10125        let forwarding = Arc::new(ForwardingTable::default());
10126        let supervisor = SupervisorHandle::new();
10127        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10128        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
10129        let handler =
10130            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10131                .with_supervisor(supervisor)
10132                .with_admission_facts_config(
10133                    Some("fed".to_string()),
10134                    Some(vec!["target".to_string()]),
10135                );
10136
10137        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
10138        hello_via_sink(
10139            &handler,
10140            &target_ctx,
10141            &mut target_rx,
10142            hello_frame("target", PROTOCOL_VERSION, 1),
10143        )
10144        .await;
10145        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
10146        hello_via_sink(
10147            &handler,
10148            &other_ctx,
10149            &mut other_rx,
10150            hello_frame("other", PROTOCOL_VERSION, 2),
10151        )
10152        .await;
10153
10154        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
10155        let expected_facts = facts.clone();
10156        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
10157        let route_handler = handler.clone();
10158        let route_task = tokio::spawn(async move {
10159            route_handler
10160                .handle_control_frame(
10161                    &client_ctx,
10162                    route_open_frame_with_admission_facts(
10163                        10,
10164                        "target",
10165                        unique_project_root("admission-facts"),
10166                        Some(subc_control::ConsumerIdentity {
10167                            module_id: "fed".to_string(),
10168                            launch_nonce: "fed-nonce".to_string(),
10169                        }),
10170                        Some(facts.clone()),
10171                    ),
10172                )
10173                .await
10174                .unwrap()
10175        });
10176        let bind_frame = target_rx.recv().await.unwrap();
10177        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10178        let ModuleControlRequest::RouteBind {
10179            admission_facts, ..
10180        } = bind
10181        else {
10182            panic!("expected route.bind")
10183        };
10184        assert_eq!(admission_facts, Some(expected_facts));
10185        handler
10186            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10187            .await
10188            .unwrap();
10189        assert!(route_task.await.unwrap().is_empty());
10190        assert!(matches!(
10191            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
10192                .unwrap(),
10193            ClientControlResponse::RouteOpen { .. }
10194        ));
10195
10196        let direct = handler
10197            .handle_control_frame(
10198                &route_ctx(ConnectionId::new(73)).0,
10199                route_open_frame_with_admission_facts(
10200                    11,
10201                    "target",
10202                    unique_project_root("admission-facts"),
10203                    None,
10204                    Some(json!({"x": 1})),
10205                ),
10206            )
10207            .await
10208            .unwrap();
10209        assert_eq!(
10210            parse_error(&direct[0])["code"],
10211            "admission_facts_not_permitted"
10212        );
10213
10214        let different_reserved = handler
10215            .handle_control_frame(
10216                &route_ctx(ConnectionId::new(77)).0,
10217                route_open_frame_with_admission_facts(
10218                    15,
10219                    "target",
10220                    unique_project_root("admission-facts"),
10221                    Some(subc_control::ConsumerIdentity {
10222                        module_id: "other".to_string(),
10223                        launch_nonce: "other-nonce".to_string(),
10224                    }),
10225                    Some(json!({"x": 1})),
10226                ),
10227            )
10228            .await
10229            .unwrap();
10230        assert_eq!(
10231            parse_error(&different_reserved[0])["code"],
10232            "admission_facts_not_permitted"
10233        );
10234
10235        let other_target = handler
10236            .handle_control_frame(
10237                &route_ctx(ConnectionId::new(74)).0,
10238                route_open_frame_with_admission_facts(
10239                    12,
10240                    "other",
10241                    unique_project_root("admission-facts"),
10242                    Some(subc_control::ConsumerIdentity {
10243                        module_id: "fed".to_string(),
10244                        launch_nonce: "fed-nonce".to_string(),
10245                    }),
10246                    Some(json!({"x": 1})),
10247                ),
10248            )
10249            .await
10250            .unwrap();
10251        assert_eq!(
10252            parse_error(&other_target[0])["code"],
10253            "admission_facts_target_not_allowed"
10254        );
10255
10256        let nonexistent = handler
10257            .handle_control_frame(
10258                &route_ctx(ConnectionId::new(75)).0,
10259                route_open_frame_with_admission_facts(
10260                    13,
10261                    "missing",
10262                    unique_project_root("admission-facts"),
10263                    None,
10264                    Some(json!({"x": 1})),
10265                ),
10266            )
10267            .await
10268            .unwrap();
10269        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
10270
10271        let described = handler
10272            .handle_control_frame(
10273                &route_ctx(ConnectionId::new(76)).0,
10274                Frame::build(
10275                    FrameType::Request,
10276                    control_flags(),
10277                    0,
10278                    0,
10279                    14,
10280                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10281                )
10282                .unwrap(),
10283            )
10284            .await
10285            .unwrap();
10286        let ClientControlResponse::ServerDescribe { capabilities, .. } =
10287            serde_json::from_slice(&described[0].body).unwrap()
10288        else {
10289            panic!("expected server.describe response")
10290        };
10291        assert!(capabilities
10292            .iter()
10293            .any(|cap| cap == "admission_facts_relay_v1"));
10294    }
10295
10296    #[tokio::test]
10297    async fn admission_facts_without_configured_carrier_are_rejected() {
10298        let registry = Arc::new(Registry::default());
10299        let forwarding = Arc::new(ForwardingTable::default());
10300        let handler = ControlHandler::with_forwarding(registry, forwarding);
10301        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10302        hello_via_sink(
10303            &handler,
10304            &target_ctx,
10305            &mut target_rx,
10306            hello_frame("target", PROTOCOL_VERSION, 1),
10307        )
10308        .await;
10309
10310        let responses = handler
10311            .handle_control_frame(
10312                &route_ctx(ConnectionId::new(79)).0,
10313                route_open_frame_with_admission_facts(
10314                    16,
10315                    "target",
10316                    unique_project_root("admission-facts"),
10317                    None,
10318                    Some(json!({"x": 1})),
10319                ),
10320            )
10321            .await
10322            .unwrap();
10323        assert_eq!(
10324            parse_error(&responses[0])["code"],
10325            "admission_facts_not_permitted"
10326        );
10327    }
10328
10329    #[tokio::test]
10330    async fn route_open_relays_consumer_capabilities_verbatim() {
10331        let registry = Arc::new(Registry::default());
10332        let forwarding = Arc::new(ForwardingTable::default());
10333        let handler =
10334            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10335        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10336        hello_via_sink(
10337            &handler,
10338            &module_ctx,
10339            &mut module_rx,
10340            hello_frame("aft", PROTOCOL_VERSION, 7),
10341        )
10342        .await;
10343
10344        let expected = vec!["elicitation".to_string(), "roots".to_string()];
10345        let expected_for_request = expected.clone();
10346        let project_root = unique_project_root("consumer-capabilities-present");
10347        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10348        let route_handler = handler.clone();
10349        let route_task = tokio::spawn(async move {
10350            route_handler
10351                .handle_control_frame(
10352                    &client_ctx,
10353                    route_open_frame_with_consumer_capabilities(
10354                        401,
10355                        "aft",
10356                        project_root,
10357                        Some(expected_for_request),
10358                    ),
10359                )
10360                .await
10361                .unwrap()
10362        });
10363        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10364            .await
10365            .unwrap()
10366            .unwrap();
10367        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10368        let ModuleControlRequest::RouteBind {
10369            consumer_capabilities,
10370            ..
10371        } = bind
10372        else {
10373            panic!("expected route.bind request, got {bind:?}");
10374        };
10375        assert_eq!(consumer_capabilities, Some(expected.clone()));
10376
10377        handler
10378            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10379            .await
10380            .unwrap();
10381        let route_response = route_task.await.unwrap();
10382        assert!(route_response.is_empty());
10383        let published = client_rx.recv().await.unwrap();
10384        assert!(matches!(
10385            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10386            ClientControlResponse::RouteOpen { .. }
10387        ));
10388    }
10389
10390    #[tokio::test]
10391    async fn route_open_without_consumer_capabilities_relays_none() {
10392        let registry = Arc::new(Registry::default());
10393        let forwarding = Arc::new(ForwardingTable::default());
10394        let handler =
10395            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10396        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
10397        hello_via_sink(
10398            &handler,
10399            &module_ctx,
10400            &mut module_rx,
10401            hello_frame("aft", PROTOCOL_VERSION, 7),
10402        )
10403        .await;
10404
10405        let project_root = unique_project_root("consumer-capabilities-absent");
10406        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
10407        let route_handler = handler.clone();
10408        let route_task = tokio::spawn(async move {
10409            route_handler
10410                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
10411                .await
10412                .unwrap()
10413        });
10414        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10415            .await
10416            .unwrap()
10417            .unwrap();
10418        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10419        let ModuleControlRequest::RouteBind {
10420            consumer_capabilities,
10421            ..
10422        } = bind
10423        else {
10424            panic!("expected route.bind request, got {bind:?}");
10425        };
10426        assert_eq!(consumer_capabilities, None);
10427
10428        handler
10429            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10430            .await
10431            .unwrap();
10432        let route_response = route_task.await.unwrap();
10433        assert!(route_response.is_empty());
10434        let published = client_rx.recv().await.unwrap();
10435        assert!(matches!(
10436            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10437            ClientControlResponse::RouteOpen { .. }
10438        ));
10439    }
10440
10441    /// Opens a route to a freshly registered `aft` with `sent` as the
10442    /// route.open's role_versions, acks the bind, and returns the role_versions
10443    /// the module's bind carried.
10444    async fn bind_role_versions_for(
10445        sent: Option<BTreeMap<String, String>>,
10446        connection: u64,
10447    ) -> Option<BTreeMap<String, String>> {
10448        let registry = Arc::new(Registry::default());
10449        let forwarding = Arc::new(ForwardingTable::default());
10450        let handler =
10451            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10452        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
10453        hello_via_sink(
10454            &handler,
10455            &module_ctx,
10456            &mut module_rx,
10457            hello_frame("aft", PROTOCOL_VERSION, 7),
10458        )
10459        .await;
10460        let project_root = unique_project_root("role-versions");
10461        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
10462        let route_handler = handler.clone();
10463        let route_task = tokio::spawn(async move {
10464            route_handler
10465                .handle_control_frame(
10466                    &client_ctx,
10467                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
10468                )
10469                .await
10470                .unwrap()
10471        });
10472        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10473            .await
10474            .expect("a well-formed route.open reaches the module as a bind")
10475            .unwrap();
10476        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10477        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
10478            panic!("expected route.bind request, got {bind:?}");
10479        };
10480        handler
10481            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10482            .await
10483            .unwrap();
10484        assert!(route_task.await.unwrap().is_empty());
10485        let published = client_rx.recv().await.unwrap();
10486        assert!(matches!(
10487            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10488            ClientControlResponse::RouteOpen { .. }
10489        ));
10490        role_versions
10491    }
10492
10493    #[tokio::test]
10494    async fn route_open_relays_role_versions_verbatim() {
10495        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
10496        assert_eq!(
10497            bind_role_versions_for(Some(sent.clone()), 141).await,
10498            Some(sent)
10499        );
10500    }
10501
10502    /// An empty map declares nothing, so the provider sees no field rather
10503    /// than an empty object it would have to treat as a second "none".
10504    #[tokio::test]
10505    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
10506        assert_eq!(bind_role_versions_for(None, 143).await, None);
10507        assert_eq!(
10508            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
10509            None
10510        );
10511    }
10512
10513    #[tokio::test]
10514    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
10515        let registry = Arc::new(Registry::default());
10516        let forwarding = Arc::new(ForwardingTable::default());
10517        let handler =
10518            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10519        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
10520        hello_via_sink(
10521            &handler,
10522            &module_ctx,
10523            &mut module_rx,
10524            hello_frame("aft", PROTOCOL_VERSION, 7),
10525        )
10526        .await;
10527
10528        let nine: BTreeMap<String, String> = (0..9)
10529            .map(|index| (format!("role-{index}"), "v1".to_string()))
10530            .collect();
10531        for (label, malformed) in [
10532            (
10533                "invalid role name",
10534                role_versions(&[("Tool_Provider", "v1")]),
10535            ),
10536            ("invalid version", role_versions(&[("tool-provider", "v0")])),
10537            ("nine entries", nine),
10538        ] {
10539            // A refused open answers at once; one that reached the module
10540            // would wait for its bind ack and trip this timeout.
10541            let responses = tokio::time::timeout(
10542                Duration::from_secs(1),
10543                handler.handle_control_frame(
10544                    &route_ctx(ConnectionId::new(148)).0,
10545                    route_open_frame_with_role_versions(
10546                        404,
10547                        "aft",
10548                        unique_project_root("role-versions-malformed"),
10549                        Some(malformed),
10550                    ),
10551                ),
10552            )
10553            .await
10554            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
10555            .unwrap();
10556            assert_eq!(responses.len(), 1, "{label}");
10557            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
10558            let error = parse_error(&responses[0]);
10559            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
10560            assert_eq!(
10561                error["detail"]["field"], "role_versions",
10562                "{label}: {error}"
10563            );
10564            assert!(
10565                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
10566                "{label}: a malformed declaration is terminal"
10567            );
10568            assert!(
10569                module_rx.try_recv().is_err(),
10570                "{label}: the module must never see a bind"
10571            );
10572        }
10573    }
10574
10575    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
10576    /// consumer can tell this daemon forwards the field from one that would
10577    /// drop it.
10578    #[tokio::test]
10579    async fn route_role_versions_capability_is_advertised() {
10580        let handler = ControlHandler::new(Arc::new(Registry::default()));
10581        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
10582        let ack = hello_via_sink(
10583            &handler,
10584            &ctx,
10585            &mut rx,
10586            hello_frame("m", PROTOCOL_VERSION, 1),
10587        )
10588        .await;
10589        let ack = parse_ack(&ack);
10590        assert!(
10591            ack.subc_capabilities
10592                .iter()
10593                .any(|c| c == "route-role-versions/v1"),
10594            "{:?}",
10595            ack.subc_capabilities
10596        );
10597
10598        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10599        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10600        let reply = handler
10601            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
10602            .await
10603            .unwrap()
10604            .pop()
10605            .unwrap();
10606        let ClientControlResponse::ServerDescribe { capabilities, .. } =
10607            serde_json::from_slice(&reply.body).unwrap()
10608        else {
10609            panic!("not a server.describe reply");
10610        };
10611        assert!(
10612            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
10613            "{capabilities:?}"
10614        );
10615    }
10616
10617    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
10618    /// catalog entry without `events` (the module declares none) from a daemon
10619    /// that never reports declared events.
10620    #[tokio::test]
10621    async fn catalog_events_capability_is_advertised() {
10622        let handler = ControlHandler::new(Arc::new(Registry::default()));
10623        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10624        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10625        let reply = handler
10626            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
10627            .await
10628            .unwrap()
10629            .pop()
10630            .unwrap();
10631        let ClientControlResponse::ServerDescribe { capabilities, .. } =
10632            serde_json::from_slice(&reply.body).unwrap()
10633        else {
10634            panic!("not a server.describe reply");
10635        };
10636        assert!(
10637            capabilities
10638                .iter()
10639                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
10640            "{capabilities:?}"
10641        );
10642    }
10643
10644    #[tokio::test]
10645    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
10646        let registry = Arc::new(Registry::default());
10647        let forwarding = Arc::new(ForwardingTable::default());
10648        let handler =
10649            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10650                .with_health_probe_timeout(Duration::from_secs(5));
10651        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
10652        hello_via_sink(
10653            &handler,
10654            &module_ctx,
10655            &mut module_rx,
10656            non_routable_hello_frame_with_control_ops(
10657                "mcp",
10658                300,
10659                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10660            ),
10661        )
10662        .await;
10663        assert!(registry
10664            .get_module("mcp")
10665            .unwrap()
10666            .unwrap()
10667            .manifest
10668            .provides
10669            .is_empty());
10670
10671        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
10672        let route_response = handler
10673            .handle_control_frame(
10674                &route_client_ctx,
10675                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
10676            )
10677            .await
10678            .unwrap();
10679        assert_eq!(route_response[0].header.ty, FrameType::Error);
10680        assert_eq!(
10681            parse_error(&route_response[0])["code"],
10682            "target_unavailable"
10683        );
10684        assert!(parse_error(&route_response[0])["message"]
10685            .as_str()
10686            .unwrap()
10687            .contains("does not provide the requested target"));
10688        assert!(module_rx.try_recv().is_err());
10689
10690        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
10691        let health_handler = handler.clone();
10692        let health_task = tokio::spawn(async move {
10693            health_handler
10694                .handle_control_frame(
10695                    &health_client_ctx,
10696                    supervisor_health_probe_frame(302, "mcp"),
10697                )
10698                .await
10699                .unwrap()
10700        });
10701        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10702            .await
10703            .unwrap()
10704            .unwrap();
10705        assert_eq!(
10706            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10707            ModuleControlRequest::HealthCheck {}
10708        );
10709        handler
10710            .handle_control_frame(
10711                &module_ctx,
10712                health_response(health_frame.header.corr, HealthStatus::Ok),
10713            )
10714            .await
10715            .unwrap();
10716        let health_response = health_task.await.unwrap();
10717        assert_eq!(health_response[0].header.ty, FrameType::Response);
10718        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10719            ClientControlResponse::SupervisorHealthProbe {
10720                module_id, status, ..
10721            } => {
10722                assert_eq!(module_id, "mcp");
10723                assert_eq!(status, HealthStatus::Ok);
10724            }
10725            other => panic!("unexpected health response: {other:?}"),
10726        }
10727
10728        // Exercise the forwarding cleanup path directly while leaving the registry
10729        // advertisement in place. If cleanup leaves a stale control sink behind,
10730        // the next probe will enqueue onto it and wait for the long probe timeout
10731        // instead of returning an immediate no-connection error.
10732        forwarding
10733            .cleanup_connection(module_ctx.connection_id)
10734            .unwrap();
10735        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
10736        let cleanup_response = tokio::time::timeout(
10737            Duration::from_millis(200),
10738            handler.handle_control_frame(
10739                &cleanup_probe_ctx,
10740                supervisor_health_probe_frame(303, "mcp"),
10741            ),
10742        )
10743        .await
10744        .expect("probe should fail immediately when the control lane is gone")
10745        .unwrap();
10746        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
10747        assert_eq!(
10748            parse_error(&cleanup_response[0])["code"],
10749            "target_unavailable"
10750        );
10751        assert!(parse_error(&cleanup_response[0])["message"]
10752            .as_str()
10753            .unwrap()
10754            .contains("no module connection"));
10755
10756        handler
10757            .cleanup_connection(module_ctx.connection_id)
10758            .unwrap();
10759    }
10760
10761    #[tokio::test]
10762    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
10763        let registry = Arc::new(Registry::default());
10764        let supervisor_handle = SupervisorHandle::new();
10765        let supervisor =
10766            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
10767                .with_handle(supervisor_handle.clone())
10768                .with_connection_file_path(
10769                    std::env::temp_dir()
10770                        .join(format!("subc-route-open-warming-{}", std::process::id())),
10771                );
10772        let module = supervisor
10773            .supervise_configured(
10774                ModuleSpec {
10775                    module_id: "warming".to_string(),
10776                    program: fake_aft_stub_path(),
10777                    args: Vec::new(),
10778                    env: Vec::new(),
10779                    reserved: false,
10780                    reserved_prefixes: Vec::new(),
10781                    protocol: ModuleProtocol::Subc,
10782                    overlap: Default::default(),
10783                },
10784                true,
10785            )
10786            .unwrap();
10787        assert_eq!(module.state().unwrap(), ModuleState::Running);
10788
10789        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
10790        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
10791        let response = handler
10792            .handle_control_frame(
10793                &ctx,
10794                route_open_frame(304, "warming", unique_project_root("warming")),
10795            )
10796            .await
10797            .unwrap();
10798        module.stop().await.unwrap();
10799
10800        assert_eq!(response[0].header.ty, FrameType::Error);
10801        let error = parse_error(&response[0]);
10802        assert_eq!(error["code"], "module_warming");
10803        assert!(error["message"]
10804            .as_str()
10805            .unwrap()
10806            .contains("state=running, enabled=true, live=false"));
10807    }
10808
10809    #[test]
10810    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
10811        let handler = ControlHandler::new(Arc::new(Registry::default()));
10812        let capture = EventCapture::default();
10813        let _subscriber =
10814            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10815        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
10816        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
10817        let pending = (0..limit).collect::<Vec<_>>();
10818        let response = handler
10819            .route_open_capacity_refusal(
10820                &ctx,
10821                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
10822                "busy",
10823                pending.len(),
10824                limit,
10825            )
10826            .unwrap();
10827        assert_eq!(parse_error(&response)["code"], "target_unavailable");
10828        let event = capture
10829            .events()
10830            .into_iter()
10831            .find(|event| {
10832                event.target == "control"
10833                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
10834            })
10835            .expect("connection admission refusal event");
10836        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10837        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10838    }
10839
10840    #[test]
10841    fn route_open_target_cap_logs_admission_reason_and_capacity() {
10842        let handler = ControlHandler::new(Arc::new(Registry::default()));
10843        let capture = EventCapture::default();
10844        let _subscriber =
10845            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10846        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
10847        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
10848        let guards = (0..limit)
10849            .map(|_| {
10850                handler
10851                    .route_bind_concurrency
10852                    .try_admit("busy", limit)
10853                    .unwrap()
10854            })
10855            .collect::<Vec<_>>();
10856        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
10857            Err(in_flight) => in_flight,
10858            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
10859        };
10860        let response = handler
10861            .route_open_target_capacity_refusal(
10862                &ctx,
10863                &route_open_frame(397, "busy", unique_project_root("target-cap")),
10864                "busy",
10865                in_flight,
10866            )
10867            .unwrap();
10868        assert_eq!(parse_error(&response)["code"], "target_unavailable");
10869        let event = capture
10870            .events()
10871            .into_iter()
10872            .find(|event| {
10873                event.target == "control"
10874                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
10875            })
10876            .expect("target admission refusal event");
10877        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10878        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10879        drop(guards);
10880    }
10881
10882    #[tokio::test(flavor = "current_thread")]
10883    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
10884        let handler = ControlHandler::new(Arc::new(Registry::default()));
10885        let capture = EventCapture::default();
10886        let _subscriber =
10887            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10888        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
10889        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10890
10891        let _ = handler
10892            .handle_client_control_request(
10893                &ctx,
10894                frame,
10895                ClientControlRequest::SupervisorSetEnabled {
10896                    module_id: "broca".to_string(),
10897                    enabled: false,
10898                },
10899            )
10900            .await
10901            .unwrap();
10902
10903        let events = capture
10904            .events()
10905            .into_iter()
10906            .filter(|event| {
10907                event.target == "control"
10908                    && event.fields.get("message").map(String::as_str)
10909                        == Some("supervisor request received")
10910            })
10911            .collect::<Vec<_>>();
10912        assert_eq!(events.len(), 1, "one supervisor request log line");
10913        let event = &events[0];
10914        assert_eq!(event.level, tracing::Level::INFO);
10915        assert_eq!(
10916            event.fields.get("op"),
10917            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
10918        );
10919        assert_eq!(
10920            event.fields.get("module_id"),
10921            Some(&"\"broca\"".to_string())
10922        );
10923        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
10924        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
10925        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
10926    }
10927
10928    #[tokio::test(flavor = "current_thread")]
10929    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
10930        let registry = Arc::new(Registry::default());
10931        let module_connection = ConnectionId::new(303);
10932        registry
10933            .register_with_control_ops(
10934                manifest("aft", PROTOCOL_VERSION),
10935                PROTOCOL_VERSION,
10936                module_connection,
10937                module_baseline_control_ops(),
10938            )
10939            .unwrap();
10940        let handler = ControlHandler::new(registry);
10941        let capture = EventCapture::default();
10942        let _subscriber =
10943            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10944        let (ctx, _rx) = route_ctx(module_connection);
10945        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10946
10947        let _ = handler
10948            .handle_client_control_request(
10949                &ctx,
10950                frame,
10951                ClientControlRequest::SupervisorSetEnabled {
10952                    module_id: "broca".to_string(),
10953                    enabled: false,
10954                },
10955            )
10956            .await
10957            .unwrap();
10958
10959        let callers = capture
10960            .events()
10961            .into_iter()
10962            .filter(|event| {
10963                event.fields.get("message").map(String::as_str)
10964                    == Some("supervisor request received")
10965            })
10966            .map(|event| event.fields.get("caller").cloned())
10967            .collect::<Vec<_>>();
10968        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
10969    }
10970
10971    #[tokio::test(flavor = "current_thread")]
10972    async fn supervisor_list_does_not_log_request_received() {
10973        let handler = ControlHandler::new(Arc::new(Registry::default()));
10974        let capture = EventCapture::default();
10975        let _subscriber =
10976            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10977        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
10978        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
10979
10980        handler
10981            .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
10982            .await
10983            .unwrap();
10984
10985        assert!(capture.events().into_iter().all(|event| {
10986            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10987        }));
10988    }
10989
10990    #[tokio::test(flavor = "current_thread")]
10991    async fn supervisor_rescan_preview_does_not_log_request_received() {
10992        let handler = ControlHandler::new(Arc::new(Registry::default()));
10993        let capture = EventCapture::default();
10994        let _subscriber =
10995            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10996        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
10997        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
10998
10999        handler
11000            .handle_client_control_request(
11001                &ctx,
11002                frame,
11003                ClientControlRequest::SupervisorRescan { preview: true },
11004            )
11005            .await
11006            .unwrap();
11007
11008        assert!(capture.events().into_iter().all(|event| {
11009            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11010        }));
11011    }
11012
11013    /// One wire code has several senders, so the refusal line names the check
11014    /// that refused. This drives the shared refusal path for ordinary refusals
11015    /// with an unregistered
11016    /// target and requires the branch label on the event.
11017    #[tokio::test]
11018    async fn route_open_refusal_names_the_check_that_refused() {
11019        let handler = ControlHandler::new(Arc::new(Registry::default()));
11020        let capture = EventCapture::default();
11021        let _subscriber =
11022            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11023        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11024        let response = handler
11025            .handle_control_frame(
11026                &ctx,
11027                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
11028            )
11029            .await
11030            .unwrap();
11031
11032        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11033        let event = capture
11034            .events()
11035            .into_iter()
11036            .find(|event| {
11037                event.target == "control"
11038                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11039            })
11040            .expect("route.open refusal event");
11041        assert_eq!(
11042            event.fields.get("reason"),
11043            Some(&"\"not_registered\"".to_string())
11044        );
11045    }
11046
11047    #[tokio::test]
11048    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
11049        let registry = Arc::new(Registry::default());
11050        let supervisor_handle = SupervisorHandle::new();
11051        let supervisor =
11052            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
11053                .with_handle(supervisor_handle.clone())
11054                .with_connection_file_path(std::env::temp_dir().join(format!(
11055                    "subc-route-open-refusal-info-{}",
11056                    std::process::id()
11057                )));
11058        let module = supervisor
11059            .supervise_configured(
11060                ModuleSpec {
11061                    module_id: "warming".to_string(),
11062                    program: fake_aft_stub_path(),
11063                    args: Vec::new(),
11064                    env: Vec::new(),
11065                    reserved: false,
11066                    reserved_prefixes: Vec::new(),
11067                    protocol: ModuleProtocol::Subc,
11068                    overlap: Default::default(),
11069                },
11070                true,
11071            )
11072            .unwrap();
11073        assert_eq!(module.state().unwrap(), ModuleState::Running);
11074
11075        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11076        assert!(handler
11077            .counters()
11078            .snapshot()
11079            .get("route_open_refused_by_code")
11080            .is_none());
11081        let capture = EventCapture::default();
11082        let _subscriber =
11083            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11084        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
11085        let response = handler
11086            .handle_control_frame(
11087                &ctx,
11088                route_open_frame(394, "warming", unique_project_root("refusal-info")),
11089            )
11090            .await
11091            .unwrap();
11092        module.stop().await.unwrap();
11093
11094        assert_eq!(parse_error(&response[0])["code"], "module_warming");
11095        let event = capture
11096            .events()
11097            .into_iter()
11098            .find(|event| {
11099                event.target == "control"
11100                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
11101            })
11102            .expect("route.open refusal event");
11103        assert_eq!(
11104            event.fields.get("module_id"),
11105            Some(&"\"warming\"".to_string())
11106        );
11107        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
11108        assert_eq!(
11109            event.fields.get("reason"),
11110            Some(&"\"supervised_not_registered\"".to_string())
11111        );
11112        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
11113        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
11114        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
11115        assert_eq!(
11116            handler.counters().snapshot()["route_open_refused_by_code"],
11117            json!({ "module_warming": 1 })
11118        );
11119    }
11120
11121    const OUTAGE_START: &str = "route.open refusing module: not serving";
11122    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
11123
11124    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
11125        capture
11126            .events()
11127            .into_iter()
11128            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
11129            .collect()
11130    }
11131
11132    fn supervise_stub(
11133        registry: &Arc<Registry>,
11134        module_id: &str,
11135        enabled: bool,
11136    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
11137        let supervisor_handle = SupervisorHandle::new();
11138        let supervisor =
11139            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
11140                .with_handle(supervisor_handle.clone())
11141                .with_connection_file_path(std::env::temp_dir().join(format!(
11142                    "subc-route-outage-{module_id}-{}",
11143                    std::process::id()
11144                )));
11145        let module = supervisor
11146            .supervise_configured(
11147                ModuleSpec {
11148                    module_id: module_id.to_string(),
11149                    program: fake_aft_stub_path(),
11150                    args: Vec::new(),
11151                    env: Vec::new(),
11152                    reserved: false,
11153                    reserved_prefixes: Vec::new(),
11154                    protocol: ModuleProtocol::Subc,
11155                    overlap: Default::default(),
11156                },
11157                enabled,
11158            )
11159            .unwrap();
11160        (supervisor_handle, module)
11161    }
11162
11163    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
11164        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
11165            module_id: module_id.to_string(),
11166            drain_timeout_ms: Some(50),
11167        })
11168        .unwrap();
11169        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11170    }
11171
11172    /// Two handlers built over one forwarding table must share one outage
11173    /// tracker; separate trackers would each log their own opening line for
11174    /// the same outage.
11175    #[test]
11176    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
11177        let registry = Arc::new(Registry::default());
11178        let forwarding = Arc::new(ForwardingTable::default());
11179        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11180        let second = ControlHandler::with_forwarding(registry, forwarding);
11181        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
11182    }
11183
11184    /// A client can name any module id it likes. Refusing an unknown one,
11185    /// however often, must not create outage state or outage lines, or the
11186    /// tracker would be a memory sink any client could fill.
11187    #[tokio::test(flavor = "current_thread")]
11188    async fn route_open_unknown_module_refusals_add_no_outage_state() {
11189        let handler = ControlHandler::new(Arc::new(Registry::default()));
11190        let capture = EventCapture::default();
11191        let _subscriber =
11192            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11193        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
11194        for corr in 0..8 {
11195            let response = handler
11196                .handle_control_frame(
11197                    &ctx,
11198                    route_open_frame(
11199                        380 + corr,
11200                        &format!("nobody-{corr}"),
11201                        unique_project_root("outage-unknown"),
11202                    ),
11203                )
11204                .await
11205                .unwrap();
11206            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11207        }
11208
11209        assert_eq!(handler.route_outages.tracked_module_count(), 0);
11210        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
11211        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
11212    }
11213
11214    /// Drives the refusal path end to end: a supervised module that served
11215    /// before and stopped being registered with no instruction to stop is a
11216    /// WARN, and the same module refused after an operator `supervisor.restart`
11217    /// is an INFO.
11218    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11219    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
11220        let registry = Arc::new(Registry::default());
11221        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
11222        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11223        let capture = EventCapture::default();
11224        let _subscriber =
11225            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11226        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
11227        // The stub never registers, so pretend it served once: otherwise every
11228        // refusal would fall in its startup window.
11229        handler.route_outages.record_accepted("outage-restart");
11230
11231        let response = handler
11232            .handle_control_frame(
11233                &ctx,
11234                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
11235            )
11236            .await
11237            .unwrap();
11238        assert_eq!(response[0].header.ty, FrameType::Error);
11239        let starts = outage_lines(&capture, OUTAGE_START);
11240        assert_eq!(starts.len(), 1, "{starts:?}");
11241        assert_eq!(starts[0].level, tracing::Level::WARN);
11242        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
11243        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
11244        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
11245        handler.route_outages.record_accepted("outage-restart");
11246        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
11247
11248        let restart = handler
11249            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
11250            .await
11251            .unwrap();
11252        assert_eq!(
11253            restart[0].header.ty,
11254            FrameType::Response,
11255            "{:?}",
11256            parse_error(&restart[0])
11257        );
11258        handler
11259            .handle_control_frame(
11260                &ctx,
11261                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
11262            )
11263            .await
11264            .unwrap();
11265        module.stop().await.unwrap();
11266
11267        let starts = outage_lines(&capture, OUTAGE_START);
11268        assert_eq!(starts.len(), 2, "{starts:?}");
11269        assert_eq!(starts[1].level, tracing::Level::INFO);
11270        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
11271    }
11272
11273    /// A restart refused before it touched the module (here: the module is
11274    /// disabled) must clear its operator mark, so the next real outage is
11275    /// still reported as a warning.
11276    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11277    async fn failed_operator_restart_leaves_no_operator_mark() {
11278        let registry = Arc::new(Registry::default());
11279        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
11280        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11281        let capture = EventCapture::default();
11282        let _subscriber =
11283            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11284        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
11285        handler.route_outages.record_accepted("outage-disabled");
11286
11287        let restart = handler
11288            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
11289            .await
11290            .unwrap();
11291        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
11292        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
11293
11294        handler
11295            .handle_control_frame(
11296                &ctx,
11297                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
11298            )
11299            .await
11300            .unwrap();
11301        let starts = outage_lines(&capture, OUTAGE_START);
11302        assert_eq!(starts.len(), 1, "{starts:?}");
11303        assert_eq!(starts[0].level, tracing::Level::WARN);
11304    }
11305
11306    #[tokio::test(flavor = "current_thread")]
11307    async fn route_open_unknown_module_escapes_target_module_id() {
11308        let handler = ControlHandler::new(Arc::new(Registry::default()));
11309        let capture = EventCapture::default();
11310        let _subscriber =
11311            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11312        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
11313        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11314        let response = handler
11315            .handle_control_frame(
11316                &ctx,
11317                route_open_frame(
11318                    395,
11319                    hostile_module_id,
11320                    unique_project_root("hostile-target-module-id"),
11321                ),
11322            )
11323            .await
11324            .unwrap();
11325
11326        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11327        let event = capture
11328            .events()
11329            .into_iter()
11330            .find(|event| {
11331                event.target == "control"
11332                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11333            })
11334            .expect("route.open unknown-module refusal event");
11335        let logged = event.fields.get("module_id").expect("module_id field");
11336        assert!(!logged.bytes().any(|byte| byte < 0x20));
11337        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11338    }
11339
11340    #[tokio::test(flavor = "current_thread")]
11341    async fn route_open_module_rejection_uses_daemon_counter_key() {
11342        let registry = Arc::new(Registry::default());
11343        let forwarding = Arc::new(ForwardingTable::default());
11344        let handler =
11345            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11346        let module_connection = ConnectionId::new(95);
11347        let (module_ctx, mut module_rx) = route_ctx(module_connection);
11348        hello_via_sink(
11349            &handler,
11350            &module_ctx,
11351            &mut module_rx,
11352            hello_frame("aft", PROTOCOL_VERSION, 395),
11353        )
11354        .await;
11355
11356        let client_connection = ConnectionId::new(96);
11357        let (client_ctx, _client_rx) = route_ctx(client_connection);
11358        let capture = EventCapture::default();
11359        let _subscriber =
11360            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11361        let (route_task, bind) = relay_route_open(
11362            &handler,
11363            client_connection,
11364            &client_ctx.egress,
11365            &mut module_rx,
11366            396,
11367            "aft",
11368            "hostile-module-code",
11369        )
11370        .await;
11371        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
11372        let rejection = Frame::build(
11373            FrameType::Error,
11374            control_flags(),
11375            0,
11376            0,
11377            bind.header.corr,
11378            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
11379        )
11380        .unwrap();
11381        handler
11382            .handle_control_frame(&module_ctx, rejection)
11383            .await
11384            .unwrap();
11385
11386        let response = route_task.await.unwrap();
11387        assert_eq!(parse_error(&response[0])["code"], hostile_code);
11388        let counters = handler.counters().snapshot();
11389        assert_eq!(
11390            counters["route_open_refused_by_code"],
11391            json!({ "module_rejected": 1 })
11392        );
11393        assert!(counters["route_open_refused_by_code"]
11394            .get(hostile_code)
11395            .is_none());
11396
11397        let event = capture
11398            .events()
11399            .into_iter()
11400            .find(|event| {
11401                event.target == "control"
11402                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
11403            })
11404            .expect("route.open module-rejection refusal event");
11405        let logged = event.fields.get("module_code").expect("module_code field");
11406        assert!(!logged.bytes().any(|byte| byte < 0x20));
11407        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11408    }
11409
11410    #[tokio::test]
11411    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
11412        let registry = Arc::new(Registry::default());
11413        let supervisor_handle = SupervisorHandle::new();
11414        let missing_program = std::env::temp_dir().join(format!(
11415            "subc-route-open-missing-program-{}",
11416            std::process::id()
11417        ));
11418        let supervisor =
11419            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
11420                .with_handle(supervisor_handle.clone());
11421        let module = supervisor
11422            .supervise_configured(
11423                ModuleSpec {
11424                    module_id: "failed".to_string(),
11425                    program: missing_program,
11426                    args: Vec::new(),
11427                    env: Vec::new(),
11428                    reserved: false,
11429                    reserved_prefixes: Vec::new(),
11430                    protocol: ModuleProtocol::Subc,
11431                    overlap: Default::default(),
11432                },
11433                true,
11434            )
11435            .unwrap();
11436        assert_eq!(module.state().unwrap(), ModuleState::Failed);
11437
11438        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11439        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
11440        let response = handler
11441            .handle_control_frame(
11442                &ctx,
11443                route_open_frame(305, "failed", unique_project_root("failed")),
11444            )
11445            .await
11446            .unwrap();
11447
11448        assert_eq!(response[0].header.ty, FrameType::Error);
11449        let error = parse_error(&response[0]);
11450        assert_eq!(error["code"], "target_unavailable");
11451        assert!(error["message"]
11452            .as_str()
11453            .unwrap()
11454            .contains("state=failed, enabled=true, live=false"));
11455    }
11456
11457    #[tokio::test]
11458    async fn route_open_role_mismatch_remains_target_unavailable() {
11459        let registry = Arc::new(Registry::default());
11460        let handler = ControlHandler::new(Arc::clone(&registry));
11461        handler
11462            .handle_control(
11463                ConnectionId::new(41),
11464                non_routable_hello_frame_with_control_ops("health-only", 306, None),
11465            )
11466            .unwrap();
11467
11468        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
11469        let response = handler
11470            .handle_control_frame(
11471                &ctx,
11472                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
11473            )
11474            .await
11475            .unwrap();
11476
11477        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11478        assert!(parse_error(&response[0])["message"]
11479            .as_str()
11480            .unwrap()
11481            .contains("does not provide the requested target"));
11482    }
11483
11484    #[tokio::test]
11485    async fn route_open_inactive_registration_remains_target_unavailable() {
11486        let registry = Arc::new(Registry::default());
11487        let handler = ControlHandler::new(Arc::clone(&registry));
11488        handler
11489            .handle_control(
11490                ConnectionId::new(43),
11491                hello_frame("inactive", PROTOCOL_VERSION, 308),
11492            )
11493            .unwrap();
11494        assert!(registry
11495            .set_module_state_for_test("inactive", ChannelState::Closed)
11496            .unwrap());
11497
11498        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
11499        let response = handler
11500            .handle_control_frame(
11501                &ctx,
11502                route_open_frame(309, "inactive", unique_project_root("inactive")),
11503            )
11504            .await
11505            .unwrap();
11506
11507        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11508        assert!(parse_error(&response[0])["message"]
11509            .as_str()
11510            .unwrap()
11511            .contains("is not active"));
11512    }
11513
11514    #[tokio::test]
11515    async fn late_health_reply_is_recorded_through_the_module_response_path() {
11516        let registry = Arc::new(Registry::default());
11517        let forwarding = Arc::new(ForwardingTable::default());
11518        let supervisor_handle = SupervisorHandle::new();
11519        let supervisor =
11520            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
11521                .with_forwarding(Arc::clone(&forwarding))
11522                .with_handle(supervisor_handle.clone());
11523        let module = supervisor
11524            .supervise_configured(
11525                crate::ModuleSpec {
11526                    module_id: "late-health-response".to_string(),
11527                    program: PathBuf::from("disabled-module"),
11528                    args: Vec::new(),
11529                    env: Vec::new(),
11530                    reserved: false,
11531                    reserved_prefixes: Vec::new(),
11532                    protocol: ModuleProtocol::Subc,
11533                    overlap: Default::default(),
11534                },
11535                false,
11536            )
11537            .unwrap();
11538        let handler =
11539            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11540                .with_supervisor(supervisor_handle);
11541        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
11542        handler
11543            .handle_control_frame(
11544                &module_ctx,
11545                hello_frame_with_control_ops(
11546                    "late-health-response",
11547                    PROTOCOL_VERSION,
11548                    7,
11549                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11550                ),
11551            )
11552            .await
11553            .unwrap();
11554        let probe_started_at = Instant::now() - Duration::from_millis(80);
11555        let pending = forwarding
11556            .begin_health_probe_rpc_for(
11557                "late-health-response",
11558                MODULE_CONTROL_OP_HEALTH_CHECK,
11559                probe_started_at,
11560                Instant::now() - Duration::from_millis(1),
11561            )
11562            .unwrap();
11563        assert!(forwarding
11564            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
11565            .unwrap());
11566
11567        let responses = handler
11568            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
11569            .await
11570            .unwrap();
11571
11572        assert!(responses.is_empty());
11573        let health = module.status().unwrap().health;
11574        assert_eq!(health.late_answer_count, 1);
11575        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
11576    }
11577
11578    #[tokio::test]
11579    async fn health_probe_timeout_and_module_death_are_typed() {
11580        let registry = Arc::new(Registry::default());
11581        let forwarding = Arc::new(ForwardingTable::default());
11582        let handler =
11583            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11584                .with_health_probe_timeout(Duration::from_millis(50));
11585        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11586        hello_via_sink(
11587            &handler,
11588            &module_ctx,
11589            &mut module_rx,
11590            hello_frame_with_control_ops(
11591                "aft",
11592                PROTOCOL_VERSION,
11593                7,
11594                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11595            ),
11596        )
11597        .await;
11598
11599        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11600        let responses = handler
11601            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
11602            .await
11603            .unwrap();
11604        assert_eq!(responses[0].header.ty, FrameType::Error);
11605        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
11606        let _ = module_rx.try_recv();
11607
11608        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
11609        let health_handler = handler.clone();
11610        let death_task = tokio::spawn(async move {
11611            health_handler
11612                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
11613                .await
11614                .unwrap()
11615        });
11616        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11617            .await
11618            .unwrap()
11619            .unwrap();
11620        handler
11621            .cleanup_connection(module_ctx.connection_id)
11622            .unwrap();
11623        let responses = death_task.await.unwrap();
11624        assert_eq!(responses[0].header.ty, FrameType::Error);
11625        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
11626    }
11627
11628    #[test]
11629    fn hello_requires_exact_protocol_version() {
11630        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
11631            let registry = Arc::new(Registry::default());
11632            let handler = ControlHandler::new(Arc::clone(&registry));
11633            let responses = handler
11634                .handle_control(
11635                    ConnectionId::new(connection),
11636                    hello_frame("aft", offered, 9),
11637                )
11638                .unwrap();
11639
11640            assert_eq!(responses.len(), 1);
11641            assert_eq!(responses[0].header.ty, FrameType::Error);
11642            let error = parse_error(&responses[0]);
11643            assert_eq!(error["code"], "version_unsupported");
11644            assert!(registry.get_module("aft").unwrap().is_none());
11645            assert_eq!(registry.active_registration_count().unwrap(), 0);
11646        }
11647    }
11648
11649    #[test]
11650    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
11651        let registry = Arc::new(Registry::default());
11652        let forwarding = Arc::new(ForwardingTable::default());
11653        let handler =
11654            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11655        let module_connection = ConnectionId::new(301);
11656        let registration = registry
11657            .register_with_control_ops(
11658                manifest("aft-push", PROTOCOL_VERSION),
11659                PROTOCOL_VERSION,
11660                module_connection,
11661                module_baseline_control_ops(),
11662            )
11663            .unwrap();
11664        let (module_tx, _module_rx) = mpsc::channel(8);
11665        let endpoint = forwarding
11666            .register_module_connection(
11667                module_connection,
11668                "aft-push".to_string(),
11669                PROTOCOL_VERSION,
11670                manifest_concurrency(&registration.manifest),
11671                FrameSink::new(module_tx),
11672            )
11673            .unwrap();
11674
11675        // A push op this version does not know is ignored (forward-compat), not errored.
11676        let unknown = Frame::build(
11677            FrameType::Push,
11678            control_flags(),
11679            0,
11680            0,
11681            5,
11682            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
11683        )
11684        .unwrap();
11685        let out = handler.handle_status_update(endpoint, unknown).unwrap();
11686        assert!(
11687            out.is_empty(),
11688            "unknown push op must be ignored, got {out:?}"
11689        );
11690
11691        // A malformed body for a KNOWN op is a real error worth surfacing.
11692        let malformed = Frame::build(
11693            FrameType::Push,
11694            control_flags(),
11695            0,
11696            0,
11697            6,
11698            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
11699        )
11700        .unwrap();
11701        let out = handler.handle_status_update(endpoint, malformed).unwrap();
11702        assert_eq!(out.len(), 1);
11703        assert_eq!(out[0].header.ty, FrameType::Error);
11704        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
11705    }
11706
11707    #[test]
11708    fn hello_rejected_when_connection_already_owns_client_routes() {
11709        let registry = Arc::new(Registry::default());
11710        let forwarding = Arc::new(ForwardingTable::default());
11711        let handler =
11712            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11713        // Commits a client route on connection 202 (bound to a module on conn 101).
11714        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
11715        let client_connection = ConnectionId::new(202);
11716
11717        // That same connection now tries to register as a module: rejected, so one
11718        // connection never holds both client-route and module-endpoint state.
11719        let responses = handler
11720            .handle_control(
11721                client_connection,
11722                hello_frame("aft-second", PROTOCOL_VERSION, 9),
11723            )
11724            .unwrap();
11725        assert_eq!(responses[0].header.ty, FrameType::Error);
11726        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
11727        assert!(registry.get_module("aft-second").unwrap().is_none());
11728    }
11729
11730    #[tokio::test]
11731    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
11732        let registry = Arc::new(Registry::default());
11733        let forwarding = Arc::new(ForwardingTable::default());
11734        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
11735        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
11736        hello_via_sink(
11737            &handler,
11738            &module_ctx,
11739            &mut module_rx,
11740            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
11741        )
11742        .await;
11743        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
11744        let pending = forwarding
11745            .begin_route_bind_relay_for_test(
11746                client_ctx.connection_id,
11747                client_ctx.egress.clone(),
11748                2,
11749                "alpha",
11750            )
11751            .unwrap();
11752        forwarding
11753            .complete_pending_relay(
11754                module_ctx.connection_id,
11755                pending.corr,
11756                RouteBindRelayOutcome::Accepted,
11757            )
11758            .unwrap();
11759        client_rx.try_recv().unwrap();
11760        for module_id in ["beta", "alpha"] {
11761            let replies = handler
11762                .handle_control_frame(
11763                    &module_ctx,
11764                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
11765                )
11766                .await
11767                .unwrap();
11768            assert_eq!(replies.len(), 1, "second HELLO must be refused");
11769            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
11770        }
11771        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
11772        assert!(registry.get_module("beta").unwrap().is_none());
11773        assert!(matches!(
11774            forwarding
11775                .lookup_data_route(
11776                    client_ctx.connection_id,
11777                    pending.client_channel,
11778                    pending.client_epoch,
11779                )
11780                .unwrap(),
11781            DataRoute::Client(DataRouteState::Bound(_))
11782        ));
11783        assert!(handler
11784            .hello_launch_nonces
11785            .lock()
11786            .unwrap()
11787            .presented(module_ctx.connection_id, Some("alpha-nonce")));
11788        assert!(module_rx.try_recv().is_err());
11789    }
11790
11791    #[test]
11792    fn reserved_module_hello_requires_matching_launch_nonce() {
11793        let registry = Arc::new(Registry::default());
11794        let supervisor = SupervisorHandle::new();
11795        // The supervisor recorded the nonce it injected when it spawned the reserved
11796        // module; the HELLO verifier checks against the same shared handle.
11797        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
11798        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
11799
11800        // A HELLO with NO nonce is rejected.
11801        let no_nonce = handler
11802            .handle_control(
11803                ConnectionId::new(1),
11804                hello_frame("vault", PROTOCOL_VERSION, 1),
11805            )
11806            .unwrap();
11807        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
11808        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
11809        assert!(registry.get_module("vault").unwrap().is_none());
11810
11811        // A HELLO with the WRONG nonce is rejected.
11812        let wrong = handler
11813            .handle_control(
11814                ConnectionId::new(2),
11815                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
11816            )
11817            .unwrap();
11818        assert_eq!(wrong[0].header.ty, FrameType::Error);
11819        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
11820        assert!(registry.get_module("vault").unwrap().is_none());
11821
11822        // A HELLO with the CORRECT nonce registers.
11823        let ok = handler
11824            .handle_control(
11825                ConnectionId::new(3),
11826                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
11827            )
11828            .unwrap();
11829        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
11830        assert!(registry.get_module("vault").unwrap().is_some());
11831    }
11832
11833    #[test]
11834    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
11835        let registry = Arc::new(Registry::default());
11836        let supervisor = SupervisorHandle::new();
11837        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11838        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11839        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
11840
11841        let squat = handler
11842            .handle_control(
11843                ConnectionId::new(1),
11844                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
11845            )
11846            .unwrap();
11847        assert_eq!(squat[0].header.ty, FrameType::Error);
11848        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
11849        assert!(parse_error(&squat[0])["message"]
11850            .as_str()
11851            .unwrap()
11852            .contains("fed:"));
11853
11854        let accepted_peer = handler
11855            .handle_control(
11856                ConnectionId::new(2),
11857                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
11858            )
11859            .unwrap();
11860        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
11861
11862        let accepted_short = handler
11863            .handle_control(
11864                ConnectionId::new(3),
11865                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
11866            )
11867            .unwrap();
11868        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
11869
11870        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
11871            let response = handler
11872                .handle_control(
11873                    ConnectionId::new(conn),
11874                    hello_frame(module_id, PROTOCOL_VERSION, conn),
11875                )
11876                .unwrap();
11877            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
11878        }
11879    }
11880
11881    #[test]
11882    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
11883        let registry = Arc::new(Registry::default());
11884        let supervisor = SupervisorHandle::new();
11885        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11886        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11887        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
11888        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
11889
11890        let owner_nonce = handler
11891            .handle_control(
11892                ConnectionId::new(1),
11893                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
11894            )
11895            .unwrap();
11896        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
11897        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
11898        assert!(registry.get_module("fed:special").unwrap().is_none());
11899
11900        let exact_nonce = handler
11901            .handle_control(
11902                ConnectionId::new(2),
11903                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
11904            )
11905            .unwrap();
11906        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
11907        assert!(registry.get_module("fed:special").unwrap().is_some());
11908    }
11909
11910    #[test]
11911    fn non_reserved_module_ignores_launch_nonce() {
11912        let registry = Arc::new(Registry::default());
11913        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
11914        // registration succeeds whether a spawned process echoes a nonce or not.
11915        let handler = ControlHandler::new(Arc::clone(&registry));
11916        let no_nonce = handler
11917            .handle_control(
11918                ConnectionId::new(1),
11919                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
11920            )
11921            .unwrap();
11922        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
11923        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
11924
11925        let echoed_nonce = handler
11926            .handle_control(
11927                ConnectionId::new(2),
11928                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
11929            )
11930            .unwrap();
11931        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
11932        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
11933    }
11934
11935    #[test]
11936    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
11937        let handler = ControlHandler::default();
11938        let conn = ConnectionId::new(1);
11939        let malformed = Frame::build(
11940            FrameType::Hello,
11941            control_flags(),
11942            0,
11943            0,
11944            3,
11945            b"{not json".to_vec(),
11946        )
11947        .unwrap();
11948
11949        let error = handler.handle_control(conn, malformed).unwrap();
11950        assert_eq!(error[0].header.ty, FrameType::Error);
11951        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
11952
11953        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11954        let pong = handler.handle_control(conn, ping).unwrap();
11955        assert_eq!(pong[0].header.ty, FrameType::Pong);
11956        assert_eq!(pong[0].header.corr, 4);
11957    }
11958
11959    #[test]
11960    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
11961        let registry = Arc::new(Registry::default());
11962        let handler = ControlHandler::new(Arc::clone(&registry));
11963
11964        handler
11965            .handle_control(
11966                ConnectionId::new(1),
11967                hello_frame("aft", PROTOCOL_VERSION, 1),
11968            )
11969            .unwrap();
11970        let duplicate = handler
11971            .handle_control(
11972                ConnectionId::new(2),
11973                hello_frame("aft", PROTOCOL_VERSION, 2),
11974            )
11975            .unwrap();
11976
11977        assert_eq!(duplicate[0].header.ty, FrameType::Error);
11978        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11979        let registration = registry.get_module("aft").unwrap().unwrap();
11980        assert_eq!(registration.connection_id, ConnectionId::new(1));
11981    }
11982
11983    #[test]
11984    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
11985        let registry = Arc::new(Registry::default());
11986        let forwarding = Arc::new(ForwardingTable::default());
11987        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
11988        let handler =
11989            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11990                .with_process_liveness(process_liveness);
11991        let (ctx, route_channel, route_epoch) =
11992            bind_liveness_route(&registry, &forwarding, "aft-dead");
11993        let responses = handler
11994            .handle_route_poll(
11995                &ctx,
11996                route_poll_frame(41, PollKind::Liveness, route_channel),
11997                route_channel,
11998                route_epoch,
11999                PollKind::Liveness,
12000            )
12001            .unwrap();
12002
12003        assert_eq!(responses.len(), 1);
12004        assert_eq!(responses[0].header.ty, FrameType::Response);
12005        assert_route_poll_liveness(&responses[0], false);
12006    }
12007
12008    #[test]
12009    fn liveness_poll_without_process_source_uses_bound_route() {
12010        let registry = Arc::new(Registry::default());
12011        let forwarding = Arc::new(ForwardingTable::default());
12012        let handler =
12013            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
12014        let (ctx, route_channel, route_epoch) =
12015            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
12016        let responses = handler
12017            .handle_route_poll(
12018                &ctx,
12019                route_poll_frame(42, PollKind::Liveness, route_channel),
12020                route_channel,
12021                route_epoch,
12022                PollKind::Liveness,
12023            )
12024            .unwrap();
12025
12026        assert_route_poll_liveness(&responses[0], true);
12027    }
12028
12029    #[test]
12030    fn liveness_poll_untracked_process_source_uses_bound_route() {
12031        let registry = Arc::new(Registry::default());
12032        let forwarding = Arc::new(ForwardingTable::default());
12033        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
12034        let handler =
12035            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12036                .with_process_liveness(process_liveness);
12037        let (ctx, route_channel, route_epoch) =
12038            bind_liveness_route(&registry, &forwarding, "aft-untracked");
12039        let responses = handler
12040            .handle_route_poll(
12041                &ctx,
12042                route_poll_frame(43, PollKind::Liveness, route_channel),
12043                route_channel,
12044                route_epoch,
12045                PollKind::Liveness,
12046            )
12047            .unwrap();
12048
12049        assert_route_poll_liveness(&responses[0], true);
12050    }
12051
12052    #[tokio::test]
12053    async fn unknown_op_returns_unknown_control_op() {
12054        let handler = ControlHandler::default();
12055        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
12056        let request = Frame::build(
12057            FrameType::Request,
12058            control_flags(),
12059            0,
12060            0,
12061            55,
12062            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
12063        )
12064        .unwrap();
12065
12066        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12067
12068        assert_eq!(response.len(), 1);
12069        assert_eq!(response[0].header.ty, FrameType::Error);
12070        assert_eq!(response[0].header.corr, 55);
12071        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
12072    }
12073
12074    #[tokio::test]
12075    async fn supervisor_provenance_rejects_unknown_exact_module() {
12076        let handler = ControlHandler::default();
12077        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
12078        let request = Frame::build(
12079            FrameType::Request,
12080            control_flags(),
12081            0,
12082            0,
12083            57,
12084            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
12085        )
12086        .unwrap();
12087
12088        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12089
12090        assert_eq!(response.len(), 1);
12091        assert_eq!(response[0].header.ty, FrameType::Error);
12092        assert_eq!(response[0].header.corr, 57);
12093        let error = parse_error(&response[0]);
12094        assert_eq!(error["code"], "unknown_module");
12095        assert_eq!(error["message"], "module_id 'missing' is not supervised");
12096    }
12097
12098    #[test]
12099    fn provenance_probe_override_keeps_handler_tests_deterministic() {
12100        let expected = subc_control::RunningImageAgreement::Unavailable {
12101            reason: subc_control::RunningImageUnavailableReason::HashFailed,
12102        };
12103        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
12104        assert_eq!(handler.provenance_probe_override, Some(expected));
12105    }
12106
12107    #[test]
12108    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
12109        let verdict = reload_verdict(
12110            std::path::Path::new("/bin/new"),
12111            Some(std::path::Path::new("/bin/old")),
12112            subc_control::RunningImageAgreement::Unavailable {
12113                reason: subc_control::RunningImageUnavailableReason::HashFailed,
12114            },
12115        );
12116        assert!(matches!(
12117            verdict.path,
12118            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
12119                if configured == std::path::Path::new("/bin/new")
12120                    && spawned_from == std::path::Path::new("/bin/old")
12121        ));
12122    }
12123
12124    #[cfg(windows)]
12125    #[test]
12126    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
12127        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
12128        let configured = dir.join("module.exe");
12129        std::fs::write(&configured, b"executable path fixture").unwrap();
12130        let spawned = std::fs::canonicalize(&configured).unwrap();
12131        assert_ne!(
12132            configured, spawned,
12133            "fixture must exercise different path spellings"
12134        );
12135        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
12136        let lower_case = configured.to_string_lossy().to_lowercase();
12137        for spelling in [
12138            &configured,
12139            Path::new(&forward_slashes),
12140            Path::new(&lower_case),
12141        ] {
12142            let verdict = reload_verdict(
12143                spelling,
12144                Some(&spawned),
12145                subc_control::RunningImageAgreement::Unavailable {
12146                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12147                },
12148            );
12149            assert_eq!(
12150                verdict.path,
12151                ReloadPathAgreement::Match,
12152                "configured {spelling:?}, spawned {spawned:?}"
12153            );
12154        }
12155    }
12156
12157    #[cfg(windows)]
12158    #[test]
12159    fn windows_reload_path_keeps_different_existing_program_pending() {
12160        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
12161        let old = dir.join("old.exe");
12162        let new = dir.join("new.exe");
12163        std::fs::write(&old, b"same executable content").unwrap();
12164        std::fs::copy(&old, &new).unwrap();
12165        let spawned = std::fs::canonicalize(&old).unwrap();
12166        let image = subc_control::RunningImageAgreement::Unavailable {
12167            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12168        };
12169        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
12170        assert!(
12171            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
12172            if configured == new && spawned_from == spawned)
12173        );
12174        let missing = dir.join("missing.exe");
12175        assert!(matches!(
12176            reload_verdict(&missing, Some(&spawned), image).path,
12177            ReloadPathAgreement::Mismatch { .. }
12178        ));
12179    }
12180
12181    #[test]
12182    fn reload_verdict_detects_replaced_image_at_same_path() {
12183        let image = subc_control::RunningImageAgreement::Mismatch {
12184            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
12185                digest: "old".into(),
12186            },
12187            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
12188                digest: "new".into(),
12189            },
12190        };
12191        let verdict = reload_verdict(
12192            std::path::Path::new("/bin/same"),
12193            Some(std::path::Path::new("/bin/same")),
12194            image.clone(),
12195        );
12196        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12197        assert_eq!(verdict.image, image);
12198    }
12199
12200    #[test]
12201    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
12202        let image = subc_control::RunningImageAgreement::Unavailable {
12203            reason: subc_control::RunningImageUnavailableReason::NotRunning,
12204        };
12205        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
12206        assert_eq!(
12207            verdict.path,
12208            subc_control::ReloadPathAgreement::Unavailable {
12209                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
12210            }
12211        );
12212        assert_eq!(verdict.image, image);
12213
12214        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
12215            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12216        };
12217        let verdict = reload_verdict(
12218            std::path::Path::new("/bin/same"),
12219            Some(std::path::Path::new("/bin/same")),
12220            unconfirmed.clone(),
12221        );
12222        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12223        assert_eq!(verdict.image, unconfirmed);
12224    }
12225
12226    #[test]
12227    fn reload_verdict_preserves_each_image_unavailability_reason() {
12228        use subc_control::RunningImageUnavailableReason as Reason;
12229
12230        for reason in [
12231            Reason::NotRunning,
12232            Reason::UnsupportedPlatform,
12233            Reason::RunningExecutableUnreadable,
12234            Reason::SpawnedPathUnreadable,
12235            Reason::HashFailed,
12236            Reason::ProcessIdentityUnconfirmed,
12237            Reason::Unknown("future_probe_reason".to_string()),
12238        ] {
12239            let image = subc_control::RunningImageAgreement::Unavailable {
12240                reason: reason.clone(),
12241            };
12242            let verdict = reload_verdict(
12243                std::path::Path::new("/bin/same"),
12244                Some(std::path::Path::new("/bin/same")),
12245                image.clone(),
12246            );
12247            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12248            assert_eq!(verdict.image, image, "{reason:?}");
12249        }
12250    }
12251
12252    #[tokio::test]
12253    async fn malformed_control_bodies_return_invalid_control_body() {
12254        let handler = ControlHandler::default();
12255        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
12256
12257        for (corr, body) in [
12258            (56, br#"{"route_channel":1}"#.as_slice()),
12259            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
12260            (
12261                58,
12262                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
12263            ),
12264        ] {
12265            let request = Frame::build(
12266                FrameType::Request,
12267                control_flags(),
12268                0,
12269                0,
12270                corr,
12271                body.to_vec(),
12272            )
12273            .unwrap();
12274            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12275
12276            assert_eq!(response.len(), 1);
12277            assert_eq!(response[0].header.ty, FrameType::Error);
12278            assert_eq!(response[0].header.corr, corr);
12279            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
12280        }
12281    }
12282
12283    #[tokio::test]
12284    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
12285        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12286        let registry = Arc::new(Registry::default());
12287        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
12288        let router = Router::with_control_handler(Arc::clone(&control));
12289        let connection = router.begin_connection();
12290        let (ctx, mut rx) = route_ctx(connection.id());
12291
12292        router
12293            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
12294            .await
12295            .unwrap();
12296        let response = rx.recv().await.unwrap();
12297        let ack = parse_ack(&response);
12298        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12299        let channel = 1;
12300
12301        let goodbye =
12302            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
12303        router.route_for_connection(&ctx, goodbye).await.unwrap();
12304        assert!(rx.try_recv().is_err());
12305        assert!(registry.get_module("aft").unwrap().is_none());
12306
12307        router
12308            .route_for_connection(&ctx, channel_request(channel, 13))
12309            .await
12310            .unwrap();
12311        let error_frame = rx.recv().await.unwrap();
12312        assert_eq!(error_frame.header.ty, FrameType::Error);
12313        assert_eq!(error_frame.header.channel, channel);
12314        let captured = crate::router::test_log::captured_logs(&logs);
12315        assert_eq!(
12316            captured
12317                .lines()
12318                .filter(|line| {
12319                    line.contains("module_id=aft")
12320                        && line.contains("reason=explicit_goodbye")
12321                        && line.contains("module registration ended")
12322                })
12323                .count(),
12324            1,
12325            "unexpected GOODBYE registry log: {captured}"
12326        );
12327    }
12328
12329    #[tokio::test]
12330    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
12331        let registry = Arc::new(Registry::default());
12332        let handler = ControlHandler::new(registry).with_capability_config(
12333            [("prov".to_string(), true), ("cons".to_string(), true)],
12334            BTreeMap::new(),
12335        );
12336        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
12337        register_capability_manifest(
12338            &handler,
12339            &provider_ctx,
12340            &mut provider_rx,
12341            capability_manifest("prov", &["thing/v1"], &[]),
12342            1,
12343        )
12344        .await;
12345        let mut consumer = capability_manifest("cons", &[], &[]);
12346        consumer.capabilities.as_mut().unwrap().requires.push(
12347            subc_protocol::manifest::CapabilityRequirement {
12348                capability: "thing/v1".to_string(),
12349                need: subc_protocol::manifest::CapabilityNeed::Required,
12350            },
12351        );
12352        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
12353        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
12354        assert_eq!(
12355            handler.capability_evaluator.verdict("cons", "thing/v1"),
12356            Some(CapabilityVerdict::Provided)
12357        );
12358        let (mut client_rx, _) = open_route_for_capability_test(
12359            &handler,
12360            &provider_ctx,
12361            &mut provider_rx,
12362            703,
12363            3,
12364            "prov",
12365            None,
12366        )
12367        .await;
12368        let goodbye =
12369            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12370        handler
12371            .handle_control_frame(&provider_ctx, goodbye)
12372            .await
12373            .unwrap();
12374        assert_eq!(
12375            handler.capability_evaluator.verdict("cons", "thing/v1"),
12376            Some(CapabilityVerdict::NeverProvided)
12377        );
12378        let closed = client_rx
12379            .try_recv()
12380            .expect("GOODBYE pushes route.closed before route GOODBYE");
12381        assert!(
12382            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
12383            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
12384        );
12385        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
12386        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12387    }
12388
12389    struct RouteClosedFixture {
12390        handler: ControlHandler,
12391        #[cfg(unix)]
12392        supervisor: Supervisor,
12393        module_ctx: RouteCtx,
12394        #[cfg(unix)]
12395        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12396        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12397        route: Arc<crate::forwarding::RouteBinding>,
12398    }
12399
12400    async fn route_closed_fixture() -> RouteClosedFixture {
12401        let registry = Arc::new(Registry::default());
12402        let handler = ControlHandler::new(Arc::clone(&registry));
12403        #[cfg(unix)]
12404        let supervisor = Supervisor::new(registry, RestartPolicy::default())
12405            .with_forwarding(Arc::clone(&handler.forwarding));
12406        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
12407        register_capability_manifest(
12408            &handler,
12409            &module_ctx,
12410            &mut module_rx,
12411            capability_manifest("prov", &[], &[]),
12412            1,
12413        )
12414        .await;
12415        let (client_rx, bind) = open_route_for_capability_test(
12416            &handler,
12417            &module_ctx,
12418            &mut module_rx,
12419            702,
12420            2,
12421            "prov",
12422            None,
12423        )
12424        .await;
12425        let ModuleControlRequest::RouteBind {
12426            route_channel: channel,
12427            epoch: route_epoch,
12428            ..
12429        } = bind
12430        else {
12431            panic!("expected route.bind");
12432        };
12433        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
12434            handler
12435                .forwarding
12436                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
12437                .unwrap()
12438        else {
12439            panic!("the acknowledged route must be bound");
12440        };
12441        RouteClosedFixture {
12442            handler,
12443            #[cfg(unix)]
12444            supervisor,
12445            module_ctx,
12446            #[cfg(unix)]
12447            module_rx,
12448            client_rx,
12449            route,
12450        }
12451    }
12452
12453    impl RouteClosedFixture {
12454        #[cfg(unix)]
12455        async fn cut(&mut self) {
12456            let notices = async {
12457                let mut draining = self.module_rx.recv().await.unwrap();
12458                assert!(matches!(
12459                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
12460                        &draining.body
12461                    )
12462                    .unwrap(),
12463                    subc_protocol::session::ModuleControlCommand::Draining {
12464                        reason: RouteCloseReason::Restart,
12465                        ..
12466                    }
12467                ));
12468                draining.flushed.take().unwrap().send(()).unwrap();
12469                let mut closing = self.client_rx.recv().await.unwrap();
12470                assert!(matches!(
12471                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
12472                    ClientControlPush::RouteClosing {
12473                        reason: RouteCloseReason::Restart,
12474                        ..
12475                    }
12476                ));
12477                closing.flushed.take().unwrap().send(()).unwrap();
12478            };
12479            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
12480            result.unwrap();
12481        }
12482
12483        fn disconnect(&mut self) -> ClientControlPush {
12484            self.handler
12485                .cleanup_connection(self.module_ctx.connection_id)
12486                .unwrap();
12487            let closed = self.client_rx.try_recv().unwrap();
12488            assert_eq!(closed.header.ty, FrameType::Push);
12489            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
12490            assert!(matches!(
12491                &push,
12492                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
12493                    if channels == &[self.route.client_channel]
12494            ));
12495            assert_eq!(
12496                self.client_rx.try_recv().unwrap().header.ty,
12497                FrameType::Goodbye
12498            );
12499            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
12500            push
12501        }
12502    }
12503
12504    #[cfg(unix)]
12505    #[tokio::test(start_paused = true)]
12506    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
12507        let mut fixture = route_closed_fixture().await;
12508        fixture.cut().await;
12509        assert!(matches!(
12510            fixture.disconnect(),
12511            ClientControlPush::RouteClosed {
12512                reason: RouteCloseReason::Restart,
12513                drained: true,
12514                ..
12515            }
12516        ));
12517    }
12518
12519    #[cfg(unix)]
12520    #[tokio::test(start_paused = true)]
12521    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
12522        let mut fixture = route_closed_fixture().await;
12523        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
12524        let started = tokio::time::Instant::now();
12525        fixture.cut().await;
12526        assert!(started.elapsed() >= Duration::from_secs(2));
12527        assert_eq!(fixture.route.flow.in_flight(), 1);
12528        assert!(matches!(
12529            fixture.disconnect(),
12530            ClientControlPush::RouteClosed {
12531                reason: RouteCloseReason::Restart,
12532                drained: false,
12533                ..
12534            }
12535        ));
12536    }
12537
12538    #[tokio::test]
12539    async fn module_connection_crash_reports_crash_and_not_drained() {
12540        let mut fixture = route_closed_fixture().await;
12541        // The server calls cleanup_connection on module EOF. Without a planned
12542        // daemon shutdown, that disconnect must remain a crash with no prior
12543        // route.closing notice, even when the route has no in-flight calls.
12544        assert!(matches!(
12545            fixture.disconnect(),
12546            ClientControlPush::RouteClosed {
12547                reason: RouteCloseReason::Crash,
12548                drained: false,
12549                ..
12550            }
12551        ));
12552    }
12553
12554    #[tokio::test]
12555    async fn dropping_router_connection_releases_registration() {
12556        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12557        let registry = Arc::new(Registry::default());
12558        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
12559        let router = Router::with_control_handler(Arc::clone(&control));
12560        let connection = router.begin_connection();
12561        let connection_id = connection.id();
12562        let (ctx, mut rx) = route_ctx(connection_id);
12563
12564        router
12565            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
12566            .await
12567            .unwrap();
12568        let response = rx.recv().await.unwrap();
12569        let ack = parse_ack(&response);
12570        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12571        assert!(registry.get_module("aft").unwrap().is_some());
12572
12573        drop(connection);
12574
12575        assert!(registry.get_module("aft").unwrap().is_none());
12576        assert_eq!(registry.active_registration_count().unwrap(), 0);
12577
12578        control
12579            .cleanup_connection(ConnectionId::new(u64::MAX))
12580            .unwrap();
12581        let captured = crate::router::test_log::captured_logs(&logs);
12582        println!("captured registry lifecycle logs:\n{captured}");
12583        let events: Vec<_> = captured
12584            .lines()
12585            .filter(|line| {
12586                line.contains("module registered module_id=aft ")
12587                    || line.contains("module registration ended")
12588            })
12589            .collect();
12590        assert_eq!(
12591            events.len(),
12592            2,
12593            "unexpected registry lifecycle logs: {captured}"
12594        );
12595        assert!(events[0].contains("module registered module_id=aft "));
12596        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
12597        assert!(events[1].contains(&format!(
12598            "module_id=aft connection_id={}",
12599            connection_id.get()
12600        )));
12601        assert!(events[1].contains("reason=connection_closed"));
12602        assert!(events[1].contains("module registration ended"));
12603        assert_eq!(
12604            captured
12605                .lines()
12606                .filter(|line| line.contains("module registered module_id=aft "))
12607                .count(),
12608            1,
12609            "legacy registration admission line must appear once: {captured}"
12610        );
12611    }
12612
12613    #[tokio::test]
12614    async fn hello_registration_keeps_legacy_module_registered_line_once() {
12615        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12616        let registry = Arc::new(Registry::default());
12617        let control = ControlHandler::new(Arc::clone(&registry));
12618        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
12619        hello_via_sink(
12620            &control,
12621            &ctx,
12622            &mut rx,
12623            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
12624        )
12625        .await;
12626
12627        let captured = crate::router::test_log::captured_logs(&logs);
12628        let admissions: Vec<_> = captured
12629            .lines()
12630            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
12631            .collect();
12632        assert_eq!(
12633            admissions.len(),
12634            1,
12635            "legacy admission line must remain exactly once: {captured}"
12636        );
12637        assert!(admissions[0].contains("routable_provider=true"));
12638        assert!(admissions[0].contains("connection_id=777"));
12639    }
12640
12641    fn capability_manifest(
12642        module_id: &str,
12643        provides: &[&str],
12644        must_never_reach: &[&str],
12645    ) -> ModuleManifest {
12646        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
12647        manifest.capabilities = Some(CapabilityDeclarations {
12648            provides: provides
12649                .iter()
12650                .map(|capability| (*capability).to_string())
12651                .collect(),
12652            requires: Vec::new(),
12653            must_never_reach: must_never_reach
12654                .iter()
12655                .map(|capability| (*capability).to_string())
12656                .collect(),
12657        });
12658        manifest
12659    }
12660
12661    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
12662        Frame::build(
12663            FrameType::Hello,
12664            control_flags(),
12665            0,
12666            0,
12667            corr,
12668            serde_json::to_vec(&ModuleHelloBody {
12669                protocol_ver: manifest.protocol_ver,
12670                manifest,
12671                control_ops: None,
12672                launch_nonce: None,
12673            })
12674            .expect("capability test HELLO serializes"),
12675        )
12676        .expect("capability test HELLO frame builds")
12677    }
12678
12679    fn catalog_update_with_capabilities_frame(
12680        corr: u64,
12681        capabilities: CapabilityDeclarations,
12682    ) -> Frame {
12683        Frame::build(
12684            FrameType::Request,
12685            control_flags(),
12686            0,
12687            0,
12688            corr,
12689            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12690                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
12691                capabilities: Some(capabilities),
12692                ready: None,
12693            })
12694            .expect("capability catalog.update serializes"),
12695        )
12696        .expect("capability catalog.update frame builds")
12697    }
12698
12699    async fn register_capability_manifest(
12700        handler: &ControlHandler,
12701        ctx: &RouteCtx,
12702        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12703        manifest: ModuleManifest,
12704        corr: u64,
12705    ) {
12706        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
12707    }
12708
12709    async fn open_route_for_capability_test(
12710        handler: &ControlHandler,
12711        target_ctx: &RouteCtx,
12712        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12713        client_connection_id: u64,
12714        corr: u64,
12715        target_module_id: &str,
12716        consumer_identity: Option<ConsumerIdentity>,
12717    ) -> (
12718        mpsc::Receiver<crate::router::OutboundFrame>,
12719        ModuleControlRequest,
12720    ) {
12721        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
12722        let route_handler = handler.clone();
12723        let target_module_id = target_module_id.to_string();
12724        let route_task = tokio::spawn(async move {
12725            route_handler
12726                .handle_control_frame(
12727                    &client_ctx,
12728                    route_open_frame_with_admission_facts(
12729                        corr,
12730                        &target_module_id,
12731                        unique_project_root("admission-facts"),
12732                        consumer_identity,
12733                        None,
12734                    ),
12735                )
12736                .await
12737                .expect("capability test route.open succeeds")
12738        });
12739        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
12740            .await
12741            .expect("capability test route.open must reach route.bind")
12742            .expect("target control receiver stays open");
12743        let bind_request: ModuleControlRequest =
12744            serde_json::from_slice(&bind.body).expect("route.bind decodes");
12745        handler
12746            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
12747            .await
12748            .expect("capability test route.bind ACK succeeds");
12749        assert!(route_task.await.expect("route.open task joins").is_empty());
12750        let opened = client_rx
12751            .recv()
12752            .await
12753            .expect("successful route.open publishes a response");
12754        assert!(matches!(
12755            serde_json::from_slice::<ClientControlResponse>(&opened.body),
12756            Ok(ClientControlResponse::RouteOpen { .. })
12757        ));
12758        (client_rx, bind_request)
12759    }
12760
12761    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
12762        assert_eq!(frame.header.ty, FrameType::Push);
12763        assert_eq!(frame.header.channel, 0);
12764        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
12765            .expect("route.closed control push decodes");
12766        let ClientControlPush::RouteClosed { channels, .. } = &push else {
12767            panic!("expected route.closed");
12768        };
12769        assert_eq!(channels.len(), 1, "exactly one violating route closed");
12770        let channels = channels.clone();
12771        assert_eq!(
12772            push,
12773            ClientControlPush::RouteClosed {
12774                module_id: target_module_id.to_string(),
12775                channels,
12776                reason: RouteCloseReason::CapabilityDenied,
12777                drained: false,
12778                abandoned: 0,
12779                excluded_subscriptions: 0,
12780                terminal: Some(false),
12781            }
12782        );
12783    }
12784
12785    #[tokio::test]
12786    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
12787        let registry = Arc::new(Registry::default());
12788        let forwarding = Arc::new(ForwardingTable::default());
12789        let supervisor = SupervisorHandle::new();
12790        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12791        let handler =
12792            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12793                .with_supervisor(supervisor);
12794        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
12795        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
12796        register_capability_manifest(
12797            &handler,
12798            &target_ctx,
12799            &mut target_rx,
12800            capability_manifest("target", &["credentials-provider/v1"], &[]),
12801            1,
12802        )
12803        .await;
12804        register_capability_manifest(
12805            &handler,
12806            &opener_ctx,
12807            &mut opener_rx,
12808            capability_manifest("opener", &[], &["credentials-provider/v1"]),
12809            2,
12810        )
12811        .await;
12812
12813        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
12814        let replies = handler
12815            .handle_control_frame(
12816                &client_ctx,
12817                route_open_frame_with_admission_facts(
12818                    3,
12819                    "target",
12820                    unique_project_root("admission-facts"),
12821                    Some(ConsumerIdentity {
12822                        module_id: "opener".to_string(),
12823                        launch_nonce: "opener-nonce".to_string(),
12824                    }),
12825                    None,
12826                ),
12827            )
12828            .await
12829            .expect("denied route.open returns a typed frame");
12830        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12831        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12832        assert!(
12833            target_rx.try_recv().is_err(),
12834            "forbidden route.open must not relay route.bind"
12835        );
12836    }
12837
12838    #[tokio::test]
12839    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
12840        let registry = Arc::new(Registry::default());
12841        let forwarding = Arc::new(ForwardingTable::default());
12842        let supervisor = SupervisorHandle::new();
12843        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12844        let handler =
12845            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12846                .with_supervisor(supervisor);
12847        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
12848        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
12849        register_capability_manifest(
12850            &handler,
12851            &target_ctx,
12852            &mut target_rx,
12853            capability_manifest("target", &["credentials-provider/v1"], &[]),
12854            1,
12855        )
12856        .await;
12857        register_capability_manifest(
12858            &handler,
12859            &old_opener_ctx,
12860            &mut old_opener_rx,
12861            capability_manifest("opener", &[], &[]),
12862            2,
12863        )
12864        .await;
12865        let (mut client_rx, _) = open_route_for_capability_test(
12866            &handler,
12867            &target_ctx,
12868            &mut target_rx,
12869            712,
12870            3,
12871            "target",
12872            Some(ConsumerIdentity {
12873                module_id: "opener".to_string(),
12874                launch_nonce: "opener-nonce".to_string(),
12875            }),
12876        )
12877        .await;
12878        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12879
12880        handler
12881            .cleanup_connection(old_opener_ctx.connection_id)
12882            .expect("old opener registration cleans up");
12883        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
12884        register_capability_manifest(
12885            &handler,
12886            &new_opener_ctx,
12887            &mut new_opener_rx,
12888            capability_manifest("opener", &[], &["credentials-provider/v1"]),
12889            4,
12890        )
12891        .await;
12892
12893        assert_capability_denied_push(
12894            client_rx
12895                .try_recv()
12896                .expect("HELLO deny addition must emit route.closed")
12897                .frame,
12898            "target",
12899        );
12900        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12901        assert!(matches!(
12902            target_rx.try_recv(),
12903            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12904        ));
12905    }
12906
12907    #[tokio::test]
12908    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
12909        let registry = Arc::new(Registry::default());
12910        let forwarding = Arc::new(ForwardingTable::default());
12911        let supervisor = SupervisorHandle::new();
12912        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12913        let handler =
12914            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12915                .with_supervisor(supervisor);
12916        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
12917        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
12918        register_capability_manifest(
12919            &handler,
12920            &target_ctx,
12921            &mut target_rx,
12922            capability_manifest("target", &[], &[]),
12923            1,
12924        )
12925        .await;
12926        register_capability_manifest(
12927            &handler,
12928            &opener_ctx,
12929            &mut opener_rx,
12930            capability_manifest("opener", &[], &["credentials-provider/v1"]),
12931            2,
12932        )
12933        .await;
12934        let (mut client_rx, _) = open_route_for_capability_test(
12935            &handler,
12936            &target_ctx,
12937            &mut target_rx,
12938            722,
12939            3,
12940            "target",
12941            Some(ConsumerIdentity {
12942                module_id: "opener".to_string(),
12943                launch_nonce: "opener-nonce".to_string(),
12944            }),
12945        )
12946        .await;
12947        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12948
12949        let replies = handler
12950            .handle_control_frame(
12951                &target_ctx,
12952                catalog_update_with_capabilities_frame(
12953                    4,
12954                    CapabilityDeclarations {
12955                        provides: vec!["credentials-provider/v1".to_string()],
12956                        requires: Vec::new(),
12957                        must_never_reach: Vec::new(),
12958                    },
12959                ),
12960            )
12961            .await
12962            .expect("claim catalog.update succeeds");
12963        assert!(matches!(
12964            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
12965            Ok(ModuleControlResponseToModule::CatalogUpdate {})
12966        ));
12967        assert_capability_denied_push(
12968            client_rx
12969                .try_recv()
12970                .expect("claim addition must emit route.closed")
12971                .frame,
12972            "target",
12973        );
12974        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12975        assert!(matches!(
12976            target_rx.try_recv(),
12977            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12978        ));
12979    }
12980
12981    #[tokio::test]
12982    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
12983        let registry = Arc::new(Registry::default());
12984        let forwarding = Arc::new(ForwardingTable::default());
12985        let supervisor = SupervisorHandle::new();
12986        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12987        let handler =
12988            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12989                .with_supervisor(supervisor);
12990        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
12991        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
12992        register_capability_manifest(
12993            &handler,
12994            &target_ctx,
12995            &mut target_rx,
12996            capability_manifest("target", &["credentials-provider/v1"], &[]),
12997            1,
12998        )
12999        .await;
13000        register_capability_manifest(
13001            &handler,
13002            &opener_ctx,
13003            &mut opener_rx,
13004            capability_manifest("opener", &[], &[]),
13005            2,
13006        )
13007        .await;
13008        let (mut client_rx, _) = open_route_for_capability_test(
13009            &handler,
13010            &target_ctx,
13011            &mut target_rx,
13012            732,
13013            3,
13014            "target",
13015            Some(ConsumerIdentity {
13016                module_id: "opener".to_string(),
13017                launch_nonce: "opener-nonce".to_string(),
13018            }),
13019        )
13020        .await;
13021        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13022
13023        handler
13024            .handle_control_frame(
13025                &target_ctx,
13026                catalog_update_with_capabilities_frame(
13027                    4,
13028                    CapabilityDeclarations {
13029                        provides: Vec::new(),
13030                        requires: Vec::new(),
13031                        must_never_reach: Vec::new(),
13032                    },
13033                ),
13034            )
13035            .await
13036            .expect("claim removal catalog.update succeeds");
13037        assert_eq!(
13038            forwarding.active_binding_count().unwrap(),
13039            1,
13040            "removing an attested target claim must leave the route census unchanged"
13041        );
13042        assert!(
13043            client_rx.try_recv().is_err(),
13044            "claim removal must not emit route.closed capability_denied"
13045        );
13046        assert!(
13047            target_rx.try_recv().is_err(),
13048            "claim removal must not send the target a route GOODBYE"
13049        );
13050    }
13051
13052    /// A direct client may open a route to a denied capability provider; this
13053    /// policy applies only to attested supervised module origins, not to direct clients.
13054    #[tokio::test]
13055    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
13056        let registry = Arc::new(Registry::default());
13057        let forwarding = Arc::new(ForwardingTable::default());
13058        let supervisor = SupervisorHandle::new();
13059        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13060        let handler =
13061            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13062                .with_supervisor(supervisor);
13063        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
13064        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
13065        register_capability_manifest(
13066            &handler,
13067            &target_ctx,
13068            &mut target_rx,
13069            capability_manifest("target", &["credentials-provider/v1"], &[]),
13070            1,
13071        )
13072        .await;
13073        register_capability_manifest(
13074            &handler,
13075            &opener_ctx,
13076            &mut opener_rx,
13077            capability_manifest("opener", &[], &["credentials-provider/v1"]),
13078            2,
13079        )
13080        .await;
13081
13082        let (_client_rx, bind) = open_route_for_capability_test(
13083            &handler,
13084            &target_ctx,
13085            &mut target_rx,
13086            742,
13087            3,
13088            "target",
13089            None,
13090        )
13091        .await;
13092        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
13093            panic!("direct scope-honesty route must bind");
13094        };
13095        assert_eq!(principal, Some(Principal::Direct));
13096        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13097    }
13098
13099    /// A module that denies a capability receives no self-route exemption when it
13100    /// also attestedly provides that capability.
13101    #[tokio::test]
13102    async fn must_never_reach_self_route_is_capability_forbidden() {
13103        let registry = Arc::new(Registry::default());
13104        let forwarding = Arc::new(ForwardingTable::default());
13105        let supervisor = SupervisorHandle::new();
13106        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
13107        let handler =
13108            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
13109                .with_supervisor(supervisor);
13110        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
13111        register_capability_manifest(
13112            &handler,
13113            &self_ctx,
13114            &mut self_rx,
13115            capability_manifest(
13116                "self-provider",
13117                &["credentials-provider/v1"],
13118                &["credentials-provider/v1"],
13119            ),
13120            1,
13121        )
13122        .await;
13123
13124        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
13125        let replies = handler
13126            .handle_control_frame(
13127                &client_ctx,
13128                route_open_frame_with_admission_facts(
13129                    2,
13130                    "self-provider",
13131                    unique_project_root("admission-facts"),
13132                    Some(ConsumerIdentity {
13133                        module_id: "self-provider".to_string(),
13134                        launch_nonce: "self-nonce".to_string(),
13135                    }),
13136                    None,
13137                ),
13138            )
13139            .await
13140            .expect("self-route refusal returns a typed frame");
13141        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13142        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13143        assert!(
13144            self_rx.try_recv().is_err(),
13145            "self denial must not relay route.bind"
13146        );
13147    }
13148
13149    #[test]
13150    fn unsupported_channel_zero_frame_returns_error() {
13151        let handler = ControlHandler::default();
13152        let request = Frame::build(
13153            FrameType::Request,
13154            control_flags(),
13155            0,
13156            0,
13157            21,
13158            b"opaque".to_vec(),
13159        )
13160        .unwrap();
13161
13162        let response = handler
13163            .handle_control(ConnectionId::new(1), request)
13164            .unwrap();
13165
13166        assert_eq!(response[0].header.ty, FrameType::Error);
13167        assert_eq!(
13168            parse_error(&response[0])["code"],
13169            "unsupported_control_frame"
13170        );
13171    }
13172
13173    /// Blue/green swap at the control-plane boundary. The supervisor that opens
13174    /// a swap is not wired yet, so the candidate is registered here directly
13175    /// into the registry and forwarding candidate slots, the way the swap's
13176    /// HELLO admission will.
13177    mod swap {
13178        use super::*;
13179
13180        const INCUMBENT: ConnectionId = ConnectionId::new(30);
13181        const CANDIDATE: ConnectionId = ConnectionId::new(40);
13182
13183        struct Swap {
13184            registry: Arc<Registry>,
13185            forwarding: Arc<ForwardingTable>,
13186            handler: ControlHandler,
13187            incumbent_ctx: RouteCtx,
13188            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13189            candidate_ctx: RouteCtx,
13190            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13191        }
13192
13193        async fn swap_with_incumbent() -> Swap {
13194            let registry = Arc::new(Registry::default());
13195            let forwarding = Arc::new(ForwardingTable::default());
13196            let handler =
13197                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13198            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
13199            hello_via_sink(
13200                &handler,
13201                &incumbent_ctx,
13202                &mut incumbent_rx,
13203                hello_frame("aft", PROTOCOL_VERSION, 7),
13204            )
13205            .await;
13206            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
13207            Swap {
13208                registry,
13209                forwarding,
13210                handler,
13211                incumbent_ctx,
13212                incumbent_rx,
13213                candidate_ctx,
13214                candidate_rx,
13215            }
13216        }
13217
13218        fn register_candidate(swap: &Swap, ready: Option<bool>) {
13219            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
13220            candidate_manifest.ready = ready;
13221            let registration = swap
13222                .registry
13223                .register_candidate_with_control_ops(
13224                    candidate_manifest,
13225                    PROTOCOL_VERSION,
13226                    CANDIDATE,
13227                    module_baseline_control_ops(),
13228                )
13229                .unwrap();
13230            swap.forwarding
13231                .register_candidate_module_connection(
13232                    CANDIDATE,
13233                    "aft".to_string(),
13234                    PROTOCOL_VERSION,
13235                    manifest_concurrency(&registration.manifest),
13236                    swap.candidate_ctx.egress.clone(),
13237                )
13238                .unwrap();
13239        }
13240
13241        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
13242            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
13243            swap.registry.promote_candidate("aft").unwrap().unwrap();
13244            cutover.incumbent.unwrap()
13245        }
13246
13247        fn keyed_total(counters: &Value, key: &str) -> u64 {
13248            counters[key]
13249                .as_object()
13250                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
13251                .unwrap_or(0)
13252        }
13253
13254        #[tokio::test]
13255        async fn replacement_logs_old_end_and_new_admission_once() {
13256            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13257            let mut swap = swap_with_incumbent().await;
13258            swap.handler
13259                .supervisor
13260                .open_swap("aft", "candidate-nonce".to_string());
13261            hello_via_sink(
13262                &swap.handler,
13263                &swap.candidate_ctx,
13264                &mut swap.candidate_rx,
13265                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
13266            )
13267            .await;
13268            cutover(&swap);
13269            swap.handler.cleanup_connection(INCUMBENT).unwrap();
13270
13271            let captured = crate::router::test_log::captured_logs(&logs);
13272            println!("captured replacement registry logs:\n{captured}");
13273            let new_admissions: Vec<_> = captured
13274                .lines()
13275                .filter(|line| {
13276                    line.contains("connection_id=40")
13277                        && line.contains("swap candidate registered; not routable until cutover")
13278                })
13279                .collect();
13280            assert_eq!(
13281                new_admissions.len(),
13282                1,
13283                "unexpected admission logs: {captured}"
13284            );
13285            assert!(new_admissions[0].contains("module_id=aft"));
13286            assert!(new_admissions[0].contains("connection_id=40"));
13287            assert!(
13288                new_admissions[0].contains("swap candidate registered; not routable until cutover")
13289            );
13290            assert!(new_admissions[0].contains("ready=true"));
13291
13292            let old_admissions: Vec<_> = captured
13293                .lines()
13294                .filter(|line| {
13295                    line.contains("module registered module_id=aft ")
13296                        && line.contains("connection_id=30")
13297                })
13298                .collect();
13299            assert_eq!(
13300                old_admissions.len(),
13301                1,
13302                "unexpected incumbent admission: {captured}"
13303            );
13304
13305            let old_ends: Vec<_> = captured
13306                .lines()
13307                .filter(|line| {
13308                    line.contains("connection_id=30") && line.contains("module registration ended")
13309                })
13310                .collect();
13311            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
13312            assert!(old_ends[0].contains("module_id=aft"));
13313            assert!(old_ends[0].contains("connection_id=30"));
13314            assert!(old_ends[0].contains("reason=replaced"));
13315            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
13316            assert_eq!(
13317                captured
13318                    .lines()
13319                    .filter(|line| line.contains("module registration promoted"))
13320                    .count(),
13321                1,
13322                "unexpected promotion logs: {captured}"
13323            );
13324        }
13325
13326        /// An ack from the incumbent for a bind it was sent before cutover,
13327        /// arriving before the incumbent is drained. The incumbent is the live
13328        /// connection carrying every other client's routes, so the ack must
13329        /// not end it: the waiting client is told to retry, the reservation is
13330        /// given back, and the incumbent is told to drop just that binding.
13331        #[tokio::test]
13332        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
13333            let mut swap = swap_with_incumbent().await;
13334            let handler = swap.handler.clone();
13335
13336            // A co-tenant route, bound on the incumbent before the swap.
13337            let cotenant = ConnectionId::new(31);
13338            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
13339            let (cotenant_task, cotenant_bind) = relay_route_open(
13340                &handler,
13341                cotenant,
13342                &cotenant_ctx.egress,
13343                &mut swap.incumbent_rx,
13344                100,
13345                "aft",
13346                "swap-cotenant",
13347            )
13348            .await;
13349            handler
13350                .handle_control_frame(
13351                    &swap.incumbent_ctx,
13352                    route_bind_ack(cotenant_bind.header.corr),
13353                )
13354                .await
13355                .unwrap();
13356            assert!(cotenant_task.await.unwrap().is_empty());
13357            let (cotenant_channel, cotenant_epoch) =
13358                published_route(&cotenant_rx.recv().await.unwrap());
13359
13360            // A second route.open, relayed to the incumbent and not yet acked.
13361            let caller = ConnectionId::new(32);
13362            let (caller_ctx, mut caller_rx) = route_ctx(caller);
13363            let (caller_task, caller_bind) = relay_route_open(
13364                &handler,
13365                caller,
13366                &caller_ctx.egress,
13367                &mut swap.incumbent_rx,
13368                101,
13369                "aft",
13370                "swap-caller",
13371            )
13372            .await;
13373            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
13374
13375            register_candidate(&swap, None);
13376            cutover(&swap);
13377
13378            // The incumbent acks after promotion and before any drain.
13379            let ack = handler
13380                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
13381                .await;
13382            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13383            if module_loop_error.is_some() {
13384                // What the connection loop does with an untranslated router
13385                // error: end the connection, releasing every route on it.
13386                handler.cleanup_connection(INCUMBENT).unwrap();
13387            }
13388
13389            // 1. The incumbent's other routes survive.
13390            assert!(
13391                cotenant_rx.try_recv().is_err(),
13392                "the co-tenant route on the incumbent was torn down by one late ack: \
13393                 {module_loop_error:?}"
13394            );
13395            assert!(matches!(
13396                swap.forwarding
13397                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
13398                    .unwrap(),
13399                DataRoute::Client(DataRouteState::Bound(_))
13400            ));
13401            assert_eq!(module_loop_error, None);
13402            assert!(swap
13403                .registry
13404                .get_module_by_connection(INCUMBENT)
13405                .unwrap()
13406                .is_some());
13407
13408            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
13409            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
13410                .await
13411                .expect("the incumbent is told to drop the abandoned binding")
13412                .unwrap()
13413                .frame;
13414            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13415            assert_eq!(goodbye.header.channel, abandoned_channel);
13416            assert_eq!(goodbye.header.epoch, abandoned_epoch);
13417            assert!(swap.incumbent_rx.try_recv().is_err());
13418
13419            // 3. The waiting client gets a retryable refusal and no route.
13420            let response = caller_task.await.unwrap();
13421            assert_eq!(response.len(), 1);
13422            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
13423            assert!(caller_rx.try_recv().is_err());
13424
13425            // 4. The reservation pair is given back, and the pending bind
13426            //    settled exactly once: one accepted open (the co-tenant) and one
13427            //    refused open (the caller), nothing counted twice.
13428            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
13429            let counters = handler.counters().snapshot();
13430            assert_eq!(
13431                keyed_total(&counters, "route_open_accepted_by_principal"),
13432                1
13433            );
13434            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
13435            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
13436        }
13437
13438        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
13439        /// id would resolve to the promoted candidate and every new route.open
13440        /// would be refused as reloading, leaving neither process routable.
13441        #[tokio::test]
13442        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
13443            let mut swap = swap_with_incumbent().await;
13444            register_candidate(&swap, None);
13445            let incumbent = cutover(&swap);
13446            swap.forwarding
13447                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
13448                .unwrap()
13449                .expect("the incumbent is still registered");
13450
13451            let client = ConnectionId::new(33);
13452            let (client_ctx, mut client_rx) = route_ctx(client);
13453            let route_handler = swap.handler.clone();
13454            let open_ctx = RouteCtx {
13455                connection_id: client,
13456                egress: client_ctx.egress.clone(),
13457            };
13458            let mut route_task = tokio::spawn(async move {
13459                route_handler
13460                    .handle_control_frame(
13461                        &open_ctx,
13462                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
13463                    )
13464                    .await
13465                    .unwrap()
13466            });
13467            let bind = tokio::select! {
13468                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
13469                response = &mut route_task => {
13470                    let response = response.unwrap();
13471                    panic!(
13472                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
13473                        parse_error(&response[0])["code"]
13474                    );
13475                }
13476            };
13477            swap.handler
13478                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
13479                .await
13480                .unwrap();
13481            assert!(route_task.await.unwrap().is_empty());
13482            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
13483            match swap
13484                .forwarding
13485                .lookup_data_route(client, channel, epoch)
13486                .unwrap()
13487            {
13488                DataRoute::Client(DataRouteState::Bound(route)) => {
13489                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
13490                }
13491                other => panic!("expected a bound route on the candidate, got {other:?}"),
13492            }
13493            assert!(swap.incumbent_rx.try_recv().is_err());
13494        }
13495
13496        /// A candidate declares itself ready with `catalog.update` on its own
13497        /// connection. If the connection-keyed registry lookups searched only the
13498        /// active slot, this would answer `not_registered` and the candidate
13499        /// would never become ready.
13500        #[tokio::test]
13501        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
13502            let swap = swap_with_incumbent().await;
13503            register_candidate(&swap, Some(false));
13504            let update = Frame::build(
13505                FrameType::Request,
13506                control_flags(),
13507                0,
13508                0,
13509                55,
13510                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13511                    provides: manifest("aft", PROTOCOL_VERSION).provides,
13512                    capabilities: None,
13513                    ready: Some(true),
13514                })
13515                .unwrap(),
13516            )
13517            .unwrap();
13518
13519            let replies = swap
13520                .handler
13521                .handle_control_frame(&swap.candidate_ctx, update)
13522                .await
13523                .unwrap();
13524
13525            assert_eq!(replies.len(), 1);
13526            assert_eq!(
13527                replies[0].header.ty,
13528                FrameType::Response,
13529                "candidate catalog.update was refused: {:?}",
13530                serde_json::from_slice::<Value>(&replies[0].body).ok()
13531            );
13532            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
13533            assert_eq!(
13534                swap.registry
13535                    .get_module("aft")
13536                    .unwrap()
13537                    .unwrap()
13538                    .connection_id,
13539                INCUMBENT
13540            );
13541        }
13542    }
13543
13544    /// The HELLO gate while the supervisor has a swap open: only the nonce it
13545    /// minted for the candidate admits a second process, into the candidate
13546    /// slot, and that check runs ahead of the reserved-module gate.
13547    mod swap_admission {
13548        use super::*;
13549
13550        const INCUMBENT_NONCE: &str = "incumbent-nonce";
13551        const CANDIDATE_NONCE: &str = "candidate-nonce";
13552
13553        fn handler_with_incumbent(
13554            module_id: &str,
13555            reserved: bool,
13556        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
13557            let registry = Arc::new(Registry::default());
13558            let supervisor = SupervisorHandle::new();
13559            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
13560            if reserved {
13561                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
13562            }
13563            let handler =
13564                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
13565            let incumbent = handler
13566                .handle_control(
13567                    ConnectionId::new(1),
13568                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
13569                )
13570                .unwrap();
13571            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
13572            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
13573            (registry, supervisor, handler)
13574        }
13575
13576        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
13577        /// admits every nonce, so while a swap is open the swap gate is the only
13578        /// thing between a key-holder and the candidate slot. A nonce the
13579        /// supervisor did not mint, or none at all, is refused, and neither the
13580        /// incumbent's registration nor the candidate slot moves.
13581        #[test]
13582        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
13583            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
13584
13585            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
13586                let replies = handler
13587                    .handle_control(
13588                        ConnectionId::new(connection),
13589                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
13590                    )
13591                    .unwrap();
13592                assert_eq!(replies[0].header.ty, FrameType::Error);
13593                assert_eq!(
13594                    parse_error(&replies[0])["code"],
13595                    "swap_token_invalid",
13596                    "nonce {nonce:?}"
13597                );
13598            }
13599            assert!(registry.get_candidate("aft").unwrap().is_none());
13600            assert_eq!(
13601                registry.get_module("aft").unwrap().unwrap().connection_id,
13602                ConnectionId::new(1)
13603            );
13604
13605            // Control: the minted token is admitted, into the candidate slot,
13606            // and only once.
13607            let admitted = handler
13608                .handle_control(
13609                    ConnectionId::new(4),
13610                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
13611                )
13612                .unwrap();
13613            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
13614            assert_eq!(
13615                registry
13616                    .get_candidate("aft")
13617                    .unwrap()
13618                    .unwrap()
13619                    .connection_id,
13620                ConnectionId::new(4)
13621            );
13622            assert_eq!(
13623                registry.get_module("aft").unwrap().unwrap().connection_id,
13624                ConnectionId::new(1),
13625                "the candidate must not take the active slot"
13626            );
13627            let replayed = handler
13628                .handle_control(
13629                    ConnectionId::new(5),
13630                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
13631                )
13632                .unwrap();
13633            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
13634
13635            // The case only this gate covers: the incumbent has died mid-swap,
13636            // so its duplicate refusal is gone too, and without the gate a
13637            // key-holder would take the id's ACTIVE slot.
13638            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
13639            let squatter = handler
13640                .handle_control(
13641                    ConnectionId::new(6),
13642                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
13643                )
13644                .unwrap();
13645            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
13646            assert!(
13647                registry.get_module("aft").unwrap().is_none(),
13648                "a squatter took the active slot of an id being swapped"
13649            );
13650        }
13651
13652        /// Design mutation arm (iii). A reserved module's candidate presents a
13653        /// nonce the reserved gate has never seen (that gate holds the
13654        /// incumbent's), so the swap gate must run first or the candidate is
13655        /// refused `reserved_module` and a reserved module can never be swapped.
13656        #[test]
13657        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
13658            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
13659
13660            let replies = handler
13661                .handle_control(
13662                    ConnectionId::new(2),
13663                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
13664                )
13665                .unwrap();
13666
13667            assert_eq!(
13668                replies[0].header.ty,
13669                FrameType::HelloAck,
13670                "reserved candidate refused: {:?}",
13671                serde_json::from_slice::<Value>(&replies[0].body).ok()
13672            );
13673            assert_eq!(
13674                registry
13675                    .get_candidate("vault")
13676                    .unwrap()
13677                    .unwrap()
13678                    .connection_id,
13679                ConnectionId::new(2)
13680            );
13681        }
13682
13683        /// With no swap open the gate is inert: the incumbent's reserved gate
13684        /// and duplicate refusal behave exactly as before.
13685        #[test]
13686        fn without_an_open_swap_the_ordinary_gates_decide() {
13687            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
13688            supervisor.close_swap("vault");
13689
13690            let candidate = handler
13691                .handle_control(
13692                    ConnectionId::new(2),
13693                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
13694                )
13695                .unwrap();
13696            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
13697            let duplicate = handler
13698                .handle_control(
13699                    ConnectionId::new(3),
13700                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
13701                )
13702                .unwrap();
13703            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13704            assert!(registry.get_candidate("vault").unwrap().is_none());
13705        }
13706    }
13707
13708    /// `scope.sync` and `scope.describe` through the real control handler: who
13709    /// may sync is decided by the registration and launch nonce of the module
13710    /// connection, never by the request body.
13711    mod scopes {
13712        mod activity {
13713            include!("activity_scope_tests.rs");
13714        }
13715        mod delta {
13716            include!("scope_handler_delta_tests.rs");
13717        }
13718        mod call_modes {
13719            include!("scope_call_mode_tests.rs");
13720        }
13721
13722        use subc_protocol::scope::{
13723            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
13724            ScopeStatus,
13725        };
13726
13727        use super::*;
13728
13729        const OWNER: &str = "prefrontal-core";
13730
13731        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
13732            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
13733        }
13734
13735        async fn call(
13736            handler: &ControlHandler,
13737            ctx: &RouteCtx,
13738            request: &ModuleControlRequestFromModule,
13739        ) -> Frame {
13740            let body = serde_json::to_vec(request).unwrap();
13741            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
13742            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
13743            assert_eq!(replies.len(), 1, "{replies:?}");
13744            replies.pop().unwrap()
13745        }
13746
13747        async fn sync(
13748            handler: &ControlHandler,
13749            ctx: &RouteCtx,
13750            generation: u64,
13751            scopes: Vec<ScopeRecord>,
13752        ) -> Result<ModuleControlResponseToModule, String> {
13753            let reply = call(
13754                handler,
13755                ctx,
13756                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
13757            )
13758            .await;
13759            match reply.header.ty {
13760                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
13761                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
13762            }
13763        }
13764
13765        async fn describe(
13766            handler: &ControlHandler,
13767            ctx: &RouteCtx,
13768            owner: &str,
13769            scope_ref: &str,
13770        ) -> ModuleControlResponseToModule {
13771            let reply = call(
13772                handler,
13773                ctx,
13774                &ModuleControlRequestFromModule::ScopeDescribe {
13775                    owner: Principal::Reserved {
13776                        module_id: owner.to_string(),
13777                    },
13778                    scope_ref: scope_ref.to_string(),
13779                },
13780            )
13781            .await;
13782            assert_eq!(
13783                reply.header.ty,
13784                FrameType::Response,
13785                "{:?}",
13786                parse_error(&reply)
13787            );
13788            serde_json::from_slice(&reply.body).unwrap()
13789        }
13790
13791        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
13792        async fn module(
13793            handler: &ControlHandler,
13794            connection: u64,
13795            module_id: &str,
13796            nonce: Option<&str>,
13797        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13798            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
13799            hello_via_sink(
13800                handler,
13801                &ctx,
13802                &mut rx,
13803                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
13804            )
13805            .await;
13806            (ctx, rx)
13807        }
13808
13809        /// `direct` and every other client connection has no registration, so
13810        /// it can neither sync nor own a scope.
13811        #[tokio::test]
13812        async fn a_client_connection_cannot_sync_or_describe() {
13813            let handler = ControlHandler::new(Arc::new(Registry::default()));
13814            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
13815            for request in [
13816                ModuleControlRequestFromModule::ScopeSync {
13817                    generation: 1,
13818                    scopes: vec![head("s", 1)],
13819                },
13820                ModuleControlRequestFromModule::ScopeDescribe {
13821                    owner: Principal::Direct,
13822                    scope_ref: "s".to_string(),
13823                },
13824            ] {
13825                let reply = call(&handler, &ctx, &request).await;
13826                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
13827            }
13828            assert!(
13829                !handler
13830                    .scopes
13831                    .read()
13832                    .unwrap()
13833                    .describe(
13834                        &Principal::Reserved {
13835                            module_id: OWNER.to_string()
13836                        },
13837                        "s"
13838                    )
13839                    .owner_synced
13840            );
13841        }
13842
13843        /// A module the supervisor did not spawn registers without a launch
13844        /// nonce, so it is never an owner's current launch.
13845        #[tokio::test]
13846        async fn a_module_without_a_supervised_launch_cannot_sync() {
13847            let handler = ControlHandler::new(Arc::new(Registry::default()));
13848            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
13849            assert_eq!(
13850                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
13851                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13852            );
13853        }
13854
13855        #[tokio::test]
13856        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
13857            let supervisor = SupervisorHandle::new();
13858            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13859            let handler = ControlHandler::new(Arc::new(Registry::default()))
13860                .with_supervisor(supervisor.clone());
13861            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13862            sync(&handler, &incumbent, 1, vec![head("s", 1)])
13863                .await
13864                .expect("the current launch syncs");
13865
13866            // A swap candidate registers with the swap token and is refused
13867            // while the incumbent keeps syncing.
13868            supervisor.open_swap(OWNER, "n2".to_string());
13869            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
13870            assert_eq!(
13871                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13872                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13873            );
13874            sync(&handler, &incumbent, 2, vec![head("s", 1)])
13875                .await
13876                .expect("the serving owner syncs during the swap");
13877
13878            // The swap fails and is rolled back. The candidate never held sync
13879            // authority, and still cannot sync.
13880            supervisor.close_swap(OWNER);
13881            assert_eq!(
13882                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13883                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13884            );
13885            sync(&handler, &incumbent, 3, vec![head("s", 1)])
13886                .await
13887                .expect("the serving owner syncs after the rollback");
13888            handler.cleanup_connection(candidate.connection_id).unwrap();
13889
13890            // A swap that cuts over. Promotion records the candidate's nonce as
13891            // the module's spawn nonce, which is what `set_spawn_nonce` does
13892            // here; the promoted connection then takes authority at any
13893            // generation and the superseded incumbent is refused.
13894            supervisor.open_swap(OWNER, "n3".to_string());
13895            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
13896            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
13897            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
13898                .await
13899                .expect("the promoted launch takes authority");
13900            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13901                panic!("unexpected reply {reply:?}");
13902            };
13903            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
13904            assert_eq!(
13905                sync(&handler, &incumbent, 4, Vec::new()).await,
13906                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13907            );
13908        }
13909
13910        /// Authority dies with its connection: the cleanup path releases it,
13911        /// so the owner's next connection takes it at any generation.
13912        #[tokio::test]
13913        async fn closing_the_authority_connection_frees_sync_authority() {
13914            let supervisor = SupervisorHandle::new();
13915            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13916            let handler = ControlHandler::new(Arc::new(Registry::default()))
13917                .with_supervisor(supervisor.clone());
13918            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13919            sync(&handler, &first, 10, vec![head("s", 1)])
13920                .await
13921                .unwrap();
13922            handler.cleanup_connection(first.connection_id).unwrap();
13923
13924            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
13925            sync(&handler, &second, 1, vec![head("s", 1)])
13926                .await
13927                .expect("the next connection takes the released authority");
13928        }
13929
13930        #[tokio::test]
13931        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
13932            let supervisor = SupervisorHandle::new();
13933            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13934            let handler =
13935                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
13936            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
13937            sync(&handler, &first, 10, vec![head("s", 1)])
13938                .await
13939                .unwrap();
13940            handler
13941                .handle_control_frame(
13942                    &first,
13943                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
13944                )
13945                .await
13946                .unwrap();
13947            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
13948            sync(&handler, &second, 1, vec![head("s", 1)])
13949                .await
13950                .expect("GOODBYE releases authority even if the old socket remains open");
13951        }
13952
13953        #[tokio::test]
13954        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
13955            let registry = Arc::new(Registry::default());
13956            let supervisor_handle = SupervisorHandle::new();
13957            let supervisor =
13958                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
13959                    .with_handle(supervisor_handle.clone())
13960                    .with_daemon_incarnation("incarnation-7".to_string());
13961            // Configured with enabled: false, so the supervisor lists the
13962            // module without spawning a process for it.
13963            supervisor
13964                .supervise_configured(
13965                    ModuleSpec {
13966                        module_id: OWNER.to_string(),
13967                        program: PathBuf::from("/nonexistent/prefrontal-core"),
13968                        args: Vec::new(),
13969                        env: Vec::new(),
13970                        reserved: false,
13971                        reserved_prefixes: Vec::new(),
13972                        protocol: ModuleProtocol::Subc,
13973                        overlap: Default::default(),
13974                    },
13975                    false,
13976                )
13977                .unwrap();
13978            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
13979            let handler =
13980                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
13981            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
13982
13983            // Configured but not yet synced: a reader waits for the owner.
13984            let ModuleControlResponseToModule::ScopeDescribe {
13985                status,
13986                daemon_incarnation,
13987                owner_synced,
13988                owner_configured,
13989                scope,
13990                ..
13991            } = describe(&handler, &reader, OWNER, "s").await
13992            else {
13993                panic!("not a describe reply");
13994            };
13995            assert_eq!(status, ScopeStatus::NotLive);
13996            assert_eq!(daemon_incarnation, "incarnation-7");
13997            assert!(!owner_synced);
13998            assert!(owner_configured);
13999            assert!(scope.is_none());
14000
14001            // Not a supervised module: the owner will never sync, and a reader
14002            // refuses rather than waits.
14003            let ModuleControlResponseToModule::ScopeDescribe {
14004                status,
14005                owner_configured,
14006                ..
14007            } = describe(&handler, &reader, "ghost", "s").await
14008            else {
14009                panic!("not a describe reply");
14010            };
14011            assert_eq!(status, ScopeStatus::NotLive);
14012            assert!(!owner_configured);
14013
14014            // Live, with the stamp fields and the computed owner_authorized.
14015            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
14016            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
14017            let ModuleControlResponseToModule::ScopeDescribe {
14018                status,
14019                scope_epoch,
14020                owner_synced,
14021                scope,
14022                ..
14023            } = describe(&handler, &reader, OWNER, "s").await
14024            else {
14025                panic!("not a describe reply");
14026            };
14027            assert_eq!(status, ScopeStatus::Live);
14028            assert_eq!(scope_epoch, Some(4));
14029            assert!(owner_synced);
14030            let stamp = scope.expect("a live scope carries its stamp");
14031            assert!(
14032                stamp.owner_authorized,
14033                "prefrontal-core is the default authority"
14034            );
14035            assert_eq!(stamp.kind, ScopeKind::Head);
14036        }
14037
14038        #[tokio::test]
14039        async fn scope_authority_owners_decides_owner_authorized() {
14040            let supervisor = SupervisorHandle::new();
14041            supervisor.set_spawn_nonce("broca", "b1".to_string());
14042            let handler = ControlHandler::new(Arc::new(Registry::default()))
14043                .with_supervisor(supervisor)
14044                .with_scope_authority_owners(vec!["broca".to_string()]);
14045            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
14046            let mut gated = head("s", 1);
14047            gated.attributes.agent_id = Some("agent".to_string());
14048            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
14049            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
14050                describe(&handler, &broca, "broca", "s").await
14051            else {
14052                panic!("not a describe reply");
14053            };
14054            assert!(scope.unwrap().owner_authorized);
14055        }
14056
14057        /// With route admission, the stamp, the commit re-check and drains in
14058        /// place, the feature is advertised: the module ops in HELLO_ACK, and
14059        /// `scopes/v1` in HELLO_ACK and `server.describe`.
14060        #[tokio::test]
14061        async fn scope_ops_and_the_scopes_capability_are_advertised() {
14062            let handler = ControlHandler::new(Arc::new(Registry::default()));
14063            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
14064            let ack = hello_via_sink(
14065                &handler,
14066                &ctx,
14067                &mut rx,
14068                hello_frame("m", PROTOCOL_VERSION, 1),
14069            )
14070            .await;
14071            let ack = parse_ack(&ack);
14072            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
14073                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
14074            }
14075            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
14076
14077            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
14078            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14079            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14080            let reply = handler
14081                .handle_control_frame(&client, frame)
14082                .await
14083                .unwrap()
14084                .pop()
14085                .unwrap();
14086            let ClientControlResponse::ServerDescribe { capabilities, .. } =
14087                serde_json::from_slice(&reply.body).unwrap()
14088            else {
14089                panic!("not a server.describe reply");
14090            };
14091            assert!(
14092                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
14093                "{capabilities:?}"
14094            );
14095        }
14096
14097        // ---- route admission, stamps, commit re-check and drains ----------
14098
14099        const PLEXUS: &str = "plexus";
14100        const OTHER: &str = "other";
14101        const AFT: &str = "aft";
14102        const BROCA: &str = "broca";
14103        const MAGIC: &str = "magic-context";
14104
14105        fn nonce(module_id: &str) -> String {
14106            format!("nonce-{module_id}")
14107        }
14108
14109        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14110            let (tx, rx) = mpsc::channel(64);
14111            (
14112                RouteCtx {
14113                    connection_id: ConnectionId::new(connection),
14114                    egress: FrameSink::new(tx),
14115                },
14116                rx,
14117            )
14118        }
14119
14120        /// A daemon with a configured owner (prefrontal-core) registered on its
14121        /// own module connection, two routable targets (plexus, other), and
14122        /// launch nonces minted for the modules that open routes as carriers.
14123        struct Rig {
14124            handler: ControlHandler,
14125            forwarding: Arc<ForwardingTable>,
14126            owner: RouteCtx,
14127            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14128            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
14129            generation: u64,
14130            next_connection: u64,
14131            _supervisor: Supervisor,
14132        }
14133
14134        async fn rig() -> Rig {
14135            rig_with_flow_support(true).await
14136        }
14137
14138        async fn rig_with_flow_support(flow_support: bool) -> Rig {
14139            rig_with_scope_capabilities(flow_support, false).await
14140        }
14141
14142        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
14143            let mut provides = Vec::new();
14144            if flow_support {
14145                provides.push("flow-scopes/v1");
14146            }
14147            if run_support {
14148                provides.push("agent-run-scopes/v1");
14149            }
14150            rig_with_target_capabilities(&provides).await
14151        }
14152
14153        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
14154            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
14155            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
14156            // Decoder versions alone cannot admit scoped calls: only a capability
14157            // declaration promises the required run or flow behaviour.
14158            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
14159            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
14160            Frame::build(
14161                FrameType::Hello,
14162                control_flags(),
14163                0,
14164                0,
14165                connection,
14166                serde_json::to_vec(&body).unwrap(),
14167            )
14168            .unwrap()
14169        }
14170
14171        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
14172            let registry = Arc::new(Registry::default());
14173            let forwarding = Arc::new(ForwardingTable::default());
14174            let supervisor_handle = SupervisorHandle::new();
14175            let supervisor =
14176                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
14177                    .with_handle(supervisor_handle.clone());
14178            supervisor
14179                .supervise_configured(
14180                    ModuleSpec {
14181                        module_id: OWNER.to_string(),
14182                        program: PathBuf::from("/nonexistent/prefrontal-core"),
14183                        args: Vec::new(),
14184                        env: Vec::new(),
14185                        reserved: false,
14186                        reserved_prefixes: Vec::new(),
14187                        protocol: ModuleProtocol::Subc,
14188                        overlap: Default::default(),
14189                    },
14190                    false,
14191                )
14192                .unwrap();
14193            for module_id in [OWNER, AFT, BROCA, MAGIC] {
14194                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
14195            }
14196            let handler =
14197                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14198                    .with_supervisor(supervisor_handle);
14199            let (owner, mut owner_rx) = wide_ctx(1);
14200            hello_via_sink(
14201                &handler,
14202                &owner,
14203                &mut owner_rx,
14204                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
14205            )
14206            .await;
14207            let mut modules = BTreeMap::new();
14208            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
14209                let (ctx, mut rx) = wide_ctx(connection);
14210                let hello = hello_with_capabilities(module_id, connection, provides);
14211                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
14212                modules.insert(module_id.to_string(), (ctx, rx));
14213            }
14214            Rig {
14215                handler,
14216                forwarding,
14217                owner,
14218                _owner_rx: owner_rx,
14219                modules,
14220                generation: 0,
14221                next_connection: 100,
14222                _supervisor: supervisor,
14223            }
14224        }
14225
14226        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
14227            ScopeCarrier::new(Principal::Reserved {
14228                module_id: module_id.to_string(),
14229            })
14230            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
14231        }
14232
14233        /// The scope most tests open under: aft carries to any module, broca
14234        /// only to plexus and other, and the owner delegates as agent-1.
14235        fn session(scope_epoch: u64) -> ScopeRecord {
14236            let mut record = head("s", scope_epoch);
14237            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
14238            record.attributes.agent_id = Some("agent-1".to_string());
14239            record.attributes.delegates = true;
14240            record
14241        }
14242
14243        impl Rig {
14244            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
14245                self.generation += 1;
14246                sync(&self.handler, &self.owner, self.generation, scopes)
14247                    .await
14248                    .expect("the owner's sync is accepted");
14249            }
14250
14251            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
14252                ScopeSelector {
14253                    owner: Principal::Reserved {
14254                        module_id: OWNER.to_string(),
14255                    },
14256                    scope_ref: scope_ref.to_string(),
14257                    scope_epoch,
14258                }
14259            }
14260
14261            fn open_frame(
14262                &mut self,
14263                opener: Option<&str>,
14264                target: &str,
14265                scope: Option<ScopeSelector>,
14266            ) -> (
14267                RouteCtx,
14268                mpsc::Receiver<crate::router::OutboundFrame>,
14269                Frame,
14270            ) {
14271                self.next_connection += 1;
14272                let (ctx, rx) = wide_ctx(self.next_connection);
14273                let root = unique_project_root("scoped-open");
14274                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
14275                    target: RouteTarget::ToolProvider {
14276                        module_id: target.to_string(),
14277                    },
14278                    identity: BindIdentity::new(
14279                        root.path().to_path_buf(),
14280                        "unit".to_string(),
14281                        "session".to_string(),
14282                    ),
14283                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
14284                        module_id: module_id.to_string(),
14285                        launch_nonce: nonce(module_id),
14286                    }),
14287                    consumer_capabilities: None,
14288                    role_versions: None,
14289                    admission_facts: None,
14290                    scope,
14291                })
14292                .unwrap();
14293                let frame = Frame::build(
14294                    FrameType::Request,
14295                    control_flags(),
14296                    0,
14297                    0,
14298                    self.next_connection,
14299                    body,
14300                )
14301                .unwrap();
14302                (ctx, rx, frame)
14303            }
14304
14305            /// Open and expect a refusal before anything is relayed.
14306            async fn refused(
14307                &mut self,
14308                opener: Option<&str>,
14309                target: &str,
14310                scope: Option<ScopeSelector>,
14311            ) -> String {
14312                self.refusal_body(opener, target, scope).await["code"]
14313                    .as_str()
14314                    .unwrap()
14315                    .to_string()
14316            }
14317
14318            async fn refusal_body(
14319                &mut self,
14320                opener: Option<&str>,
14321                target: &str,
14322                scope: Option<ScopeSelector>,
14323            ) -> Value {
14324                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
14325                let replies = tokio::time::timeout(
14326                    Duration::from_secs(2),
14327                    self.handler.handle_control_frame(&ctx, frame),
14328                )
14329                .await
14330                .expect("the open must be refused before waiting for a bind ack")
14331                .unwrap();
14332                assert_eq!(replies.len(), 1, "{replies:?}");
14333                assert_eq!(replies[0].header.ty, FrameType::Error);
14334                let (_, module_rx) = self.modules.get_mut(target).unwrap();
14335                assert!(
14336                    module_rx.try_recv().is_err(),
14337                    "a refused open relays nothing"
14338                );
14339                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
14340                parse_error(&replies[0])
14341            }
14342
14343            /// Start an open and return its task and the bind the target got.
14344            async fn relayed(
14345                &mut self,
14346                opener: Option<&str>,
14347                target: &str,
14348                scope: Option<ScopeSelector>,
14349            ) -> Relayed {
14350                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
14351                let handler = self.handler.clone();
14352                let task_ctx = ctx.clone();
14353                let task = tokio::spawn(async move {
14354                    handler
14355                        .handle_control_frame(&task_ctx, frame)
14356                        .await
14357                        .unwrap()
14358                });
14359                let (_, module_rx) = self.modules.get_mut(target).unwrap();
14360                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
14361                    .await
14362                    .expect("the target receives the relayed route.bind")
14363                    .unwrap()
14364                    .frame;
14365                Relayed {
14366                    target: target.to_string(),
14367                    client: ctx,
14368                    client_rx: rx,
14369                    task,
14370                    bind,
14371                }
14372            }
14373
14374            async fn ack(&self, relayed: &Relayed) {
14375                let (module, _) = &self.modules[&relayed.target];
14376                self.handler
14377                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
14378                    .await
14379                    .unwrap();
14380            }
14381
14382            /// Open, ack and return the bound route.
14383            async fn bound(
14384                &mut self,
14385                opener: Option<&str>,
14386                target: &str,
14387                scope: Option<ScopeSelector>,
14388            ) -> Bound {
14389                let relayed = self.relayed(opener, target, scope).await;
14390                self.ack(&relayed).await;
14391                let Relayed {
14392                    target,
14393                    client,
14394                    mut client_rx,
14395                    task,
14396                    bind,
14397                } = relayed;
14398                assert!(
14399                    task.await.unwrap().is_empty(),
14400                    "the open is answered by commit"
14401                );
14402                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
14403                Bound {
14404                    target,
14405                    client,
14406                    client_rx,
14407                    channel,
14408                    epoch,
14409                    bind,
14410                }
14411            }
14412
14413            fn live(&self, route: &Bound) -> bool {
14414                matches!(
14415                    self.forwarding
14416                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
14417                        .unwrap(),
14418                    DataRoute::Client(DataRouteState::Bound(_))
14419                )
14420            }
14421        }
14422
14423        struct Relayed {
14424            target: String,
14425            client: RouteCtx,
14426            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14427            task: tokio::task::JoinHandle<Vec<Frame>>,
14428            bind: Frame,
14429        }
14430
14431        struct Bound {
14432            target: String,
14433            client: RouteCtx,
14434            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14435            channel: u16,
14436            epoch: u32,
14437            bind: Frame,
14438        }
14439
14440        impl Bound {
14441            /// The reason of the `route.closed` this client was sent, after
14442            /// checking it also got a GOODBYE on exactly this route.
14443            fn closed_reason(&mut self) -> RouteCloseReason {
14444                let mut reason = None;
14445                let mut goodbye = false;
14446                while let Ok(outbound) = self.client_rx.try_recv() {
14447                    let frame = outbound.frame;
14448                    match frame.header.ty {
14449                        FrameType::Goodbye => {
14450                            assert_eq!(
14451                                (frame.header.channel, frame.header.epoch),
14452                                (self.channel, self.epoch)
14453                            );
14454                            goodbye = true;
14455                        }
14456                        FrameType::Push => {
14457                            let ClientControlPush::RouteClosed {
14458                                reason: r,
14459                                module_id,
14460                                ..
14461                            } = serde_json::from_slice(&frame.body).unwrap()
14462                            else {
14463                                panic!("unexpected push");
14464                            };
14465                            assert_eq!(module_id, self.target);
14466                            reason = Some(r);
14467                        }
14468                        other => panic!("unexpected frame {other:?}"),
14469                    }
14470                }
14471                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
14472                reason.expect("the client is told why the route closed")
14473            }
14474
14475            fn untouched(&mut self) -> bool {
14476                self.client_rx.try_recv().is_err()
14477            }
14478
14479            fn stamp(&self) -> Option<ScopeStamp> {
14480                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
14481                    ModuleControlRequest::RouteBind { scope, .. } => scope,
14482                    other => panic!("expected a route.bind, got {other:?}"),
14483                }
14484            }
14485        }
14486
14487        #[tokio::test]
14488        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
14489        ) {
14490            let mut rig = rig().await;
14491            let mut record = session(1);
14492            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
14493            record.child_owners = vec![Principal::Reserved {
14494                module_id: MAGIC.to_string(),
14495            }];
14496            rig.sync(vec![record]).await;
14497            let scope = || Some(rig_selector("s", Some(1)));
14498
14499            // Admitted: the owner, a bare carrier to any module, a targeted
14500            // carrier to its listed module.
14501            rig.bound(Some(OWNER), PLEXUS, scope()).await;
14502            rig.bound(Some(AFT), OTHER, scope()).await;
14503            rig.bound(Some(BROCA), PLEXUS, scope()).await;
14504
14505            // Refused scope_not_carrier: a targeted carrier to an unlisted
14506            // module, a module that is not listed at all (a child owner is not
14507            // a carrier), and a direct key-holder.
14508            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
14509                assert_eq!(
14510                    rig.refused(opener, target, scope()).await,
14511                    error_codes::SCOPE_NOT_CARRIER,
14512                    "{opener:?} -> {target}"
14513                );
14514            }
14515        }
14516
14517        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
14518            ScopeSelector {
14519                owner: Principal::Reserved {
14520                    module_id: OWNER.to_string(),
14521                },
14522                scope_ref: scope_ref.to_string(),
14523                scope_epoch,
14524            }
14525        }
14526
14527        #[tokio::test]
14528        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
14529            let mut rig = rig().await;
14530            rig.sync(vec![session(1)]).await;
14531            for opener in [OWNER, AFT] {
14532                assert_eq!(
14533                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
14534                        .await,
14535                    error_codes::SCOPE_EPOCH_REQUIRED,
14536                    "{opener}"
14537                );
14538            }
14539        }
14540
14541        #[tokio::test]
14542        async fn admission_separates_not_synced_not_live_and_ended() {
14543            let mut rig = rig().await;
14544            // Before the configured owner's first sync: retryable.
14545            let code = rig
14546                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14547                .await;
14548            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
14549            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
14550
14551            // An owner that is not configured will never sync: terminal.
14552            let ghost = ScopeSelector {
14553                owner: Principal::Reserved {
14554                    module_id: "ghost".to_string(),
14555                },
14556                scope_ref: "s".to_string(),
14557                scope_epoch: Some(1),
14558            };
14559            assert_eq!(
14560                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
14561                error_codes::SCOPE_NOT_LIVE
14562            );
14563
14564            rig.sync(vec![session(2)]).await;
14565            assert_eq!(
14566                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
14567                    .await,
14568                error_codes::SCOPE_NOT_LIVE
14569            );
14570            for epoch in [1, 3] {
14571                assert_eq!(
14572                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
14573                        .await,
14574                    error_codes::SCOPE_ENDED,
14575                    "epoch {epoch}"
14576                );
14577            }
14578            // Control: the live epoch is admitted.
14579            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
14580                .await;
14581        }
14582
14583        #[tokio::test]
14584        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
14585            let mut rig = rig().await;
14586            rig.sync(vec![session(1)]).await;
14587            let route = rig
14588                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14589                .await;
14590            let stamp = route.stamp().expect("a scoped bind carries the stamp");
14591            assert_eq!(stamp.scope_ref, "s");
14592            assert_eq!(stamp.scope_epoch, 1);
14593            assert_eq!(stamp.kind, ScopeKind::Head);
14594            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
14595            assert!(stamp.attributes.delegates);
14596            assert!(stamp.owner_authorized);
14597            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14598            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
14599
14600            // broca owns a scope of its own on its own module connection; it is
14601            // not in scope_authority_owners, so its stamp is not authorized.
14602            let (broca, mut broca_rx) = wide_ctx(50);
14603            hello_via_sink(
14604                &rig.handler,
14605                &broca,
14606                &mut broca_rx,
14607                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
14608            )
14609            .await;
14610            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
14611                .await
14612                .unwrap();
14613            let own = ScopeSelector {
14614                owner: Principal::Reserved {
14615                    module_id: BROCA.to_string(),
14616                },
14617                scope_ref: "b".to_string(),
14618                scope_epoch: Some(1),
14619            };
14620            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
14621            assert!(!route.stamp().unwrap().owner_authorized);
14622        }
14623
14624        #[tokio::test]
14625        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
14626            let mut rig = rig().await;
14627            let mut record = head("s", 1);
14628            record.carriers = vec![carrier(AFT, None)];
14629            let flow_id = "Flow:run-7/step_2!~";
14630            record.attributes.flow_id = Some(flow_id.to_string());
14631            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
14632                .await
14633                .unwrap();
14634            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14635                panic!("not a sync reply");
14636            };
14637            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
14638            let route = rig
14639                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14640                .await;
14641            assert!(rig.live(&route), "the stamped bind committed");
14642            let stamp = route.stamp().expect("a flow scope carries a stamp");
14643            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
14644            assert_eq!(stamp.attributes.agent_id, None);
14645            assert!(!stamp.attributes.delegates);
14646            assert!(stamp.owner_authorized);
14647            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14648            assert_eq!(unscoped.stamp(), None);
14649        }
14650
14651        #[tokio::test]
14652        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
14653            let mut rig = rig_with_flow_support(false).await;
14654            let mut record = session(1);
14655            record.attributes.flow_id = Some("flow:7".to_string());
14656            rig.sync(vec![record]).await;
14657            for opener in [OWNER, AFT] {
14658                let body = rig
14659                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
14660                    .await;
14661                assert_eq!(body["code"], "target_flow_unsupported");
14662                let message = body["message"].as_str().unwrap();
14663                for required in [PLEXUS, "flow-scopes/v1"] {
14664                    assert!(message.contains(required), "{message}");
14665                }
14666            }
14667        }
14668
14669        #[tokio::test]
14670        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
14671            let mut rig = rig_with_flow_support(true).await;
14672            let mut record = session(1);
14673            record.attributes.flow_id = Some("flow:7".to_string());
14674            rig.sync(vec![record]).await;
14675            for opener in [OWNER, AFT] {
14676                let route = rig
14677                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
14678                    .await;
14679                assert!(rig.live(&route));
14680                assert_eq!(
14681                    route.stamp().unwrap().attributes.flow_id.as_deref(),
14682                    Some("flow:7")
14683                );
14684            }
14685        }
14686
14687        #[tokio::test]
14688        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
14689            use std::future::Future;
14690
14691            let mut rig = rig_with_flow_support(true).await;
14692            let mut record = session(1);
14693            record.attributes.flow_id = Some("flow:7".to_string());
14694            rig.sync(vec![record]).await;
14695            let (client, mut client_rx, frame) =
14696                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
14697            // Hold the route-open response permit so admission sees the first
14698            // target but relay reservation cannot capture an endpoint yet.
14699            for _ in 0..64 {
14700                client.egress.try_send(route_bind_ack(1)).unwrap();
14701            }
14702            let handler = rig.handler.clone();
14703            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
14704            std::future::poll_fn(|cx| {
14705                assert!(open.as_mut().poll(cx).is_pending());
14706                std::task::Poll::Ready(())
14707            })
14708            .await;
14709            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14710
14711            let old_connection = rig.modules[PLEXUS].0.connection_id;
14712            rig.handler.cleanup_connection(old_connection).unwrap();
14713            let (replacement, mut replacement_rx) = wide_ctx(200);
14714            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
14715            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
14716            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
14717            let hello = Frame::build(
14718                FrameType::Hello,
14719                control_flags(),
14720                0,
14721                0,
14722                200,
14723                serde_json::to_vec(&body).unwrap(),
14724            )
14725            .unwrap();
14726            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
14727
14728            client_rx.try_recv().unwrap();
14729            let replies = tokio::time::timeout(Duration::from_secs(2), open)
14730                .await
14731                .expect("the replacement is refused without waiting for a bind ack")
14732                .unwrap();
14733            assert_eq!(replies.len(), 1);
14734            let body = parse_error(&replies[0]);
14735            assert_eq!(body["code"], "target_flow_unsupported");
14736            for required in [PLEXUS, "flow-scopes/v1"] {
14737                assert!(body["message"].as_str().unwrap().contains(required));
14738            }
14739            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
14740            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
14741            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14742            assert!(rig
14743                .handler
14744                .registry
14745                .get_module_by_connection(replacement.connection_id)
14746                .unwrap()
14747                .is_some());
14748        }
14749
14750        #[tokio::test]
14751        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
14752        {
14753            let mut rig = rig_with_flow_support(false).await;
14754            rig.sync(vec![session(1)]).await;
14755            let route = rig
14756                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14757                .await;
14758            assert!(rig.live(&route));
14759            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
14760            let mut record = session(1);
14761            record.attributes.flow_id = Some("flow:7".to_string());
14762            rig.sync(vec![record]).await;
14763            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
14764            assert!(rig.live(&unscoped));
14765            assert_eq!(unscoped.stamp(), None);
14766        }
14767
14768        #[tokio::test]
14769        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
14770            let mut rig = rig().await;
14771            let mut record = session(1);
14772            record.attributes.flow_id = Some("flow:7".to_string());
14773            rig.sync(vec![record.clone()]).await;
14774            let before = rig.forwarding.published_scope_tag(OWNER, "s");
14775            let mut owner_route = rig
14776                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14777                .await;
14778            let mut carrier_route = rig
14779                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
14780                .await;
14781            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14782            rig.sync(vec![record.clone()]).await;
14783            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14784            assert!(rig.live(&owner_route) && owner_route.untouched());
14785            assert!(rig.live(&carrier_route) && carrier_route.untouched());
14786
14787            record.attributes.flow_id = Some("flow:8".to_string());
14788            rig.sync(vec![record]).await;
14789            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
14790            let before = before.unwrap();
14791            assert_eq!(after.scope_epoch, before.scope_epoch);
14792            assert!(after.version > before.version);
14793            for route in [&mut owner_route, &mut carrier_route] {
14794                assert!(!rig.live(route));
14795                assert_eq!(
14796                    route.closed_reason(),
14797                    RouteCloseReason::ScopeDelegationChanged
14798                );
14799            }
14800            assert!(rig.live(&unscoped) && unscoped.untouched());
14801            // Each provider also receives a GOODBYE for its drained route;
14802            // consume it before expecting the next route.bind on that sink.
14803            for target in [PLEXUS, OTHER] {
14804                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
14805                let goodbye = module_rx
14806                    .try_recv()
14807                    .expect("the provider sees the drain")
14808                    .frame;
14809                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14810                assert!(module_rx.try_recv().is_err());
14811            }
14812            let rebound = rig
14813                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14814                .await;
14815            assert_eq!(
14816                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
14817                Some("flow:8")
14818            );
14819        }
14820
14821        /// The owner's sync lands between admission and the module's ack. The
14822        /// open is refused by name, the module's other routes stay up, and the
14823        /// reserved pair is released. Changed content is retryable; an ended
14824        /// scope is not.
14825        #[tokio::test]
14826        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
14827            let mut rig = rig().await;
14828            rig.sync(vec![session(1)]).await;
14829            let mut cotenant = rig
14830                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14831                .await;
14832
14833            let mut changed = session(1);
14834            changed.child_owners.push(Principal::Reserved {
14835                module_id: MAGIC.to_string(),
14836            });
14837            let mut ended = None;
14838            for (code, next) in [
14839                (error_codes::SCOPE_CHANGED, vec![changed]),
14840                (error_codes::SCOPE_ENDED, Vec::new()),
14841            ] {
14842                let relayed = rig
14843                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14844                    .await;
14845                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
14846                ended = Some(next.is_empty());
14847                rig.sync(next).await;
14848                rig.ack(&relayed).await;
14849                let replies = relayed.task.await.unwrap();
14850                assert_eq!(replies.len(), 1, "{replies:?}");
14851                assert_eq!(parse_error(&replies[0])["code"], code);
14852                assert_eq!(
14853                    subc_protocol::error_codes::is_retryable_route_open(code),
14854                    code == error_codes::SCOPE_CHANGED
14855                );
14856                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14857                // The module is told to drop just the binding it created.
14858                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14859                // Collected, because ending the scope also closes the co-tenant
14860                // route, whose GOODBYE comes first.
14861                let mut goodbyes = Vec::new();
14862                while let Ok(outbound) = plexus_rx.try_recv() {
14863                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
14864                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
14865                }
14866                assert!(
14867                    goodbyes.contains(&(bind_channel, bind_epoch)),
14868                    "{goodbyes:?}"
14869                );
14870                assert!(rig
14871                    .handler
14872                    .registry
14873                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
14874                    .unwrap()
14875                    .is_some());
14876            }
14877            assert_eq!(ended, Some(true));
14878            // The co-tenant stayed up through the change, and closed only when
14879            // the scope ended, by the drain rule rather than by the commit.
14880            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
14881        }
14882
14883        /// Each row of the drain table on one set of routes: the owner's, a
14884        /// bare carrier's, and a targeted carrier's to each of its targets.
14885        #[tokio::test]
14886        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
14887            struct Case {
14888                name: &'static str,
14889                change: fn(&mut ScopeRecord),
14890                /// Closed routes by index: owner->plexus, aft->plexus,
14891                /// broca->plexus, broca->other.
14892                closed: [Option<RouteCloseReason>; 4],
14893            }
14894            use RouteCloseReason::*;
14895            let cases = [
14896                Case {
14897                    name: "a carrier entry removed",
14898                    change: |r| {
14899                        r.carriers.retain(|c| {
14900                            c.principal
14901                                != Principal::Reserved {
14902                                    module_id: AFT.to_string(),
14903                                }
14904                        })
14905                    },
14906                    closed: [None, Some(ScopeCarrierRemoved), None, None],
14907                },
14908                Case {
14909                    name: "a target removed from a carrier",
14910                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
14911                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
14912                },
14913                Case {
14914                    name: "a bare carrier narrowed to targets",
14915                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
14916                    closed: [None, Some(ScopeCarrierRemoved), None, None],
14917                },
14918                Case {
14919                    name: "delegates turned off",
14920                    change: |r| r.attributes.delegates = false,
14921                    closed: [Some(ScopeDelegationChanged); 4],
14922                },
14923                Case {
14924                    name: "agent_id changed",
14925                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
14926                    closed: [Some(ScopeDelegationChanged); 4],
14927                },
14928                Case {
14929                    name: "a carrier added, child owners changed, the record re-sent",
14930                    change: |r| {
14931                        r.carriers.push(carrier(MAGIC, None));
14932                        r.child_owners.push(Principal::Reserved {
14933                            module_id: MAGIC.to_string(),
14934                        });
14935                    },
14936                    closed: [None; 4],
14937                },
14938                Case {
14939                    name: "a target added",
14940                    change: |r| {
14941                        r.carriers[1]
14942                            .targets
14943                            .as_mut()
14944                            .unwrap()
14945                            .push("third".to_string())
14946                    },
14947                    closed: [None; 4],
14948                },
14949                Case {
14950                    name: "delegates turned on",
14951                    change: |r| r.attributes.delegates = true,
14952                    closed: [None; 4],
14953                },
14954            ];
14955            for case in cases {
14956                let mut rig = rig().await;
14957                rig.sync(vec![session(1)]).await;
14958                let scope = || Some(rig_selector("s", Some(1)));
14959                let mut routes = [
14960                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
14961                    rig.bound(Some(AFT), PLEXUS, scope()).await,
14962                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
14963                    rig.bound(Some(BROCA), OTHER, scope()).await,
14964                ];
14965                let mut record = session(1);
14966                (case.change)(&mut record);
14967                rig.sync(vec![record]).await;
14968                for (index, expected) in case.closed.iter().enumerate() {
14969                    let route = &mut routes[index];
14970                    match expected {
14971                        Some(reason) => {
14972                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
14973                            assert_eq!(
14974                                route.closed_reason(),
14975                                *reason,
14976                                "{}: route {index}",
14977                                case.name
14978                            );
14979                        }
14980                        None => {
14981                            assert!(rig.live(route), "{}: route {index} closed", case.name);
14982                            assert!(
14983                                route.untouched(),
14984                                "{}: route {index} was told something",
14985                                case.name
14986                            );
14987                        }
14988                    }
14989                }
14990            }
14991        }
14992
14993        #[tokio::test]
14994        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
14995            // Removed, and replaced by a higher epoch.
14996            for next in [Vec::new(), vec![session(2)]] {
14997                let mut rig = rig().await;
14998                rig.sync(vec![session(1)]).await;
14999                let mut route = rig
15000                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15001                    .await;
15002                rig.sync(next).await;
15003                assert!(!rig.live(&route));
15004                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
15005            }
15006
15007            // A child whose parent ends: its routes close as parent-ended, the
15008            // child stays live, and routes under the parent close as ended.
15009            let mut rig = rig().await;
15010            let mut child = session(1);
15011            child.scope_ref = "child".to_string();
15012            child.kind = ScopeKind::Worker;
15013            child.parent = Some(ScopeParent::new(
15014                Principal::Reserved {
15015                    module_id: OWNER.to_string(),
15016                },
15017                "s".to_string(),
15018                1,
15019            ));
15020            rig.sync(vec![session(1), child.clone()]).await;
15021            let mut child_route = rig
15022                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
15023                .await;
15024            assert_eq!(
15025                child_route.stamp().unwrap().parent_state,
15026                Some(ParentState::Linked)
15027            );
15028            rig.sync(vec![child]).await;
15029            assert!(!rig.live(&child_route));
15030            assert_eq!(
15031                child_route.closed_reason(),
15032                RouteCloseReason::ScopeParentEnded
15033            );
15034        }
15035
15036        #[tokio::test]
15037        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
15038        ) {
15039            let mut rig = rig().await;
15040            rig.sync(vec![session(1)]).await;
15041            let mut route = rig
15042                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15043                .await;
15044            let before = rig.forwarding.published_scope_tag(OWNER, "s");
15045            rig.sync(vec![session(1)]).await;
15046            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15047            assert!(rig.live(&route) && route.untouched());
15048
15049            // A call in flight on the route when another carrier is added. A
15050            // forwarded REQUEST holds one credit on the route's flow until the
15051            // module answers; the router takes it exactly like this.
15052            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
15053                .forwarding
15054                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15055                .unwrap()
15056            else {
15057                panic!("the route is bound");
15058            };
15059            binding.flow.acquire_tagged(9, false).await.unwrap();
15060            let mut widened = session(1);
15061            widened.carriers.push(carrier(MAGIC, None));
15062            rig.sync(vec![widened]).await;
15063            assert!(rig.live(&route) && route.untouched());
15064            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15065            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
15066            // The call's credit is still held on an open flow, so its answer
15067            // will be delivered: closing the route would have closed the flow.
15068            assert_eq!(binding.flow.in_flight(), 1);
15069            binding
15070                .flow
15071                .acquire_tagged(10, false)
15072                .await
15073                .expect("the flow is still open");
15074        }
15075
15076        /// A swap's superseded endpoint keeps its routes until drained; ending
15077        /// the scope closes them there too.
15078        #[tokio::test]
15079        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
15080            let mut rig = rig().await;
15081            rig.sync(vec![session(1)]).await;
15082            let mut on_incumbent = rig
15083                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15084                .await;
15085
15086            // Swap plexus: register a candidate and cut over, leaving the
15087            // incumbent superseded with the route still on it.
15088            let (candidate, _candidate_rx) = wide_ctx(9);
15089            let registration = rig
15090                .handler
15091                .registry
15092                .register_candidate_with_control_ops(
15093                    manifest(PLEXUS, PROTOCOL_VERSION),
15094                    PROTOCOL_VERSION,
15095                    candidate.connection_id,
15096                    module_baseline_control_ops(),
15097                )
15098                .unwrap();
15099            rig.forwarding
15100                .register_candidate_module_connection(
15101                    candidate.connection_id,
15102                    PLEXUS.to_string(),
15103                    PROTOCOL_VERSION,
15104                    manifest_concurrency(&registration.manifest),
15105                    candidate.egress.clone(),
15106                )
15107                .unwrap();
15108            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
15109            rig.handler
15110                .registry
15111                .promote_candidate(PLEXUS)
15112                .unwrap()
15113                .unwrap();
15114            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
15115
15116            rig.sync(Vec::new()).await;
15117            assert!(!rig.live(&on_incumbent));
15118            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
15119            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15120            let goodbye = incumbent_rx
15121                .try_recv()
15122                .expect("the superseded endpoint is told")
15123                .frame;
15124            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15125        }
15126    }
15127}
15128
15129#[cfg(test)]
15130mod concurrency_default_exposure_tests {
15131    use super::*;
15132
15133    fn hello_body(role_json: &str) -> Vec<u8> {
15134        format!(
15135            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
15136        )
15137        .into_bytes()
15138    }
15139
15140    fn manifest_from(body: &[u8]) -> ModuleManifest {
15141        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
15142        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
15143            .expect("manifest parses")
15144    }
15145
15146    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
15147
15148    #[test]
15149    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
15150        let body = hello_body(&format!(
15151            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
15152        ));
15153        let manifest = manifest_from(&body);
15154        // Precondition: serde really resolved it to the default, so the typed
15155        // manifest alone cannot answer the question this probe exists for.
15156        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
15157        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
15158    }
15159
15160    #[test]
15161    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
15162        let body = hello_body(&format!(
15163            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
15164        ));
15165        let manifest = manifest_from(&body);
15166        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
15167        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
15168    }
15169
15170    #[test]
15171    fn non_management_roles_are_never_reported() {
15172        let body = hello_body(
15173            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
15174        );
15175        let manifest = manifest_from(&body);
15176        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
15177    }
15178}