Skip to main content

subc_daemon/
control.rs

1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6    fmt,
7    net::SocketAddr,
8    path::{Path, PathBuf},
9    sync::{Arc, Mutex, RwLock},
10    time::{Duration, Instant as StdInstant},
11};
12
13use crate::scopes::admission::denied_capability;
14use serde::{Deserialize, Serialize};
15use subc_control::{
16    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
17    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
18    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
19    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
20    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
21    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
22    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
23};
24use subc_protocol::{
25    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
26    error_codes,
27    manifest::{
28        validate_hello_capability_grammar, validate_hello_confirm_operations,
29        validate_hello_display_name, validate_hello_event_declarations,
30        validate_hello_flow_operations, validate_hello_operation_deferral,
31        validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
32        validate_hello_self_signal_declarations, validate_role_declarations_value,
33        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
34        ProviderRole,
35    },
36    peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
37    provider_declarations::ProviderDeclarations,
38    scope::{
39        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
40        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
41        SCOPE_SYNC_OP,
42    },
43    session::{
44        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
45        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
46        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
47        MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS, ROLE_VERSIONS_FIELD,
48    },
49    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
50    Priority, RouteTarget, PROTOCOL_VERSION,
51};
52use tokio::time::{timeout_at, Instant};
53use tracing::{debug, info, instrument::WithSubscriber, warn};
54
55use crate::{
56    caller_identity::CallerIdentity,
57    capability_requirements::{
58        log_duplicate_claim_events, log_manifest_change, log_requirement_events,
59        CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
60        RequirementStatus, RuntimeModule,
61    },
62    child_roster::DaemonShutdownFlag,
63    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
64    forwarding::{
65        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
66        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
67        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
68    },
69    observability::{
70        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
71    },
72    privileged_prompt::{prompt_text, PrivilegedOperation},
73    provenance::{
74        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
75    },
76    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
77    router::{daemon_error_flags, RouteCtx, RouterError},
78    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
79    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
80    stderr_tail::{CaptureState, TailEntry},
81    supervise::{
82        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
83        SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
84    },
85    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
86};
87
88/// Lowest envelope version this subc build will negotiate.
89///
90/// Module HELLO negotiation is exact: peers must use the daemon's locked
91/// protocol version. Older and newer peers receive `version_unsupported` and
92/// are not registered.
93pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
94
95const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
96const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
97const CAP_PING_PONG: &str = "ping_pong_v1";
98const CAP_SESSION_ATTACH: &str = "session_attach_v1";
99const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
100
101const SUBC_CONTROL_OPS: &[&str] = &[
102    "claim.mint",
103    "claim.present",
104    ops::DAEMON_SHUTDOWN_INTENT,
105    ops::ACTIVITY_RECORD,
106    ops::ACTIVITY_QUERY,
107    ops::ACTIVITY_SUBSCRIBE,
108    ops::SERVER_DESCRIBE,
109    ops::CATALOG_LIST,
110    ops::CATALOG_WATCH,
111    ops::ROUTE_OPEN,
112    ops::ROUTE_POLL,
113    ops::ROUTE_CLOSING,
114    ops::ROUTE_CLOSED,
115    ops::SUPERVISOR_LIST,
116    ops::SUPERVISOR_RESTART,
117    ops::SUPERVISOR_SWAP,
118    ops::SUPERVISOR_RELOAD,
119    ops::SUPERVISOR_RESCAN,
120    ops::SUPERVISOR_RELEASE_RESERVED,
121    ops::SUPERVISOR_SET_ENABLED,
122    ops::SUPERVISOR_HEALTH_PROBE,
123    ops::SUPERVISOR_HEALTH,
124    ops::SUPERVISOR_STDERR_TAIL,
125    ops::SUPERVISOR_TERMINALS,
126    ops::SUPERVISOR_ROUTES,
127    ops::SUPERVISOR_PROVENANCE,
128    ops::SUPERVISOR_SPAWN_SNAPSHOT,
129    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
130];
131
132/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
133/// announced by a capability in `server.describe` instead, and clients check
134/// that capability before sending it. `ck daemon --json` prints `subc_ops`
135/// verbatim and scripts read that output, so a new name there would change it.
136/// A malformed body for one of these is still refused `invalid_control_body`
137/// rather than as an unknown op.
138const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[
139    ops::SUPERVISOR_RESCAN_CHECKED,
140    "config.status",
141    "config.plan",
142    "config.apply",
143    "config.stamp",
144    "config.unstamp",
145    "config.history",
146    "config.rollback",
147];
148
149mod caller_attestation;
150mod channel_operations;
151#[cfg(all(test, unix, feature = "test-support"))]
152#[path = "control/channel_operations_tests.rs"]
153mod channel_operations_tests;
154pub(crate) mod claims;
155#[cfg(test)]
156#[path = "control/config_advertisement_tests.rs"]
157mod config_advertisement_tests;
158mod config_gate;
159use caller_attestation::{caller_for_target, scope_acts_for_another};
160#[cfg(all(test, unix, feature = "test-support"))]
161#[path = "control/caller_attestation_tests.rs"]
162mod caller_attestation_tests;
163mod scope_expiry;
164
165mod agent_commands;
166pub(crate) mod catalog;
167mod rescan_checked;
168use rescan_checked::{RescanMode, RescanPlan};
169
170const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
171    "claim.mint",
172    "claim.present",
173    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
174    MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS,
175    "supervisor.live_roots",
176    "supervisor.agent_commands",
177    SCOPE_SYNC_OP,
178    SCOPE_APPLY_OP,
179    SCOPE_DESCRIBE_OP,
180    subc_protocol::scope::ROUTE_AUTHORIZE_OP,
181    "operator.confirm",
182];
183
184/// Module-originated ops the daemon answers but does not advertise in
185/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
186/// belongs to is incomplete, so no module is told it works before it does.
187const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
188
189const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
190
191/// How long subc waits for a module to ack a relayed route.bind before returning
192/// `module_timeout`. The ack waits on the module's own configure, which for AFT
193/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
194/// and DB-open work — on a cold page cache or a large repo that legitimately
195/// exceeds a couple of seconds. The default is generous because rejecting a VALID
196/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
197/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
198pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
199
200/// How many CONSECUTIVE full-budget relay timeouts against one target module
201/// open that module's bind-relay breaker.
202///
203/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
204/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
205/// so three consecutive full-budget timeouts take ~36s to observe: every client
206/// whose open contributed to opening the breaker had already given up on its
207/// own. That is what makes opening the breaker unable to turn a call that would
208/// have succeeded into a refusal — it can only make an already-failing module
209/// fail faster.
210///
211/// Two would be reachable inside one default deadline. One would convict a
212/// module on a single cold-cache bind, which is exactly the valid-but-slow case
213/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
214pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
215
216/// How long a module's bind-relay breaker stays open before exactly one
217/// `route.open` is let through as a probe.
218///
219/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
220/// re-pays a full-budget stall almost continuously, and the breaker stops being
221/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
222/// deadline: a client that starts retrying after the module recovers has to get
223/// a probe opportunity inside its own deadline, or the breaker converts a
224/// recovered module into a failed call — the failure it exists to prevent,
225/// pointed the other way.
226///
227/// 20s sits between those with room on both sides, and it caps what a wedged
228/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
229/// rather than one per `route.open` per connection. The stall that motivated
230/// this, with its measurements, is written up in
231/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
232/// waited the whole budget out.
233pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
234
235const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
236const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
237
238fn reload_verdict(
239    configured: &Path,
240    spawned_from: Option<&Path>,
241    image: subc_control::RunningImageAgreement,
242) -> PendingReloadVerdict {
243    let path = match spawned_from {
244        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
245            ReloadPathAgreement::Match
246        }
247        Some(spawned_from) => ReloadPathAgreement::Mismatch {
248            configured: configured.to_path_buf(),
249            spawned_from: spawned_from.to_path_buf(),
250        },
251        None => ReloadPathAgreement::Unavailable {
252            reason: if matches!(
253                image,
254                subc_control::RunningImageAgreement::Unavailable {
255                    reason: subc_control::RunningImageUnavailableReason::NotRunning
256                }
257            ) {
258                ReloadPathUnavailableReason::NotRunning
259            } else {
260                ReloadPathUnavailableReason::SpawnedPathUnavailable
261            },
262        },
263    };
264    PendingReloadVerdict::new(path, image)
265}
266
267fn launch_spec_agreement(
268    configured: &crate::supervise::ModuleSpec,
269    running: bool,
270    spawned: Option<&crate::supervise::LaunchSpecDigest>,
271) -> subc_control::LaunchSpecAgreement {
272    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
273    if !running {
274        return Agreement::Unavailable {
275            reason: Reason::NotRunning,
276        };
277    }
278    match spawned {
279        None => Agreement::Unavailable {
280            reason: Reason::SpawnedSpecUnavailable,
281        },
282        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
283            Agreement::Match
284        }
285        Some(_) => Agreement::Mismatch,
286    }
287}
288
289fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
290    if configured == spawned_from {
291        return true;
292    }
293    #[cfg(windows)]
294    {
295        // Windows spawn records the resolved verbatim path (\\?\...), while
296        // configuration keeps its original spelling. Resolve both with the OS
297        // instead of stripping prefixes or folding case, which could conflate
298        // distinct names in case-sensitive directories. Different resolved
299        // paths still differ even if their files have identical content.
300        match (
301            std::fs::canonicalize(configured),
302            std::fs::canonicalize(spawned_from),
303        ) {
304            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
305            _ => false,
306        }
307    }
308    #[cfg(not(windows))]
309    {
310        false
311    }
312}
313
314#[derive(Clone)]
315struct DaemonProvenanceFacts {
316    build: DaemonBuildProvenance,
317    pid: Option<u32>,
318    started_at_ms: Option<u64>,
319    start_clock: Option<crate::clock::StartClock>,
320    executable_path: Option<PathBuf>,
321    executable_identity: Option<SpawnedFileIdentity>,
322    process_start_time: Option<u64>,
323    probe: ExecutableIdentityProbe,
324}
325
326impl Default for DaemonProvenanceFacts {
327    fn default() -> Self {
328        Self {
329            build: DaemonBuildProvenance {
330                build_git_sha: None,
331                build_lock_digest: None,
332            },
333            pid: None,
334            started_at_ms: None,
335            start_clock: None,
336            executable_path: None,
337            executable_identity: None,
338            process_start_time: None,
339            probe: ExecutableIdentityProbe::default(),
340        }
341    }
342}
343
344#[derive(Debug, Clone)]
345struct SupervisorRescanContext {
346    supervisor: Supervisor,
347    config_path: PathBuf,
348    configured_port: Option<u16>,
349    storage_config: Option<crate::daemon_config::StorageConfig>,
350    admission_facts_carrier_module_id: Option<String>,
351    admission_facts_targets: Option<Vec<String>>,
352    scope_authority_owners: Vec<String>,
353    federated_scope_owner: String,
354    privileged_operator_prompt: PrivilegedOperatorPrompt,
355    /// Counts executed rescans that set out to change the supervised modules'
356    /// staged or applied configuration, counted before the first change so a
357    /// rescan that fails partway still counts. With the daemon incarnation it
358    /// names the state a `supervisor.rescan_checked` preview was computed
359    /// against.
360    generation: Arc<std::sync::atomic::AtomicU64>,
361}
362
363/// Refusal labels passed to `observe_route_open_refusal` that mean the target
364/// module is not serving right now, and so open or extend an outage in the
365/// route outage tracker. Every one of them is only reachable after the target
366/// was found in the registry, which is what keeps an arbitrary client-chosen
367/// id from ever creating tracker state.
368///
369/// Deliberately absent: `not_registered` and `removed` (the id may be
370/// anything a client sent, and a removed module is gone on purpose),
371/// `protocol_none` (such a module never serves routes, so nothing is out),
372/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
373/// capability and admission-facts refusals (they refuse the caller, not a
374/// module outage), and `relay_reservation_failed` (its code ranges over
375/// capacity limits as well as a vanished connection). Capacity, breaker,
376/// relay-timeout and module-rejection refusals do not pass through that
377/// function at all; the breaker logs its own transitions.
378///
379/// The two not-serving refusals that bypass that function record themselves
380/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
381/// `required_capability_unprovided` is not tracked: the module itself is up,
382/// and the outage belongs to the missing provider.
383const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
384    "reloading",
385    "supervisor_not_live",
386    "registration_not_active",
387    "no_forwarding_connection",
388    "relay_send_failed",
389];
390
391/// Observe test request cancellation relative to the connection's cleanup.
392#[cfg(test)]
393struct ActivityRequestPause {
394    connection: ConnectionId,
395    started: tokio::sync::Notify,
396    cancelled: tokio::sync::Notify,
397    cleanup_started: std::sync::atomic::AtomicBool,
398    cancelled_after_cleanup: std::sync::atomic::AtomicBool,
399}
400
401#[cfg(test)]
402struct ActivityRequestCancellation(Arc<ActivityRequestPause>);
403
404#[cfg(test)]
405impl Drop for ActivityRequestCancellation {
406    fn drop(&mut self) {
407        self.0.cancelled_after_cleanup.store(
408            self.0
409                .cleanup_started
410                .load(std::sync::atomic::Ordering::Acquire),
411            std::sync::atomic::Ordering::Release,
412        );
413        self.0.cancelled.notify_one();
414    }
415}
416
417/// A test hold: signal arrival, then wait for release on the holding thread.
418#[cfg(test)]
419type TestPause = (
420    tokio::sync::oneshot::Sender<()>,
421    std::sync::mpsc::Receiver<()>,
422);
423
424/// A test hook run once at a fixed point in a handler.
425#[cfg(test)]
426type TestHook = Box<dyn FnOnce() + Send>;
427
428/// A test hold for async code: signal arrival, then await release.
429#[cfg(test)]
430type AsyncTestPause = (
431    tokio::sync::oneshot::Sender<()>,
432    tokio::sync::oneshot::Receiver<()>,
433);
434
435/// Real channel-0 control handler for subc itself.
436#[derive(Clone)]
437pub struct ControlHandler {
438    #[cfg(test)]
439    activity_request_pause: Arc<Mutex<Option<Arc<ActivityRequestPause>>>>,
440    /// Holds connection cleanup at a numbered point: 0 is just after the agent
441    /// declaration queue closes, 1 just after the registration channel is
442    /// forgotten.
443    #[cfg(test)]
444    cleanup_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
445    /// Runs once in `route.open`'s launch-nonce attestation, right after the
446    /// nonce is resolved, to interleave a nonce withdrawal there.
447    #[cfg(test)]
448    after_nonce_attestation: Arc<Mutex<Option<TestHook>>>,
449    /// Holds a registration channel's HELLO after its reader resolved the
450    /// channel and before the blocking HELLO handler starts.
451    #[cfg(test)]
452    hello_ingress_pause: Arc<Mutex<Option<AsyncTestPause>>>,
453    /// Holds an agent command declaration at a numbered point in its handler.
454    #[cfg(test)]
455    agent_commands_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
456    claims: Arc<claims::Claims>,
457    registry: Arc<Registry>,
458    config_runtime: Option<Arc<crate::config_gate::runtime::RuntimeOwner>>,
459    config_review_admission: Arc<dyn crate::config_gate::ReviewAdmission>,
460    config_review_admission_is_default: bool,
461    activity: Option<crate::activity::ActivityLedger>,
462    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
463    catalog_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
464    forwarding: Arc<ForwardingTable>,
465    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
466    supervisor: SupervisorHandle,
467    pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
468    agent_declarations: agent_commands::Queues,
469    list_admission: Arc<tokio::sync::Semaphore>,
470    subc_capabilities: Arc<[String]>,
471    /// Daemon-wide route.bind relay budget. Used as the fallback when the
472    /// target module has no per-module override in
473    /// `route_bind_relay_timeouts`.
474    route_bind_relay_timeout: Duration,
475    /// Per-module route.bind relay budget overrides, keyed by module id. When
476    /// `handle_route_open` resolves the deadline for a target module, a
477    /// per-module entry wins over the daemon-wide value above.
478    route_bind_relay_timeouts: BTreeMap<String, Duration>,
479    /// Per-target-module bind-relay breaker state. Shared with the forwarding
480    /// table, which is where a new module connection resets it.
481    route_bind_breakers: RouteBindBreakers,
482    /// Live relay admissions keyed by target module. Shared through the
483    /// forwarding table so cloned or separately built handlers enforce one cap.
484    route_bind_concurrency: RouteBindConcurrency,
485    /// Start and end of each module's not-serving period as seen by
486    /// `route.open`, so an outage gets one line at each edge instead of only
487    /// the per-refusal INFO lines. Taken from the forwarding table, so every
488    /// handler built over one table shares it.
489    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
490    /// Consecutive relay timeouts that open a module's breaker.
491    route_bind_breaker_threshold: u32,
492    /// How long a breaker stays open before one probe is admitted.
493    route_bind_breaker_cooldown: Duration,
494    health_probe_timeout: Duration,
495    /// Central storage policy. When set, each registering module receives its
496    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
497    storage_config: Option<crate::daemon_config::StorageConfig>,
498    /// The machine id established at boot, served on every HELLO_ACK and on
499    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
500    /// changes the file, never this value. `None` serves no id.
501    machine_id: Option<crate::machine_id::MachineId>,
502    admission_facts_carrier_module_id: Option<String>,
503    admission_facts_targets: Option<Vec<String>>,
504    /// Scope records with their sync authorities and tombstones; see
505    /// `crate::scopes`. Shared by clones of this handler, so every connection
506    /// reads and writes one table.
507    scopes: Arc<RwLock<ScopeTable>>,
508    /// Wakes the expiry loop when an accepted scope change may have added a
509    /// deadline, so the loop can wait without ticking while no live scope has
510    /// one. A permit is stored when nothing waits, so no wake is lost.
511    scope_deadline_added: Arc<tokio::sync::Notify>,
512    /// The configured `scope_authority_owners`, kept so a rescan can report a
513    /// changed value as needing a daemon restart; rescan never applies it.
514    scope_authority_owners: Vec<String>,
515    federated_scope_owner: String,
516    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
517    privileged_operator_prompt: PrivilegedOperatorPrompt,
518    /// The local and peer address of each accepted TCP client connection, read
519    /// from the daemon's own socket, so a privileged request can name its caller.
520    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
521    build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
522    observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
523    captured_peers: Arc<Mutex<HashMap<ConnectionId, subc_os::local_peer::CapturedPeer>>>,
524    config_desktop_stamps:
525        Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
526    local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
527    pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
528    pub(crate) operator_apps:
529        Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
530    /// One launch entry per app, behind a leaf lock: no other lock is taken
531    /// while it is held. Entries are memory-only, so a daemon or
532    /// installed-service restart drops them and a relaunched daemon never
533    /// mistakes an older process for its own launch.
534    pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
535    /// Serializes `app.open` launches: the liveness check in the decision and
536    /// the later publish must not interleave across concurrent calls, or two
537    /// calls could both launch and the second publish would orphan the first
538    /// process. Refusals and `already_running` answers share the same lock;
539    /// `app.open` is rare, so the cost is negligible.
540    pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
541    // Portable wire samples exercise dispatch without starting an OS process.
542    #[cfg(test)]
543    pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
544    #[cfg(test)]
545    pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
546    #[cfg(test)]
547    recovery_exit_test_after_step: Option<fn(usize) -> Result<(), crate::config_gate::GateError>>,
548    #[cfg(test)]
549    recovery_exit_test_completed: Option<fn(u64)>,
550    #[cfg(test)]
551    apply_reconcile_pause: Arc<Mutex<Option<AsyncTestPause>>>,
552    #[cfg(test)]
553    recovery_exit_test_quiet_retry: bool,
554    #[cfg(test)]
555    recovery_exit_test_retry_pause: Option<fn(u64) -> Option<PathBuf>>,
556    #[cfg(test)]
557    config_delivery_test_wakes:
558        Option<tokio::sync::mpsc::UnboundedSender<(&'static str, tokio::time::Instant)>>,
559    #[cfg(all(test, unix))]
560    config_delivery_test_idle:
561        Option<tokio::sync::mpsc::UnboundedSender<tokio::sync::oneshot::Sender<()>>>,
562    /// Test-only Windows signature and flash hooks. Production builds carry
563    /// neither: with the publisher unset, the production table never reaches
564    /// verification, and the default attention hook only logs.
565    #[allow(dead_code)]
566    #[cfg(windows)]
567    pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
568    #[allow(dead_code)]
569    #[cfg(windows)]
570    pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
571    /// Test-only tracking fault hook: makes opening the tracking handle on
572    /// the launched pid fail, so a test can prove an untrackable launch is
573    /// stopped rather than left running.
574    #[allow(dead_code)]
575    #[cfg(windows)]
576    pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
577    /// Test-only count of Windows spawn attempts, so a test can prove a
578    /// refused launch started no process. Unit tests only.
579    #[cfg(all(test, windows))]
580    pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
581    #[cfg(test)]
582    caller_lookup: Option<TestCallerLookup>,
583    /// The launch nonce each module connection presented at HELLO, which is how
584    /// a `scope.sync` is matched to the owner's current launch.
585    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
586    rescan: Option<SupervisorRescanContext>,
587    connected_clients: ConnectedClients,
588    counters: DaemonCounters,
589    /// Why this daemon serves no local endpoint, when binding one failed. `None`
590    /// when the endpoint is served, and also on a handler that was never told.
591    local_endpoint_unavailable: Option<String>,
592    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
593    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
594    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
595    /// their usual severity (see [`log_requirement_events`]). A handler built
596    /// without one has a flag that is never set.
597    daemon_shutdown: DaemonShutdownFlag,
598    daemon_provenance: DaemonProvenanceFacts,
599    #[cfg(test)]
600    control_dispatch_delay: Option<Duration>,
601    #[cfg(test)]
602    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
603    /// Holds `route.open` after the route.bind is queued to the module and
604    /// before it is marked enqueued, so a test can make the module's reply (or
605    /// a client teardown) land exactly inside that window.
606    #[cfg(test)]
607    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
608    /// Holds a HELLO after its registry entry is reserved and before its
609    /// forwarding endpoint is installed, so a test can read the catalog and
610    /// open routes while the module is registered but not yet routable. A
611    /// blocking barrier because the HELLO handler is synchronous; the test
612    /// runs the HELLO on its own thread.
613    #[cfg(test)]
614    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
615    #[cfg(test)]
616    hello_ack_to_publication_hold: Option<Arc<std::sync::Barrier>>,
617    #[cfg(test)]
618    hello_before_publication_hold: Option<Arc<std::sync::Barrier>>,
619    #[cfg(test)]
620    hello_after_launch_check_hold: Option<Arc<std::sync::Barrier>>,
621    #[cfg(test)]
622    hello_inside_publication_hold: Option<Arc<std::sync::Barrier>>,
623}
624
625impl fmt::Debug for ControlHandler {
626    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
627        f.debug_struct("ControlHandler")
628            .field("registry", &self.registry)
629            .field("forwarding", &self.forwarding)
630            .field("process_liveness", &self.process_liveness.is_some())
631            .field("supervisor", &self.supervisor)
632            .field("subc_capabilities", &self.subc_capabilities)
633            .finish()
634    }
635}
636
637struct RouteOpenRequest {
638    target: RouteTarget,
639    identity: BindIdentity,
640    consumer_identity: Option<ConsumerIdentity>,
641    consumer_capabilities: Option<Vec<String>>,
642    role_versions: Option<BTreeMap<String, String>>,
643    busy_refusals: Option<bool>,
644    admission_facts: Option<serde_json::Value>,
645    scope: Option<ScopeSelector>,
646    accepts_close_reasons: Option<Vec<String>>,
647}
648
649#[derive(Clone, Copy)]
650enum RouteOpenPhase {
651    Admission,
652    ProjectRoot,
653    EgressReservation,
654    BindSend,
655    BindAckWait,
656}
657
658/// Store elapsed time for each route-open phase, including attempts that end
659/// in an early refusal or backend error. Admission also includes rechecks and
660/// commit bookkeeping performed between the other timed phases.
661struct RouteOpenTimings {
662    module_id: String,
663    principal: String,
664    admission: Duration,
665    project_root: Duration,
666    egress_reservation: Duration,
667    bind_send: Duration,
668    bind_ack_wait: Duration,
669    phase: Option<RouteOpenPhase>,
670    phase_started_at: Instant,
671}
672
673impl RouteOpenTimings {
674    fn new() -> Self {
675        Self {
676            module_id: String::new(),
677            // Refusals before attestation must not claim a verified principal.
678            principal: "unresolved".to_string(),
679            admission: Duration::ZERO,
680            project_root: Duration::ZERO,
681            egress_reservation: Duration::ZERO,
682            bind_send: Duration::ZERO,
683            bind_ack_wait: Duration::ZERO,
684            phase: Some(RouteOpenPhase::Admission),
685            phase_started_at: Instant::now(),
686        }
687    }
688
689    fn finish(&mut self) {
690        let elapsed = self.phase_started_at.elapsed();
691        match self.phase.take() {
692            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
693            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
694            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
695            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
696            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
697            None => {}
698        }
699    }
700
701    fn enter(&mut self, phase: RouteOpenPhase) {
702        self.finish();
703        self.phase = Some(phase);
704        self.phase_started_at = Instant::now();
705    }
706}
707
708struct RouteBindReservationGuard {
709    forwarding: Arc<ForwardingTable>,
710    endpoint: ModuleEndpointId,
711    relay_corr: u64,
712    armed: bool,
713}
714
715struct ModuleControlRpcGuard {
716    forwarding: Arc<ForwardingTable>,
717    endpoint: ModuleEndpointId,
718    corr: u64,
719    armed: bool,
720}
721
722impl ModuleControlRpcGuard {
723    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
724        Self {
725            forwarding,
726            endpoint,
727            corr,
728            armed: true,
729        }
730    }
731
732    fn disarm(&mut self) {
733        self.armed = false;
734    }
735}
736
737impl Drop for ModuleControlRpcGuard {
738    fn drop(&mut self) {
739        if self.armed {
740            let _ = self
741                .forwarding
742                .cancel_module_control_rpc(self.endpoint, self.corr);
743        }
744    }
745}
746
747impl RouteBindReservationGuard {
748    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
749        Self {
750            forwarding,
751            endpoint,
752            relay_corr,
753            armed: true,
754        }
755    }
756
757    fn release_and_disarm(&mut self) {
758        if !self.armed {
759            return;
760        }
761        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
762            self.endpoint,
763            self.relay_corr,
764            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
765        ) {
766            send_goodbye_target_best_effort(
767                &self.forwarding.counters(),
768                &target,
769                "canceled route.bind",
770            );
771        }
772        self.armed = false;
773    }
774
775    fn disarm(&mut self) {
776        self.armed = false;
777    }
778}
779
780impl Drop for RouteBindReservationGuard {
781    fn drop(&mut self) {
782        self.release_and_disarm();
783    }
784}
785
786/// Per-target-module circuit breaker around the `route.bind` relay.
787///
788/// The connection reader is serial per connection, so a module whose `on_bind`
789/// sits on the ack blocks every LATER frame on the connections that call it,
790/// including calls to unrelated modules. This does not make any module's bind
791/// fast; it stops the daemon paying the full budget again and again for a
792/// condition it has already observed.
793///
794/// State is keyed by TARGET MODULE and shared by every connection: a wedged
795/// module wedges everyone, so what one connection learned should protect the
796/// rest.
797///
798/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
799/// relay to that module has actually timed out, and is removed again when a
800/// relay is accepted or the module reconnects, so it cannot grow with traffic
801/// or with modules that behave.
802///
803/// # Why a `std` mutex here is not the head-of-line defect again
804///
805/// Acquisition never awaits. The critical section is a hash lookup plus a few
806/// integer updates, with no I/O and no `.await` inside it, so a reader task
807/// cannot be descheduled behind it the way it can behind
808/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
809/// primitive, held for the same kind of work, as the refusal counter this very
810/// path already increments.
811///
812/// It is also NOT on the data-plane splice path: only `route.open` and module
813/// registration touch it, so bound-route frames gain no state check and no
814/// contention.
815#[derive(Debug, Clone, Default)]
816pub(crate) struct RouteBindBreakers {
817    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
818}
819
820#[derive(Debug, Clone, Default)]
821pub(crate) struct RouteBindConcurrency {
822    modules: Arc<Mutex<HashMap<String, usize>>>,
823}
824
825struct RouteBindConcurrencyGuard {
826    concurrency: RouteBindConcurrency,
827    module_id: String,
828}
829
830impl RouteBindConcurrency {
831    /// Admit without waiting. Waiting here would move the bind stall from the
832    /// module reply to a semaphore and restore reader head-of-line blocking.
833    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
834        let mut modules = self
835            .modules
836            .lock()
837            .expect("route.bind concurrency mutex poisoned");
838        let in_flight = modules.entry(module_id.to_string()).or_default();
839        if *in_flight >= limit {
840            return Err(*in_flight);
841        }
842        *in_flight += 1;
843        Ok(RouteBindConcurrencyGuard {
844            concurrency: self.clone(),
845            module_id: module_id.to_string(),
846        })
847    }
848}
849
850impl Drop for RouteBindConcurrencyGuard {
851    fn drop(&mut self) {
852        let mut modules = self
853            .concurrency
854            .modules
855            .lock()
856            .expect("route.bind concurrency mutex poisoned");
857        let remove = {
858            let in_flight = modules
859                .get_mut(&self.module_id)
860                .expect("admitted route.bind has a concurrency entry");
861            *in_flight -= 1;
862            *in_flight == 0
863        };
864        if remove {
865            modules.remove(&self.module_id);
866        }
867    }
868}
869
870#[derive(Debug, Default)]
871struct ModuleBreakerState {
872    /// Relay timeouts observed with no accepted relay in between.
873    consecutive_timeouts: u32,
874    /// `Some` while the breaker is open: the instant the cooldown expires and
875    /// the next arrival may probe. `None` means closed.
876    cooldown_until: Option<Instant>,
877    /// A half-open probe has been admitted and has not settled yet. This is
878    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
879    /// that read the cooldown, so concurrent opens arriving at the moment the
880    /// cooldown expires cannot all decide that they are the probe.
881    probe_in_flight: Option<Arc<()>>,
882}
883
884/// What the breaker decided for one `route.open`, before any relay work.
885enum RouteBindAdmission<'a> {
886    Admitted {
887        guard: RouteBindBreakerGuard<'a>,
888        /// This open is the single half-open probe, so the transition is worth
889        /// one log line.
890        probe: bool,
891    },
892    Refused {
893        consecutive_timeouts: u32,
894        /// What is left of the cooldown. Zero when the refusal is because the
895        /// one probe is already in flight rather than because the cooldown has
896        /// not elapsed.
897        retry_in: Duration,
898        probe_in_flight: bool,
899    },
900}
901
902/// An outstanding admission, which must be told how its relay settled.
903///
904/// `Drop` settles it as inconclusive, so an early return between admission and
905/// the relay -- or the whole handler being cancelled when the client
906/// disconnects -- releases a half-open probe slot instead of leaving the
907/// breaker wedged half-open with no further probes.
908struct RouteBindBreakerGuard<'a> {
909    breakers: RouteBindBreakers,
910    module_id: &'a str,
911    probe_token: Option<Arc<()>>,
912    settled: bool,
913}
914
915impl RouteBindBreakerGuard<'_> {
916    /// The module answered within the budget and took the bind. THE ONLY
917    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
918    /// breaker, which is a transition worth logging.
919    fn record_accepted(&mut self) -> bool {
920        self.settled = true;
921        self.breakers.record_accepted(self.module_id)
922    }
923
924    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
925    /// COUNTS TOWARD OPENING.
926    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
927        self.settled = true;
928        self.breakers.record_timeout(
929            self.module_id,
930            self.probe_token.as_ref(),
931            threshold,
932            cooldown,
933        )
934    }
935
936    /// Everything else: the module REJECTED the bind, its connection went away
937    /// mid-relay, or the waiter was cancelled.
938    ///
939    /// None of these is evidence that a module is slow, and each already has
940    /// its own refusal with its own code. A module that rejects a bind in
941    /// microseconds is healthy and must never be convicted for it; a module
942    /// that died has said nothing about the module that replaces it. So these
943    /// neither increment nor reset the count -- they only release a probe slot.
944    fn record_inconclusive(&mut self) {
945        self.settled = true;
946        self.breakers
947            .record_inconclusive(self.module_id, self.probe_token.as_ref());
948    }
949}
950
951impl Drop for RouteBindBreakerGuard<'_> {
952    fn drop(&mut self) {
953        if !self.settled {
954            self.breakers
955                .record_inconclusive(self.module_id, self.probe_token.as_ref());
956        }
957    }
958}
959
960/// The breaker moved to open, reported so the caller can log it outside the
961/// lock. Opening is rare and load-bearing; the refusals that follow are
962/// frequent and are counted rather than logged.
963struct BreakerOpened {
964    consecutive_timeouts: u32,
965    /// True when a failed probe re-opened an already-open breaker, which reads
966    /// very differently in a log from a first opening.
967    reopened_after_probe: bool,
968}
969
970impl RouteBindBreakers {
971    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
972        self.modules
973            .lock()
974            .expect("route.bind breaker mutex poisoned")
975    }
976
977    /// Decide whether this `route.open` may attempt its relay. Takes the map
978    /// lock and nothing else, and never awaits.
979    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
980        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
981            probe: probe_token.is_some(),
982            guard: RouteBindBreakerGuard {
983                breakers: self.clone(),
984                module_id,
985                probe_token,
986                settled: false,
987            },
988        };
989
990        let mut modules = self.lock();
991        let Some(state) = modules.get_mut(module_id) else {
992            return admitted(None);
993        };
994        let Some(cooldown_until) = state.cooldown_until else {
995            return admitted(None);
996        };
997        if state.probe_in_flight.is_some() {
998            return RouteBindAdmission::Refused {
999                consecutive_timeouts: state.consecutive_timeouts,
1000                retry_in: Duration::ZERO,
1001                probe_in_flight: true,
1002            };
1003        }
1004        let now = Instant::now();
1005        if now < cooldown_until {
1006            return RouteBindAdmission::Refused {
1007                consecutive_timeouts: state.consecutive_timeouts,
1008                retry_in: cooldown_until - now,
1009                probe_in_flight: false,
1010            };
1011        }
1012        let token = Arc::new(());
1013        state.probe_in_flight = Some(Arc::clone(&token));
1014        admitted(Some(token))
1015    }
1016
1017    fn record_accepted(&self, module_id: &str) -> bool {
1018        self.lock()
1019            .remove(module_id)
1020            .is_some_and(|state| state.cooldown_until.is_some())
1021    }
1022
1023    fn record_timeout(
1024        &self,
1025        module_id: &str,
1026        probe_token: Option<&Arc<()>>,
1027        threshold: u32,
1028        cooldown: Duration,
1029    ) -> Option<BreakerOpened> {
1030        let mut modules = self.lock();
1031        let state = modules.entry(module_id.to_string()).or_default();
1032        let was_open = state.cooldown_until.is_some();
1033        let was_probe = Self::owns_probe(state, probe_token);
1034        if was_probe {
1035            state.probe_in_flight = None;
1036        }
1037        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
1038        if state.consecutive_timeouts < threshold {
1039            return None;
1040        }
1041        state.cooldown_until = Some(Instant::now() + cooldown);
1042        Some(BreakerOpened {
1043            consecutive_timeouts: state.consecutive_timeouts,
1044            reopened_after_probe: was_open && was_probe,
1045        })
1046    }
1047
1048    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
1049        // A relay can finish after the breaker was reset or after it opened
1050        // again and started a new probe. Only the guard whose token matches the
1051        // active probe may release it, so a late relay never frees a newer probe.
1052        state
1053            .probe_in_flight
1054            .as_ref()
1055            .zip(token)
1056            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
1057    }
1058
1059    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
1060        if let Some(state) = self.lock().get_mut(module_id) {
1061            if Self::owns_probe(state, probe_token) {
1062                state.probe_in_flight = None;
1063            }
1064        }
1065    }
1066
1067    /// Discard what was learned about a module, because the process it was
1068    /// learned about is gone. Returns the discarded count when it was non-zero.
1069    ///
1070    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
1071    /// `module_id` is a configuration identity that outlives any particular
1072    /// child; what the breaker observed was the process behind the module
1073    /// connection of the moment. When a new connection registers under that id
1074    /// the verdict's subject no longer exists, so the verdict is stale by
1075    /// construction rather than merely likely to be wrong. Keeping it would
1076    /// apply a dead process's record to a live one, which is the same defect
1077    /// class this breaker exists to stop the daemon committing.
1078    ///
1079    /// A half-open probe in flight is discarded with the rest: it was a
1080    /// question about the old process.
1081    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
1082        self.lock()
1083            .remove(module_id)
1084            .map(|state| state.consecutive_timeouts)
1085            .filter(|discarded| *discarded > 0)
1086    }
1087
1088    /// Open breakers, for the `server.describe` counters object. `None` when
1089    /// none is open, so the key stays absent rather than present-and-empty.
1090    ///
1091    /// This is the operator's answer to "is this module refusing instantly or
1092    /// is it fine?", which look identical from a client that retries and then
1093    /// succeeds.
1094    fn open_snapshot(&self) -> Option<serde_json::Value> {
1095        let now = Instant::now();
1096        let modules = self.lock();
1097        let open = modules
1098            .iter()
1099            .filter_map(|(module_id, state)| {
1100                let cooldown_until = state.cooldown_until?;
1101                Some((
1102                    module_id.clone(),
1103                    serde_json::json!({
1104                        "consecutive_timeouts": state.consecutive_timeouts,
1105                        "cooldown_remaining_ms":
1106                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
1107                        "probe_in_flight": state.probe_in_flight.is_some(),
1108                    }),
1109                ))
1110            })
1111            .collect::<serde_json::Map<String, serde_json::Value>>();
1112        (!open.is_empty()).then_some(serde_json::Value::Object(open))
1113    }
1114}
1115
1116impl ControlHandler {
1117    pub(crate) fn install_registration_service(
1118        &self,
1119        router: std::sync::Weak<crate::Router>,
1120        auth: crate::server::ServerAuth,
1121    ) {
1122        self.supervisor.install_registration_service(router, auth);
1123    }
1124    pub fn new(registry: Arc<Registry>) -> Self {
1125        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
1126    }
1127
1128    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
1129        let counters = forwarding.counters();
1130        // Taken from the forwarding table rather than created here, so that the
1131        // breaker a `route.open` consults is the same one a module's
1132        // registration resets, however many handlers are built over one table.
1133        let route_bind_breakers = forwarding.route_bind_breakers();
1134        let route_bind_concurrency = forwarding.route_bind_concurrency();
1135        let route_outages = forwarding.route_outages();
1136        Self {
1137            registry,
1138            #[cfg(test)]
1139            activity_request_pause: Arc::new(Mutex::new(None)),
1140            #[cfg(test)]
1141            cleanup_pause: Arc::new(Mutex::new(None)),
1142            #[cfg(test)]
1143            after_nonce_attestation: Arc::new(Mutex::new(None)),
1144            #[cfg(test)]
1145            hello_ingress_pause: Arc::new(Mutex::new(None)),
1146            #[cfg(test)]
1147            agent_commands_pause: Arc::new(Mutex::new(None)),
1148            claims: Arc::new(claims::Claims::default()),
1149            config_runtime: None,
1150            config_review_admission: Arc::new(crate::config_gate::MissingReviewAdmission),
1151            config_review_admission_is_default: true,
1152            activity: None,
1153            activity_streams: Arc::new(Mutex::new(HashMap::new())),
1154            catalog_streams: Arc::new(Mutex::new(HashMap::new())),
1155            forwarding,
1156            process_liveness: None,
1157            supervisor: SupervisorHandle::new(),
1158            resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1159                env!("CARGO_PKG_VERSION").to_owned(),
1160            )),
1161            agent_declarations: agent_commands::Queues::default(),
1162            list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1163            subc_capabilities: Arc::from([
1164                CAP_MANIFEST_REGISTRATION.to_string(),
1165                CAP_CHANNEL_LIFECYCLE.to_string(),
1166                CAP_PING_PONG.to_string(),
1167                CAP_SESSION_ATTACH.to_string(),
1168                CAP_ADMISSION_FACTS_RELAY.to_string(),
1169                CAP_SCOPES_V1.to_string(),
1170                subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1171                subc_protocol::peer::CALLER_ATTESTATION_CAPABILITY.to_string(),
1172                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1173                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1174                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1175                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1176                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1177                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1178                subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1179                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1180                subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1181                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1182                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1183                subc_protocol::ROUTE_CLOSE_REASONS_CAPABILITY.to_string(),
1184            ]),
1185            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1186            route_bind_relay_timeouts: BTreeMap::new(),
1187            route_bind_breakers,
1188            route_bind_concurrency,
1189            route_outages,
1190            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1191            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1192            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1193            storage_config: None,
1194            machine_id: None,
1195            admission_facts_carrier_module_id: None,
1196            admission_facts_targets: None,
1197            scopes: Arc::new(RwLock::new(ScopeTable::new(
1198                crate::daemon_config::default_scope_authority_owners(),
1199            ))),
1200            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1201            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1202            federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1203            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1204            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1205            build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1206            observed_peers: Arc::new(Mutex::new(HashMap::new())),
1207            captured_peers: Arc::new(Mutex::new(HashMap::new())),
1208            config_desktop_stamps: Arc::new(Mutex::new(HashMap::new())),
1209            local_connections: Arc::new(Mutex::new(HashSet::new())),
1210            operator_app_policy: Default::default(),
1211            operator_apps: Arc::new(Mutex::new(HashMap::new())),
1212            launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1213            app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1214            #[cfg(test)]
1215            app_open_test_platform: None,
1216            #[cfg(test)]
1217            app_open_test_launch: None,
1218            #[cfg(test)]
1219            recovery_exit_test_after_step: None,
1220            #[cfg(test)]
1221            recovery_exit_test_completed: None,
1222            #[cfg(test)]
1223            apply_reconcile_pause: Arc::new(Mutex::new(None)),
1224            #[cfg(test)]
1225            recovery_exit_test_quiet_retry: false,
1226            #[cfg(test)]
1227            recovery_exit_test_retry_pause: None,
1228            #[cfg(test)]
1229            config_delivery_test_wakes: None,
1230            #[cfg(all(test, unix))]
1231            config_delivery_test_idle: None,
1232            #[cfg(windows)]
1233            windows_verify: None,
1234            #[cfg(windows)]
1235            windows_flash: None,
1236            #[cfg(windows)]
1237            windows_track: None,
1238            #[cfg(all(test, windows))]
1239            windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1240            #[cfg(test)]
1241            caller_lookup: None,
1242            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1243            rescan: None,
1244            connected_clients: ConnectedClients::new(),
1245            local_endpoint_unavailable: None,
1246            counters,
1247            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1248            daemon_shutdown: DaemonShutdownFlag::default(),
1249            daemon_provenance: DaemonProvenanceFacts::default(),
1250            #[cfg(test)]
1251            control_dispatch_delay: None,
1252            #[cfg(test)]
1253            provenance_probe_override: None,
1254            #[cfg(test)]
1255            route_bind_relay_sent_hold: None,
1256            #[cfg(test)]
1257            hello_registry_to_forwarding_hold: None,
1258            #[cfg(test)]
1259            hello_ack_to_publication_hold: None,
1260            #[cfg(test)]
1261            hello_before_publication_hold: None,
1262            #[cfg(test)]
1263            hello_after_launch_check_hold: None,
1264            #[cfg(test)]
1265            hello_inside_publication_hold: None,
1266        }
1267    }
1268
1269    /// Set the central storage policy: registering modules then receive their
1270    /// resolved storage descriptor in HELLO_ACK.
1271    pub fn with_storage_config(
1272        mut self,
1273        storage_config: Option<crate::daemon_config::StorageConfig>,
1274    ) -> Self {
1275        self.storage_config = storage_config;
1276        self
1277    }
1278
1279    /// Open the ledger on its own writer thread; an unavailable sink does not
1280    /// prevent the daemon from serving routing or supervisor requests.
1281    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1282        self.activity = Some(crate::activity::ActivityLedger::open(
1283            run_dir.join("activity.jsonl"),
1284            incarnation,
1285        ));
1286        self.supervisor.set_activity(self.activity.clone());
1287        self
1288    }
1289
1290    /// Set the machine id served to every registering module (HELLO_ACK) and on
1291    /// `server.describe`.
1292    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1293        self.machine_id = machine_id;
1294        self
1295    }
1296
1297    /// Configure the exact reserved module and target ids permitted to relay
1298    /// opaque admission facts. Config-file loading validates this authority;
1299    /// this builder keeps the same policy available to embedded test daemons.
1300    pub fn with_admission_facts_config(
1301        mut self,
1302        carrier_module_id: Option<String>,
1303        targets: Option<Vec<String>>,
1304    ) -> Self {
1305        self.admission_facts_carrier_module_id = carrier_module_id;
1306        self.admission_facts_targets = targets;
1307        self
1308    }
1309
1310    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1311    /// Replaces the scope table with an empty one under the new list, so call it
1312    /// while building the handler, before any module can sync.
1313    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1314        self.scopes = Arc::new(RwLock::new(
1315            ScopeTable::new(owners.iter().cloned())
1316                .with_federated_owner(self.federated_scope_owner.clone()),
1317        ));
1318        self.scope_authority_owners = owners;
1319        self
1320    }
1321
1322    /// Set the exclusive federated owner while building the handler, before syncs.
1323    pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1324        self.scopes = Arc::new(RwLock::new(
1325            ScopeTable::new(self.scope_authority_owners.iter().cloned())
1326                .with_federated_owner(owner.clone()),
1327        ));
1328        self.federated_scope_owner = owner;
1329        self
1330    }
1331
1332    /// Set the startup policy before installing the rescan context or serving requests.
1333    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1334        self.privileged_operator_prompt = mode;
1335        self
1336    }
1337
1338    /// Override the route.bind relay timeout. Used by tests that assert the
1339    /// timeout path so they don't block on the production-safe default.
1340    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1341        self.route_bind_relay_timeout = timeout;
1342        self
1343    }
1344
1345    /// Install per-module route.bind relay budget overrides. A module id
1346    /// listed here wins over the daemon-wide default set via
1347    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1348    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1349    /// the same `Duration` the bind path will use.
1350    pub fn with_route_bind_relay_timeouts(
1351        mut self,
1352        timeouts: impl IntoIterator<Item = (String, Duration)>,
1353    ) -> Self {
1354        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1355        self
1356    }
1357
1358    /// Resolve the route.bind relay budget for a specific target module id.
1359    /// Per-module overrides win; the daemon-wide value (set via
1360    /// `with_route_bind_relay_timeout` or the built-in default) is the
1361    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1362    /// the same resolution `handle_route_open` will use.
1363    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1364        self.route_bind_relay_timeouts
1365            .get(module_id)
1366            .copied()
1367            .unwrap_or(self.route_bind_relay_timeout)
1368    }
1369
1370    /// Override the per-module bind-relay breaker policy.
1371    ///
1372    /// Used by tests, which cannot spend three production budgets opening a
1373    /// breaker or twenty seconds waiting for its cooldown. The production
1374    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1375    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1376    /// reasoning for the numbers.
1377    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1378        self.route_bind_breaker_threshold = threshold.max(1);
1379        self.route_bind_breaker_cooldown = cooldown;
1380        self
1381    }
1382
1383    #[cfg(test)]
1384    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1385        self.health_probe_timeout = timeout;
1386        self
1387    }
1388
1389    #[cfg(test)]
1390    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1391        self.control_dispatch_delay = Some(delay);
1392        self
1393    }
1394
1395    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1396    /// announce it is parked, once to be released. Build the barrier for two
1397    /// parties, the test being the other one.
1398    #[cfg(test)]
1399    pub(crate) fn with_route_bind_relay_sent_hold(
1400        mut self,
1401        hold: Arc<tokio::sync::Barrier>,
1402    ) -> Self {
1403        self.route_bind_relay_sent_hold = Some(hold);
1404        self
1405    }
1406
1407    /// A HELLO waits on `hold` twice between reserving its registry entry and
1408    /// installing its forwarding endpoint: once to announce it is parked, once
1409    /// to be released. Build the barrier for two parties, the test being the
1410    /// other one, and run the HELLO on a thread of its own.
1411    #[cfg(test)]
1412    pub(crate) fn with_hello_registry_to_forwarding_hold(
1413        mut self,
1414        hold: Arc<std::sync::Barrier>,
1415    ) -> Self {
1416        self.hello_registry_to_forwarding_hold = Some(hold);
1417        self
1418    }
1419
1420    #[cfg(test)]
1421    fn hold_between_registry_and_forwarding(&self) {
1422        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1423            hold.wait();
1424            hold.wait();
1425        }
1426    }
1427
1428    /// Park after the ACK is queued and forwarding is installed, until a test
1429    /// has read the ACK and issued a registry lookup from another connection.
1430    #[cfg(test)]
1431    fn hold_between_ack_and_publication(&self) {
1432        if let Some(hold) = &self.hello_ack_to_publication_hold {
1433            hold.wait();
1434            hold.wait();
1435        }
1436    }
1437
1438    #[cfg(all(test, unix))]
1439    pub(crate) fn with_hello_before_publication_hold(
1440        mut self,
1441        hold: Arc<std::sync::Barrier>,
1442    ) -> Self {
1443        self.hello_before_publication_hold = Some(hold);
1444        self
1445    }
1446
1447    #[cfg(all(test, unix))]
1448    pub(crate) fn with_hello_after_launch_check_hold(
1449        mut self,
1450        hold: Arc<std::sync::Barrier>,
1451    ) -> Self {
1452        self.hello_after_launch_check_hold = Some(hold);
1453        self
1454    }
1455
1456    #[cfg(all(test, unix))]
1457    pub(crate) fn with_hello_inside_publication_hold(
1458        mut self,
1459        hold: Arc<std::sync::Barrier>,
1460    ) -> Self {
1461        self.hello_inside_publication_hold = Some(hold);
1462        self
1463    }
1464
1465    pub fn with_process_liveness(
1466        mut self,
1467        process_liveness: Arc<dyn ModuleProcessLiveness>,
1468    ) -> Self {
1469        self.process_liveness = Some(process_liveness);
1470        self
1471    }
1472
1473    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1474        self.supervisor = supervisor;
1475        if let Some(owner) = &self.config_runtime {
1476            self.supervisor.install_binding_retirement(owner);
1477        }
1478        self.supervisor.set_activity(self.activity.clone());
1479        self
1480    }
1481
1482    pub fn with_daemon_provenance(
1483        mut self,
1484        pid: u32,
1485        started_at_ms: u64,
1486        executable_path: Option<PathBuf>,
1487        build_git_sha: Option<String>,
1488        build_lock_digest: Option<String>,
1489    ) -> Self {
1490        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1491        let process_start_time = process_start_time(pid);
1492        self.daemon_provenance = DaemonProvenanceFacts {
1493            build: DaemonBuildProvenance {
1494                build_git_sha,
1495                build_lock_digest,
1496            },
1497            pid: Some(pid),
1498            started_at_ms: Some(started_at_ms),
1499            start_clock: None,
1500            executable_path,
1501            executable_identity,
1502            process_start_time,
1503            probe: ExecutableIdentityProbe::default(),
1504        };
1505        self
1506    }
1507
1508    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1509        self.daemon_provenance.start_clock = Some(clock);
1510        self
1511    }
1512
1513    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1514    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1515        self.scopes
1516            .write()
1517            .unwrap_or_else(|poisoned| poisoned.into_inner())
1518            .set_wall_clock(Arc::new(clock));
1519        self
1520    }
1521
1522    #[cfg(test)]
1523    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1524        self.provenance_probe_override = Some(result);
1525        self
1526    }
1527
1528    /// Install the configured module set and its reserved capability bindings.
1529    /// Bindings are configuration-scoped and may point at a provider that has not
1530    /// been installed yet, so this does not require the bound module to exist.
1531    pub fn with_capability_config(
1532        self,
1533        modules: impl IntoIterator<Item = (String, bool)>,
1534        reserved_capabilities: BTreeMap<String, String>,
1535    ) -> Self {
1536        self.capability_evaluator
1537            .configure(modules, reserved_capabilities);
1538        self
1539    }
1540
1541    /// Share the supervisor's daemon-shutdown flag with this handler, so
1542    /// requirement changes after the daemon has begun its announced shutdown
1543    /// are not reported as faults.
1544    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1545        self.daemon_shutdown = flag;
1546        self
1547    }
1548
1549    pub fn with_supervisor_rescan(
1550        mut self,
1551        supervisor: Supervisor,
1552        config_path: impl Into<PathBuf>,
1553        configured_port: Option<u16>,
1554    ) -> Self {
1555        self.rescan = Some(SupervisorRescanContext {
1556            supervisor,
1557            config_path: config_path.into(),
1558            configured_port,
1559            storage_config: self.storage_config.clone(),
1560            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1561            admission_facts_targets: self.admission_facts_targets.clone(),
1562            scope_authority_owners: self.scope_authority_owners.clone(),
1563            federated_scope_owner: self.federated_scope_owner.clone(),
1564            privileged_operator_prompt: self.privileged_operator_prompt,
1565            generation: Arc::default(),
1566        });
1567        self
1568    }
1569
1570    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1571        self.connected_clients = connected_clients;
1572        self
1573    }
1574
1575    /// Report why this daemon serves no local endpoint, in `server.describe`, so
1576    /// an operator can see a bind failure without reading the daemon log.
1577    pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1578        self.local_endpoint_unavailable = reason;
1579        self
1580    }
1581
1582    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1583        Arc::clone(&self.forwarding)
1584    }
1585
1586    pub(crate) fn counters(&self) -> DaemonCounters {
1587        self.counters.clone()
1588    }
1589
1590    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1591    /// requirement event without depending on an operator polling a status command.
1592    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1593        let handler = Arc::downgrade(&self);
1594        let evaluator = self.capability_evaluator.clone();
1595        tokio::spawn(async move {
1596            loop {
1597                evaluator.wait_for_change_or_deadline().await;
1598                let Some(handler) = handler.upgrade() else {
1599                    break;
1600                };
1601                handler.refresh_capability_requirements();
1602            }
1603        });
1604    }
1605
1606    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1607    /// deadline passed during sleep is enforced on the first tick after wake.
1608    /// While no live scope carries a deadline the loop waits for an accepted
1609    /// scope change instead of ticking, so a daemon without deadlines never
1610    /// wakes for expiry.
1611    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1612        let context = scope_expiry::ScopeExpiryContext::new(&self);
1613        let deadline_added = self.scope_deadline_added.clone();
1614        tokio::spawn(async move {
1615            loop {
1616                if !context.has_deadlines() {
1617                    deadline_added.notified().await;
1618                    continue;
1619                }
1620                tokio::time::sleep(Duration::from_secs(1)).await;
1621                if let Err(error) = context.sweep() {
1622                    warn!(%error, "scope expiry sweep failed");
1623                }
1624            }
1625        });
1626    }
1627
1628    #[cfg(test)]
1629    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1630        scope_expiry::ScopeExpiryContext::new(self).sweep()
1631    }
1632
1633    fn publish_scope_changes(
1634        &self,
1635        changes: &[crate::scopes::ScopeTagChange],
1636        expired: &[crate::scopes::ScopeExpired],
1637    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1638        scope_expiry::ScopeExpiryContext::new(self).publish_scope_changes(changes, expired)
1639    }
1640
1641    fn runtime_capability_snapshot(
1642        &self,
1643    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1644        let runtime = self
1645            .supervisor
1646            .list()
1647            .into_iter()
1648            .map(|module| {
1649                let status = module.status().map_err(|err| {
1650                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1651                })?;
1652                Ok(RuntimeModule {
1653                    module_id: status.module_id,
1654                    state: status.state,
1655                    enabled: status.enabled,
1656                })
1657            })
1658            .collect::<Result<Vec<_>, RouterError>>()?;
1659        // Include every registration, published or not. The capability
1660        // evaluator counts a module as providing its capabilities as soon as it
1661        // has registered, even before `catalog.list` shows it, so a HELLO in
1662        // progress does not change other modules' required-capability
1663        // verdicts. See `Registry::list_registered_modules`.
1664        let registrations = self.registry.list_registered_modules().map_err(|err| {
1665            RouterError::backend(
1666                0,
1667                0,
1668                format!("failed to list capability registrations: {err}"),
1669            )
1670        })?;
1671        let registrations = registrations
1672            .into_iter()
1673            .map(|registration| RegisteredModule {
1674                module_id: registration.manifest.module_id,
1675                module_version: registration.manifest.module_version,
1676                capabilities: registration.manifest.capabilities,
1677            })
1678            .collect();
1679        Ok((runtime, registrations))
1680    }
1681
1682    /// The capability side effects of a module becoming the active registration
1683    /// for its id: cache its manifest (logging how it differs from the cached
1684    /// one), run the deny census when its declarations call for one, and recompute the
1685    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1686    /// candidate's does not, and the supervisor does it at promotion instead,
1687    /// through [`crate::supervise::SwapPromotionObserver`].
1688    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1689        let cached_registration = RegisteredModule {
1690            module_id: registration.manifest.module_id.clone(),
1691            module_version: registration.manifest.module_version.clone(),
1692            capabilities: registration.manifest.capabilities.clone(),
1693        };
1694        let change = self.capability_evaluator.record_hello(&cached_registration);
1695        log_manifest_change(&cached_registration.module_id, &change);
1696        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1697            self.enforce_capability_denies();
1698        }
1699        self.refresh_capability_requirements();
1700    }
1701
1702    /// Point the shared supervisor handle at this handler for swap promotions.
1703    /// Called wherever a handler is put behind the `Arc` the router serves, so
1704    /// it can be held weakly.
1705    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1706        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1707            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1708        self.supervisor.set_swap_promotion_observer(observer);
1709    }
1710
1711    pub fn refresh_capability_requirements(&self) {
1712        match self.runtime_capability_snapshot() {
1713            Ok((runtime, registrations)) => {
1714                log_requirement_events(
1715                    self.capability_evaluator
1716                        .evaluate_now(&runtime, &registrations),
1717                    &self.daemon_shutdown,
1718                );
1719            }
1720            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1721        }
1722    }
1723
1724    /// Reconcile only live, attested route bindings after a capability deny edge
1725    /// or target claim was added. This is deliberately a control-plane census:
1726    /// the opaque forwarding hot path must not grow a per-frame capability check.
1727    fn enforce_capability_denies(&self) {
1728        let registrations = match self.registry.list_registered_modules() {
1729            Ok(snapshot) => snapshot,
1730            Err(err) => {
1731                warn!(error = %err, "failed to read registrations for capability deny census");
1732                return;
1733            }
1734        };
1735        let manifests = registrations
1736            .into_iter()
1737            .map(|registration| {
1738                (
1739                    registration.manifest.module_id.clone(),
1740                    registration.manifest,
1741                )
1742            })
1743            .collect::<BTreeMap<_, _>>();
1744        let census = match self.forwarding.route_census(None) {
1745            Ok(census) => census,
1746            Err(err) => {
1747                warn!(error = %err, "failed to read route census for capability deny enforcement");
1748                return;
1749            }
1750        };
1751
1752        for (target_module_id, routes) in census {
1753            let Some(target_manifest) = manifests.get(&target_module_id) else {
1754                continue;
1755            };
1756            let mut closed_routes = Vec::new();
1757            let mut goodbyes = Vec::new();
1758            for route in routes {
1759                let Principal::Reserved {
1760                    module_id: opening_module_id,
1761                } = &route.principal
1762                else {
1763                    continue;
1764                };
1765                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1766                    continue;
1767                };
1768                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1769                    continue;
1770                };
1771
1772                match self.forwarding.release_client_route(
1773                    route.goodbye_target.connection_id,
1774                    route.goodbye_target.channel,
1775                    route.goodbye_target.epoch,
1776                ) {
1777                    Ok(RouteRelease::Removed(module_goodbye)) => {
1778                        warn!(
1779                            opening_module_id,
1780                            target_module_id,
1781                            capability,
1782                            "force-closing route because an attested capability deny edge now matches"
1783                        );
1784                        goodbyes.push(module_goodbye);
1785                        goodbyes.push(route.goodbye_target.clone());
1786                        closed_routes.push(route);
1787                    }
1788                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1789                    Err(err) => warn!(
1790                        opening_module_id,
1791                        target_module_id,
1792                        capability,
1793                        error = %err,
1794                        "failed to force-close capability-denied route"
1795                    ),
1796                }
1797            }
1798
1799            if closed_routes.is_empty() {
1800                continue;
1801            }
1802            send_route_control_pushes(
1803                &self.forwarding,
1804                closed_routes,
1805                ClientControlPush::RouteClosed {
1806                    module_id: target_module_id,
1807                    channels: Vec::new(),
1808                    reason: RouteCloseReason::CapabilityDenied,
1809                    drained: false,
1810                    abandoned: 0,
1811                    excluded_subscriptions: 0,
1812                    terminal: Some(false),
1813                },
1814            );
1815            // route.closed records the reason; the route GOODBYE ends the
1816            // client's handle and settles any calls still pending on it.
1817            self.emit_route_goodbyes(goodbyes);
1818        }
1819    }
1820
1821    /// Why a registered module is not accepting new route binds, or `None` when
1822    /// it is. This is the module's effective readiness: its declared readiness
1823    /// first, then every `need: required` capability it declares evaluating to
1824    /// `provided`. `route.open` and `catalog.list` both read it here so the
1825    /// catalog never reports a module routable that `route.open` would refuse.
1826    fn not_ready_reason(
1827        &self,
1828        registration: &crate::registry::ModuleRegistration,
1829    ) -> Option<NotReadyReason> {
1830        if !registration.ready {
1831            return Some(NotReadyReason {
1832                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1833                capability: None,
1834            });
1835        }
1836        self.first_unprovided_required_capability(registration)
1837            .map(|capability| NotReadyReason {
1838                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1839                capability: Some(capability),
1840            })
1841    }
1842
1843    /// The lexicographically first capability this registration declares
1844    /// `need: required` whose evaluator verdict is not `provided`.
1845    ///
1846    /// The verdicts are the capability evaluator's own; nothing here decides
1847    /// what "provided" means. The evaluator counts a capability provided as
1848    /// soon as a module claiming it has REGISTERED, not once that module is
1849    /// ready. That distinction is what keeps two modules that require each
1850    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1851    /// is ready", each would wait for the other to become ready first and
1852    /// neither ever would. Do not tighten it to readiness.
1853    ///
1854    /// A required capability with no verdict at all means this registration's
1855    /// HELLO or catalog.update landed after the last recompute; recompute once
1856    /// rather than let a missing verdict read as either answer. If it is still
1857    /// missing (the recompute itself failed) the capability counts as
1858    /// unprovided: the refusal is retryable, and routing a module whose
1859    /// required provider is unknown is the outcome this check exists to stop.
1860    fn first_unprovided_required_capability(
1861        &self,
1862        registration: &crate::registry::ModuleRegistration,
1863    ) -> Option<String> {
1864        let required = registration
1865            .manifest
1866            .capabilities
1867            .iter()
1868            .flat_map(|declarations| declarations.requires.iter())
1869            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1870            .map(|requirement| requirement.capability.as_str())
1871            .collect::<BTreeSet<_>>();
1872        if required.is_empty() {
1873            return None;
1874        }
1875        let module_id = registration.manifest.module_id.as_str();
1876        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1877        if required
1878            .iter()
1879            .any(|capability| verdict(capability).is_none())
1880        {
1881            self.refresh_capability_requirements();
1882        }
1883        required
1884            .into_iter()
1885            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1886            .map(str::to_string)
1887    }
1888
1889    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1890        self.capability_evaluator
1891            .statuses()
1892            .into_iter()
1893            .map(capability_requirement_status)
1894            .collect()
1895    }
1896
1897    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1898    /// registration-release watch. The signal is what the supervisor waits on
1899    /// before spawning a replacement, so it must only fire once forwarding
1900    /// teardown is also done (see [`Self::cleanup_connection`] /
1901    /// [`Self::handle_goodbye`]). Unpublished HELLO rollback uses the separate
1902    /// registry-only path below because publication already owns the spawn lock.
1903    fn deregister_connection(
1904        &self,
1905        connection_id: ConnectionId,
1906        reason: RegistrationEndReason,
1907    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1908        if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
1909            spawn.end_primary_registration();
1910        }
1911        self.registry
1912            .deregister_connection_with_reason(connection_id, reason)
1913    }
1914
1915    // HELLO publication owns the spawn transition guard. claim.mint is allowed
1916    // only after HELLO acceptance, so a failed first HELLO has no bearer tokens
1917    // or claim-bound consumers to revoke. Remove only its unpublished registry
1918    // entry without re-entering primary-registration teardown.
1919    fn rollback_unpublished_hello(
1920        &self,
1921        connection: ConnectionId,
1922    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1923        self.registry.deregister_connection_with_reason(
1924            connection,
1925            RegistrationEndReason::RegistrationFailed,
1926        )
1927    }
1928
1929    fn request_refusal_close(
1930        &self,
1931        connection: ConnectionId,
1932        sink: Option<&crate::FrameSink>,
1933        reason: CloseReason,
1934    ) {
1935        if let Some(sink) = sink {
1936            sink.terminate_ingress();
1937        }
1938        if self.claims.close_notifications_enabled() {
1939            self.forwarding.request_connection_close(connection, reason);
1940        }
1941    }
1942
1943    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1944        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1945            return None;
1946        }
1947        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1948            parse_client_control_request(&frame.body)
1949        else {
1950            return None;
1951        };
1952        Some(target_module_id(&target).to_string())
1953    }
1954
1955    pub(crate) fn route_open_capacity_refusal(
1956        &self,
1957        ctx: &RouteCtx,
1958        frame: &Frame,
1959        target_module_id: &str,
1960        in_flight: usize,
1961        limit: usize,
1962    ) -> Result<Frame, RouterError> {
1963        // Sent before the request reaches `handle_route_open`, so the caller
1964        // fields come from the raw frame here.
1965        let caller = self.route_open_caller_from_frame(ctx, frame);
1966        self.route_open_admission_refusal_frame(
1967            ctx,
1968            &caller,
1969            frame,
1970            target_module_id,
1971            "open_admission_full",
1972            (in_flight, limit),
1973            format!(
1974                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1975            ),
1976        )
1977    }
1978
1979    fn route_open_target_capacity_refusal(
1980        &self,
1981        ctx: &RouteCtx,
1982        caller: &RouteOpenCaller,
1983        frame: &Frame,
1984        target_module_id: &str,
1985        in_flight: usize,
1986    ) -> Result<Frame, RouterError> {
1987        self.route_open_admission_refusal_frame(
1988            ctx,
1989            caller,
1990            frame,
1991            target_module_id,
1992            "target_binds_full",
1993            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1994            format!(
1995                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1996            ),
1997        )
1998    }
1999
2000    /// Admission pressure clears as existing binds settle, so its refusal must
2001    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
2002    /// `module_reloading`, `module_warming`, `target_unavailable`, or
2003    /// `module_timeout`. `target_unavailable` is honest for an attempt that
2004    /// cannot currently reach its target; `module_timeout` would falsely claim
2005    /// that a wait expired. A new, cleaner code would be terminal to deployed
2006    /// clients, so it requires a client-tolerance rollout before daemon emission.
2007    #[allow(clippy::too_many_arguments)]
2008    fn route_open_admission_refusal_frame(
2009        &self,
2010        ctx: &RouteCtx,
2011        caller: &RouteOpenCaller,
2012        frame: &Frame,
2013        target_module_id: &str,
2014        reason: &'static str,
2015        (in_flight, limit): (usize, usize),
2016        message: impl Into<String>,
2017    ) -> Result<Frame, RouterError> {
2018        let code = error_codes::TARGET_UNAVAILABLE;
2019        self.counters.increment_route_open_refused(code);
2020        info!(
2021            target: "control",
2022            code,
2023            reason,
2024            module_id = ?target_module_id,
2025            connection_id = ctx.connection_id.get(),
2026            principal = %caller.principal,
2027            scope = %caller.scope,
2028            harness = %caller.harness,
2029            in_flight,
2030            limit,
2031            "route.open refused"
2032        );
2033        control_error_frame(frame, code, message.into())
2034    }
2035
2036    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
2037    ///
2038    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
2039    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
2040    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
2041    #[cfg(test)]
2042    pub fn handle_control(
2043        &self,
2044        connection_id: ConnectionId,
2045        frame: Frame,
2046    ) -> Result<Vec<Frame>, RouterError> {
2047        match frame.header.ty {
2048            FrameType::Ping => Ok(vec![pong(&frame)?]),
2049            FrameType::Hello => self.handle_hello(connection_id, None, frame),
2050            FrameType::Goodbye => self.handle_goodbye(connection_id),
2051            ty => Ok(vec![control_error_frame(
2052                &frame,
2053                "unsupported_control_frame",
2054                format!("unsupported channel-0 frame {ty:?}"),
2055            )?]),
2056        }
2057    }
2058
2059    pub async fn handle_control_frame(
2060        &self,
2061        ctx: &RouteCtx,
2062        frame: Frame,
2063    ) -> Result<Vec<Frame>, RouterError> {
2064        self.handle_control_frame_timed(ctx, frame, None).await
2065    }
2066
2067    #[cfg(any(test, feature = "test-support"))]
2068    pub fn with_test_operator_app_policy(
2069        mut self,
2070        policy: crate::operator_apps::TestOperatorAppPolicy,
2071    ) -> Self {
2072        self.operator_app_policy = policy.into();
2073        self
2074    }
2075
2076    /// Inject the Windows signature check for tests. The injected check
2077    /// answers the verified version string or the rejection reason; the
2078    /// daemon enforces the floor itself on the returned version.
2079    #[cfg(all(windows, any(test, feature = "test-support")))]
2080    pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
2081        self.windows_verify = Some(verify);
2082        self
2083    }
2084
2085    /// Record which pids the daemon would flash for `already_running`.
2086    #[cfg(all(windows, any(test, feature = "test-support")))]
2087    pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
2088        self.windows_flash = Some(flash);
2089        self
2090    }
2091
2092    /// Inject a tracking failure for tests: makes `track_launched_process`
2093    /// fail so a test can prove a launch that cannot be tracked is stopped,
2094    /// never left running.
2095    #[cfg(all(windows, any(test, feature = "test-support")))]
2096    pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
2097        self.windows_track = Some(track);
2098        self
2099    }
2100
2101    pub(crate) fn record_captured_peer(
2102        &self,
2103        connection: ConnectionId,
2104        peer: subc_os::local_peer::CapturedPeer,
2105    ) {
2106        self.record_observed_peer(connection, peer.observation().clone());
2107        let identity = self.operator_app_policy.config_identity(peer.clone());
2108        #[cfg(windows)]
2109        let identity = identity.or_else(|| self.config_windows_launch_identity(&peer));
2110        if let Some(identity) = identity {
2111            // Keep the connect-time launch incarnation. Looking it up afresh
2112            // from a numeric pipe-client PID could match a later reused PID.
2113            self.config_desktop_stamps
2114                .lock()
2115                .unwrap_or_else(|p| p.into_inner())
2116                .insert(connection, identity);
2117        }
2118        self.captured_peers
2119            .lock()
2120            .unwrap_or_else(|p| p.into_inner())
2121            .insert(connection, peer.clone());
2122        #[cfg(windows)]
2123        if let Some(stamp) = self.stamp_windows_launch(&peer) {
2124            self.operator_apps
2125                .lock()
2126                .unwrap_or_else(|p| p.into_inner())
2127                .insert(connection, stamp);
2128        }
2129        if let Some(stamp) = self.operator_app_policy.stamp(peer) {
2130            self.operator_apps
2131                .lock()
2132                .unwrap_or_else(|p| p.into_inner())
2133                .insert(connection, stamp);
2134        }
2135    }
2136
2137    pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
2138        self.local_connections
2139            .lock()
2140            .unwrap_or_else(|p| p.into_inner())
2141            .insert(connection);
2142    }
2143
2144    // Keep the connection-level delivery decision shared with route.bind. The
2145    // target's capability opt-in is separate from recognition of the caller.
2146    fn connection_operator_app(
2147        &self,
2148        stamp: Option<&crate::operator_apps::OperatorAppStamp>,
2149    ) -> Option<subc_protocol::peer::OperatorApp> {
2150        stamp
2151            .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
2152            .map(|stamp| stamp.app.clone())
2153    }
2154
2155    fn handle_connection_describe(
2156        &self,
2157        ctx: &RouteCtx,
2158        frame: Frame,
2159    ) -> Result<Vec<Frame>, RouterError> {
2160        if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
2161            return Ok(vec![control_error_frame(
2162                &frame,
2163                "invalid_control_body",
2164                "malformed connection.describe body",
2165            )?]);
2166        }
2167        let local = self
2168            .local_connections
2169            .lock()
2170            .unwrap_or_else(|p| p.into_inner())
2171            .contains(&ctx.connection_id);
2172        let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
2173        let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
2174        Ok(vec![control_response_body_frame(
2175            &frame,
2176            &subc_control::ConnectionDescribeReply::new(local, operator_app),
2177            ops::CONNECTION_DESCRIBE,
2178        )?])
2179    }
2180
2181    pub(crate) fn record_observed_peer(
2182        &self,
2183        connection: ConnectionId,
2184        peer: subc_os::local_peer::ObservedPeer,
2185    ) {
2186        use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
2187        let peer = match peer {
2188            OsPeer::Unobserved => ObservedPeer::unobserved(),
2189            OsPeer::Observed(peer) => {
2190                let observed = match peer.user {
2191                    PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
2192                    PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
2193                };
2194                observed
2195                    .with_exe(
2196                        peer.exe
2197                            .and_then(|path| path.into_os_string().into_string().ok()),
2198                    )
2199                    .with_signing_id(peer.signing_id)
2200            }
2201        };
2202        self.observed_peers
2203            .lock()
2204            .unwrap_or_else(|p| p.into_inner())
2205            .insert(connection, peer);
2206    }
2207
2208    pub(crate) fn record_caller_endpoints(
2209        &self,
2210        connection: ConnectionId,
2211        endpoints: Option<(SocketAddr, SocketAddr)>,
2212    ) {
2213        let mut callers = self
2214            .caller_endpoints
2215            .lock()
2216            .unwrap_or_else(|p| p.into_inner());
2217        callers.remove(&connection);
2218        if let Some(endpoints) = endpoints {
2219            callers.insert(connection, endpoints);
2220        }
2221    }
2222
2223    #[cfg(test)]
2224    pub(crate) fn with_caller_lookup_for_test(
2225        mut self,
2226        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2227    ) -> Self {
2228        self.caller_lookup = Some(Arc::new(lookup));
2229        self
2230    }
2231
2232    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2233        let endpoints = self
2234            .caller_endpoints
2235            .lock()
2236            .unwrap_or_else(|p| p.into_inner())
2237            .get(&connection)
2238            .copied();
2239        #[cfg(test)]
2240        let lookup = self.caller_lookup.clone();
2241        tokio::task::spawn_blocking(move || {
2242            #[cfg(test)]
2243            if let Some(lookup) = lookup {
2244                return lookup(endpoints);
2245            }
2246            endpoints
2247                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2248                .unwrap_or_default()
2249        })
2250        .await
2251        .unwrap_or_default()
2252    }
2253
2254    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2255        frame.header.channel == 0
2256            && frame.header.ty == FrameType::Request
2257            && self
2258                .forwarding
2259                .module_endpoint_for_connection(ctx.connection_id)
2260                .is_ok_and(|endpoint| endpoint.is_none())
2261            && parse_client_control_request(&frame.body)
2262                .ok()
2263                .is_some_and(|request| privileged_operation(&request).is_some())
2264    }
2265
2266    pub(crate) async fn handle_control_frame_timed(
2267        &self,
2268        ctx: &RouteCtx,
2269        frame: Frame,
2270        dispatch_started_at: Option<StdInstant>,
2271    ) -> Result<Vec<Frame>, RouterError> {
2272        let first = ctx.egress.first_ingress_frame();
2273        self.handle_control_frame_ingress(ctx, frame, dispatch_started_at, first)
2274            .await
2275    }
2276
2277    pub(crate) async fn handle_control_frame_ingress(
2278        &self,
2279        ctx: &RouteCtx,
2280        frame: Frame,
2281        dispatch_started_at: Option<StdInstant>,
2282        first: bool,
2283    ) -> Result<Vec<Frame>, RouterError> {
2284        if ctx.egress.ingress_terminal() {
2285            return Ok(Vec::new());
2286        }
2287        #[cfg(test)]
2288        self.claims.record_dispatch(frame.header.corr);
2289        let claim_binding = self.claims.binding(ctx.connection_id);
2290        let mut prefix_attempt = (frame.header.ty == FrameType::Hello
2291            && claim_binding
2292                .as_ref()
2293                .is_some_and(|binding| binding.prefix_module_id().is_some()))
2294        .then(|| self.prefix_hello_attempt(ctx));
2295        if claim_binding
2296            .as_ref()
2297            .is_some_and(|binding| binding.prefix_module_id().is_some())
2298        {
2299            let registered = self
2300                .forwarding
2301                .module_endpoint_for_connection(ctx.connection_id)
2302                .map_err(RouterError::Forwarding)?
2303                .is_some();
2304            if !registered && frame.header.ty != FrameType::Hello {
2305                return self
2306                    .refuse_claim_connection(ctx, &frame, "claim_hello_required")
2307                    .await;
2308            }
2309            if registered && frame.header.ty == FrameType::Goodbye {
2310                ctx.egress.terminate_ingress();
2311                let result = self.handle_goodbye(ctx.connection_id);
2312                self.claims.connection_closed(ctx.connection_id);
2313                self.request_refusal_close(
2314                    ctx.connection_id,
2315                    Some(&ctx.egress),
2316                    CloseReason::new("claim_registration_ended", "prefix registration ended"),
2317                );
2318                return result;
2319            }
2320            if registered && frame.header.ty == FrameType::Request {
2321                let allowed =
2322                    serde_json::from_slice::<ControlOpProbe>(&frame.body).is_ok_and(|probe| {
2323                        matches!(
2324                            probe.op.as_str(),
2325                            "catalog.update"
2326                                | "catalog.update_role_declarations"
2327                                | "scope.describe"
2328                                | "supervisor.live_roots"
2329                                | "activity.record"
2330                        )
2331                    });
2332                if !allowed {
2333                    return Ok(vec![control_error_frame(
2334                        &frame,
2335                        channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2336                        "prefix registrations have only brokered module operations",
2337                    )?]);
2338                }
2339            }
2340            if registered
2341                && frame.header.ty == FrameType::Push
2342                && !serde_json::from_slice::<ControlOpProbe>(&frame.body)
2343                    .is_ok_and(|probe| probe.op == "route.status")
2344            {
2345                return Ok(vec![control_error_frame(
2346                    &frame,
2347                    channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2348                    "prefix registrations may push only route.status for routes they serve",
2349                )?]);
2350            }
2351        }
2352        match frame.header.ty {
2353            FrameType::Ping => Ok(vec![pong(&frame)?]),
2354            FrameType::Hello => {
2355                if claim_binding
2356                    .as_ref()
2357                    .is_some_and(|binding| binding.prefix_module_id().is_none())
2358                {
2359                    return self
2360                        .refuse_claim_connection(ctx, &frame, "claim_hello_not_allowed")
2361                        .await;
2362                }
2363                // Resolved once, here: the blocking HELLO below may outlive this
2364                // reader and run after cleanup has forgotten the channel, and it
2365                // must still be judged as this channel's HELLO.
2366                let channel_spawn = self.supervisor.registration_spawn(ctx.connection_id);
2367                if let Some(spawn) = &channel_spawn {
2368                    spawn.wait_launch().await;
2369                    // Refused launches close the channel. Leave their pending
2370                    // HELLO unanswered instead of racing a reply against shutdown.
2371                    if !spawn.launch_ready() {
2372                        ctx.egress.terminate_ingress();
2373                        return Ok(Vec::new());
2374                    }
2375                    #[cfg(test)]
2376                    self.pause_hello_ingress_for_test().await;
2377                } else if claim_binding.is_none() {
2378                    if let Ok(hello) = serde_json::from_slice::<serde_json::Value>(&frame.body) {
2379                        if let (Some(module), Some(nonce)) = (
2380                            hello
2381                                .get("manifest")
2382                                .and_then(|value| value.get("module_id"))
2383                                .and_then(|value| value.as_str()),
2384                            hello.get("launch_nonce").and_then(|value| value.as_str()),
2385                        ) {
2386                            if !self
2387                                .config_runtime
2388                                .as_ref()
2389                                .is_some_and(|owner| owner.module_is_gated(module))
2390                                && !self.supervisor.wait_program_binding(module, nonce).await
2391                            {
2392                                self.request_refusal_close(
2393                                    ctx.connection_id,
2394                                    Some(&ctx.egress),
2395                                    CloseReason::new(
2396                                        "program_refused",
2397                                        "the spawned program binding failed",
2398                                    ),
2399                                );
2400                                return Ok(vec![control_error_frame(
2401                                    &frame,
2402                                    if self
2403                                        .config_runtime
2404                                        .as_ref()
2405                                        .is_some_and(|owner| owner.is_recovering())
2406                                    {
2407                                        "config_gated"
2408                                    } else {
2409                                        "program_refused"
2410                                    },
2411                                    "the spawned program binding failed",
2412                                )?]);
2413                            }
2414                        }
2415                    }
2416                }
2417                let handler = self.clone();
2418                let connection = ctx.connection_id;
2419                let egress = ctx.egress.clone();
2420                let correlation = frame.header.corr;
2421                let dispatcher = tracing::dispatcher::get_default(|dispatcher| dispatcher.clone());
2422                let span = tracing::Span::current();
2423                let hello_binding = claim_binding.clone();
2424                let replies = tokio::task::spawn_blocking(move || {
2425                    tracing::dispatcher::with_default(&dispatcher, || {
2426                        span.in_scope(|| {
2427                            handler.handle_hello_checked(
2428                                connection,
2429                                Some(egress),
2430                                frame,
2431                                true,
2432                                hello_binding,
2433                                channel_spawn,
2434                            )
2435                        })
2436                    })
2437                })
2438                .await
2439                .map_err(|error| RouterError::backend(0, correlation, error.to_string()))??;
2440                #[cfg(test)]
2441                if prefix_attempt.is_some() {
2442                    self.claims.pause_prefix_commit_for_test(false).await;
2443                }
2444                if let Some(attempt) = &mut prefix_attempt {
2445                    let accepted = !replies
2446                        .iter()
2447                        .any(|reply| reply.header.ty == FrameType::Error)
2448                        && self
2449                            .registry
2450                            .get_module_by_connection(ctx.connection_id)
2451                            .map_err(|err| RouterError::backend(0, correlation, err.to_string()))?
2452                            .is_some();
2453                    if accepted {
2454                        attempt.accepted = true;
2455                    } else {
2456                        ctx.egress.terminate_ingress();
2457                        for reply in replies {
2458                            ctx.egress.send_flushed(reply).await?;
2459                        }
2460                        return Ok(Vec::new());
2461                    }
2462                }
2463                Ok(replies)
2464            }
2465            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2466            FrameType::Cancel => {
2467                // Channel-0 cancellation withdraws approvals and subscriptions
2468                // and fences pending agent declarations;
2469                // successful cancellation has no ordinary reply.
2470                // A config Apply may also own an OS prompt. Cancel that prompt
2471                // before config cancellation can report the request handled.
2472                if self
2473                    .agent_declarations
2474                    .cancel(ctx.connection_id, frame.header.corr)
2475                    || self
2476                        .forwarding
2477                        .operator_confirms()
2478                        .cancel(ctx.connection_id, frame.header.corr)
2479                    || self.cancel_config_approval(ctx.connection_id, frame.header.corr)
2480                    || self
2481                        .supervisor
2482                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2483                    || self.cancel_activity_subscription(ctx, &frame)?
2484                    || self.cancel_catalog_subscription(ctx.connection_id, frame.header.corr)
2485                {
2486                    Ok(Vec::new())
2487                } else {
2488                    Ok(vec![control_error_frame(
2489                        &frame,
2490                        "unknown_subscription",
2491                        "no subscription has this correlation id",
2492                    )?])
2493                }
2494            }
2495            FrameType::Request => {
2496                // This additive operation is not part of the existing exhaustive
2497                // module-control enum. Probe the op before decoding that enum.
2498                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2499                let claim_present = op.as_ref().is_some_and(|probe| probe.op == "claim.present");
2500                if let Some(refusal) = self
2501                    .recovery_request_refusal(
2502                        ctx,
2503                        &frame,
2504                        op.as_ref().map(|probe| probe.op.as_str()),
2505                    )
2506                    .await?
2507                {
2508                    if claim_present {
2509                        self.reclaim_presented_expiry(&frame);
2510                        return self
2511                            .refuse_claim_connection(ctx, &frame, "config_gated")
2512                            .await;
2513                    }
2514                    return Ok(vec![refusal]);
2515                }
2516                if claim_present {
2517                    return self.handle_claim_present(ctx, &frame, first).await;
2518                }
2519                if op.as_ref().is_some_and(|probe| probe.op == "claim.mint") {
2520                    return self.handle_claim_mint(ctx, &frame).await;
2521                }
2522                if let Some(refusal) = self.registration_operation_refusal(
2523                    ctx,
2524                    &frame,
2525                    op.as_ref().map(|probe| probe.op.as_str()),
2526                )? {
2527                    return Ok(vec![refusal]);
2528                }
2529                if op
2530                    .as_ref()
2531                    .is_some_and(|probe| probe.op == ops::CATALOG_WATCH)
2532                {
2533                    return self.handle_catalog_watch(ctx, frame);
2534                }
2535                if op
2536                    .as_ref()
2537                    .is_some_and(|probe| probe.op == MODULE_TO_SUBC_OP_CATALOG_UPDATE)
2538                    && subc_protocol::provider_declarations::catalog_update_has_provider_declarations(&frame.body)
2539                        .unwrap_or(false)
2540                {
2541                    return Ok(vec![control_error_frame(
2542                        &frame,
2543                        error_codes::CATALOG_UPDATE_FROZEN_DECLARATIONS,
2544                        "provider_declarations are fixed at HELLO; restart the module to change them",
2545                    )?]);
2546                }
2547                if let Some(probe) = op
2548                    .as_ref()
2549                    .filter(|probe| config_gate::is_module_config_op(&probe.op))
2550                {
2551                    return self
2552                        .handle_module_config_request(ctx, frame, &probe.op)
2553                        .await;
2554                }
2555                if op
2556                    .as_ref()
2557                    .is_some_and(|probe| config_gate::is_config_op(&probe.op))
2558                {
2559                    return self.handle_config_gate_request(ctx, frame).await;
2560                }
2561                if op
2562                    .as_ref()
2563                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2564                {
2565                    return self.handle_activity_record(ctx, frame).await;
2566                }
2567                if op.as_ref().is_some_and(|probe| {
2568                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2569                }) {
2570                    return self.handle_activity_read(ctx, frame).await;
2571                }
2572                if op
2573                    .as_ref()
2574                    .is_some_and(|probe| probe.op == "operator.confirm")
2575                {
2576                    return self.handle_operator_confirm(ctx, frame);
2577                }
2578                if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2579                    return self
2580                        .observe_control_wait(
2581                            ctx,
2582                            None,
2583                            ops::APP_OPEN,
2584                            self.handle_app_open(ctx, frame),
2585                        )
2586                        .await;
2587                }
2588                if op
2589                    .as_ref()
2590                    .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2591                {
2592                    return self.handle_connection_describe(ctx, frame);
2593                }
2594                if op
2595                    .as_ref()
2596                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2597                {
2598                    return self.handle_scope_served_by(ctx.connection_id, frame);
2599                }
2600                if self
2601                    .forwarding
2602                    .module_endpoint_for_connection(ctx.connection_id)
2603                    .map_err(RouterError::Forwarding)?
2604                    .is_some()
2605                {
2606                    if !is_known_module_request_op(&frame.body) {
2607                        return Ok(vec![control_error_frame(
2608                            &frame,
2609                            "unsupported_control_frame",
2610                            "module-originated channel-0 REQUEST is not supported",
2611                        )?]);
2612                    }
2613                    let request = match parse_module_control_request_from_module(&frame.body) {
2614                        Ok(request) => request,
2615                        Err((err, ControlRequestBodyError::UnknownOp)) => {
2616                            return Ok(vec![control_error_frame(
2617                                &frame,
2618                                "unsupported_control_frame",
2619                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
2620                            )?])
2621                        }
2622                        Err((err, ControlRequestBodyError::InvalidBody)) => {
2623                            return Ok(vec![control_error_frame(
2624                                &frame,
2625                                "invalid_control_body",
2626                                format!("malformed module control body: {err}"),
2627                            )?])
2628                        }
2629                    };
2630                    let op = module_control_request_op(&request);
2631                    let corr = frame.header.corr;
2632                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
2633                    if let ModuleControlRequestFromModule::AgentCommands {
2634                        started,
2635                        finished,
2636                        known,
2637                    } = request
2638                    {
2639                        return self
2640                            .agent_declarations
2641                            .enqueue(
2642                                self,
2643                                ctx,
2644                                frame,
2645                                crate::resource_totals::Declaration {
2646                                    started,
2647                                    finished,
2648                                    known,
2649                                },
2650                            )
2651                            .await;
2652                    }
2653                    let result =
2654                        self.handle_module_control_request(ctx.connection_id, frame, request);
2655                    log_slow_control_dispatch(
2656                        dispatch_started_at,
2657                        op,
2658                        ctx.connection_id,
2659                        corr,
2660                        None,
2661                    );
2662                    return result;
2663                }
2664
2665                if is_known_module_request_op(&frame.body) {
2666                    let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2667                        .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2668                    let code = if is_agents {
2669                        "not_supervised"
2670                    } else if op
2671                        .as_ref()
2672                        .is_some_and(|probe| probe.op == subc_protocol::scope::ROUTE_AUTHORIZE_OP)
2673                    {
2674                        error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED
2675                    } else {
2676                        "not_registered"
2677                    };
2678                    return Ok(vec![control_error_frame(
2679                        &frame,
2680                        code,
2681                        "module-originated control requests require the module's own registered connection",
2682                    )?]);
2683                }
2684
2685                let request = match parse_client_control_request(&frame.body) {
2686                    Ok(request) => request,
2687                    Err((err, ControlRequestBodyError::UnknownOp)) => {
2688                        return Ok(vec![control_error_frame(
2689                            &frame,
2690                            "unknown_control_op",
2691                            format!("unknown client control op: {err}"),
2692                        )?])
2693                    }
2694                    Err((err, ControlRequestBodyError::InvalidBody)) => {
2695                        return Ok(vec![control_error_frame(
2696                            &frame,
2697                            "invalid_control_body",
2698                            format!("malformed client control body: {err}"),
2699                        )?])
2700                    }
2701                };
2702                let op = client_control_request_op(&request);
2703                let corr = frame.header.corr;
2704                log_control_dispatch_arrival(op, ctx.connection_id, corr);
2705                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2706                #[cfg(test)]
2707                if let Some(delay) = self.control_dispatch_delay {
2708                    tokio::time::sleep(delay).await;
2709                }
2710                let result = self
2711                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2712                    .await;
2713                if let Some(timings) = route_open_timings.as_mut() {
2714                    timings.finish();
2715                }
2716                log_slow_control_dispatch(
2717                    dispatch_started_at,
2718                    op,
2719                    ctx.connection_id,
2720                    corr,
2721                    route_open_timings.as_ref(),
2722                );
2723                result
2724            }
2725            FrameType::Push => {
2726                let Some(endpoint) = self
2727                    .forwarding
2728                    .module_endpoint_for_connection(ctx.connection_id)
2729                    .map_err(RouterError::Forwarding)?
2730                else {
2731                    return Ok(vec![control_error_frame(
2732                        &frame,
2733                        "unsupported_control_frame",
2734                        "client-originated channel-0 PUSH is not supported",
2735                    )?]);
2736                };
2737                self.handle_status_update(endpoint, frame)
2738            }
2739            FrameType::Response | FrameType::Error
2740                if self
2741                    .forwarding
2742                    .module_endpoint_for_connection(ctx.connection_id)
2743                    .map_err(RouterError::Forwarding)?
2744                    .is_some() =>
2745            {
2746                self.handle_module_relay_response(ctx.connection_id, frame)
2747            }
2748            ty => Ok(vec![control_error_frame(
2749                &frame,
2750                "unsupported_control_frame",
2751                format!("unsupported channel-0 frame {ty:?}"),
2752            )?]),
2753        }
2754    }
2755
2756    #[cfg(test)]
2757    fn pause_cleanup_for_test(&self, stage: u8) {
2758        let mut slot = self.cleanup_pause.lock().unwrap_or_else(|p| p.into_inner());
2759        if slot.as_ref().is_some_and(|(at, _)| *at == stage) {
2760            let (_, (reached, release)) = slot.take().unwrap();
2761            drop(slot);
2762            let _ = reached.send(());
2763            let _ = release.recv_timeout(Duration::from_secs(5));
2764        }
2765    }
2766
2767    /// Run `hook` once, in the next `route.open` that attests a launch nonce,
2768    /// right after the nonce is resolved.
2769    #[cfg(all(test, unix))]
2770    pub(crate) fn after_nonce_attestation_for_test(&self, hook: impl FnOnce() + Send + 'static) {
2771        *self
2772            .after_nonce_attestation
2773            .lock()
2774            .unwrap_or_else(|p| p.into_inner()) = Some(Box::new(hook));
2775    }
2776
2777    #[cfg(test)]
2778    pub(super) fn pause_after_nonce_attestation_for_test(&self) {
2779        let hook = self
2780            .after_nonce_attestation
2781            .lock()
2782            .unwrap_or_else(|p| p.into_inner())
2783            .take();
2784        if let Some(hook) = hook {
2785            hook();
2786        }
2787    }
2788
2789    #[cfg(test)]
2790    async fn pause_hello_ingress_for_test(&self) {
2791        let pause = self
2792            .hello_ingress_pause
2793            .lock()
2794            .unwrap_or_else(|p| p.into_inner())
2795            .take();
2796        if let Some((reached, release)) = pause {
2797            let _ = reached.send(());
2798            let _ = release.await;
2799        }
2800    }
2801
2802    pub fn cleanup_connection(
2803        &self,
2804        connection_id: ConnectionId,
2805    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2806        // The server loop joins its offloaded request tasks (route opens,
2807        // privileged requests and activity reads) before this cleanup runs, and
2808        // inline dispatch has ended. Two kinds of work are not joined. Agent
2809        // command declarations run on their queue's detached worker, so the
2810        // queue is closed first: every declaration still queued or validating
2811        // has its commit fence cancelled before anything below can change how
2812        // the connection resolves. A HELLO whose reader was dropped keeps
2813        // running on its blocking thread. It carries the channel spawn its
2814        // reader resolved at ingress instead of re-reading the index forgotten
2815        // below, so it is still judged as that channel's HELLO: the retired
2816        // spawn refuses it before it records anything, and a channel HELLO
2817        // never records a launch nonce, so it cannot leave this connection any
2818        // nonce authority.
2819        self.agent_declarations.close(connection_id);
2820        self.close_catalog_subscriptions(connection_id);
2821        #[cfg(test)]
2822        self.pause_cleanup_for_test(0);
2823        if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
2824            spawn.retire_registration_channel();
2825        }
2826        self.supervisor
2827            .forget_registration_connection(connection_id);
2828        #[cfg(test)]
2829        self.pause_cleanup_for_test(1);
2830        self.local_connections
2831            .lock()
2832            .unwrap_or_else(|p| p.into_inner())
2833            .remove(&connection_id);
2834        self.caller_endpoints
2835            .lock()
2836            .unwrap_or_else(|p| p.into_inner())
2837            .remove(&connection_id);
2838        self.build_class_warned
2839            .lock()
2840            .unwrap_or_else(|p| p.into_inner())
2841            .remove(&connection_id);
2842        self.observed_peers
2843            .lock()
2844            .unwrap_or_else(|p| p.into_inner())
2845            .remove(&connection_id);
2846        self.captured_peers
2847            .lock()
2848            .unwrap_or_else(|p| p.into_inner())
2849            .remove(&connection_id);
2850        self.config_desktop_stamps
2851            .lock()
2852            .unwrap_or_else(|p| p.into_inner())
2853            .remove(&connection_id);
2854        self.operator_apps
2855            .lock()
2856            .unwrap_or_else(|p| p.into_inner())
2857            .remove(&connection_id);
2858        self.forwarding
2859            .operator_confirms()
2860            .connection_closed(connection_id);
2861        let cleanup = self.cleanup_connection_with_end_reason(
2862            connection_id,
2863            RegistrationEndReason::ConnectionClosed,
2864        );
2865        // A draining writer can still be reached by a route until forwarding
2866        // cleanup finishes. Keep its claim sink visible to launch retirement.
2867        self.claims.connection_closed(connection_id);
2868        cleanup
2869    }
2870
2871    fn cleanup_connection_with_end_reason(
2872        &self,
2873        connection_id: ConnectionId,
2874        requested_reason: RegistrationEndReason,
2875    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2876        #[cfg(test)]
2877        if let Some(pause) = self
2878            .activity_request_pause
2879            .lock()
2880            .unwrap()
2881            .as_ref()
2882            .filter(|pause| pause.connection == connection_id)
2883        {
2884            pause
2885                .cleanup_started
2886                .store(true, std::sync::atomic::Ordering::Release);
2887        }
2888        if let Some(runtime) = &self.config_runtime {
2889            runtime.disconnected(connection_id);
2890        }
2891        self.agent_declarations.close(connection_id);
2892        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2893            self.registry
2894                .get_module_by_connection(connection_id)?
2895                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2896                .and_then(|module| module.registration_end_reason().ok().flatten())
2897                .unwrap_or(requested_reason)
2898        } else {
2899            requested_reason
2900        };
2901        let crash_closed = self
2902            .registry
2903            .get_module_by_connection(connection_id)?
2904            .and_then(|registration| {
2905                self.forwarding
2906                    .module_endpoint_for_connection(connection_id)
2907                    .ok()
2908                    .flatten()
2909                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2910                    .map(|routes| (registration.manifest.module_id, routes))
2911            });
2912        let crash_closed = crash_closed.map(|(module_id, routes)| {
2913            let terminal = match self.supervisor.get(&module_id) {
2914                None => false,
2915                Some(module) => match module.will_recover_after_connection_loss() {
2916                    Ok(will_recover) => !will_recover,
2917                    Err(err) => {
2918                        warn!(
2919                            %module_id,
2920                            error = %err,
2921                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2922                        );
2923                        false
2924                    }
2925                },
2926            };
2927            // The forwarding table gates all providers at the start of daemon
2928            // shutdown, before their connections are closed. An ordinary
2929            // module disconnect still reports crash if that gate is not set.
2930            let (reason, drained) = match self
2931                .forwarding
2932                .daemon_drain_result_for_connection(connection_id)
2933            {
2934                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2935                Ok(None) => (RouteCloseReason::Crash, false),
2936                Err(err) => {
2937                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2938                    (RouteCloseReason::Crash, false)
2939                }
2940            };
2941            (module_id, routes, reason, drained, terminal)
2942        });
2943        let registrations = self.deregister_connection(connection_id, end_reason);
2944        // Read before teardown: the spawn whose registration this connection
2945        // was. Its registration ending is an authority change for it.
2946        let ended_authority = self
2947            .forwarding
2948            .target_authority_for_connection(connection_id)
2949            .ok()
2950            .flatten();
2951        let cleanup = if crash_closed.is_some() {
2952            self.forwarding.cleanup_connection_counted(connection_id)
2953        } else {
2954            // A module connection's teardown needs the count of abandoned
2955            // route.bind relays for its route.closed notice below. Any other
2956            // connection, such as a client's, sends no such notice and needs
2957            // only its routes released, so it uses the route-only wrapper and
2958            // reports zero.
2959            self.forwarding
2960                .cleanup_connection(connection_id)
2961                .map(|released| crate::forwarding::ConnectionCleanup {
2962                    released,
2963                    abandoned_relays: 0,
2964                })
2965        };
2966        // The route.closed push waits for forwarding teardown because only
2967        // teardown knows how many pending route.bind relays it aborted. It still
2968        // goes out before the GOODBYEs for the released routes, and its targets
2969        // were captured above, before teardown removed those routes.
2970        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2971            let abandoned = cleanup
2972                .as_ref()
2973                .map_or(0, |cleanup| cleanup.abandoned_relays);
2974            send_route_control_pushes(
2975                &self.forwarding,
2976                routes,
2977                ClientControlPush::RouteClosed {
2978                    module_id,
2979                    channels: Vec::new(),
2980                    reason,
2981                    drained,
2982                    abandoned,
2983                    excluded_subscriptions: 0,
2984                    terminal: Some(terminal),
2985                },
2986            );
2987        }
2988        if let Ok(cleanup) = cleanup {
2989            self.emit_route_goodbyes(cleanup.released);
2990        }
2991        if let Some(record) = ended_authority {
2992            self.change_spawn_authority(&record);
2993        }
2994        // Signal the registration-release watch only now that BOTH registry and
2995        // forwarding teardown are done, so a supervisor waiting to spawn a
2996        // replacement never observes release while old routes still exist.
2997        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2998            crate::supervise::notify_registration_release();
2999            self.capability_evaluator.wake_deadline_loop();
3000            self.refresh_capability_requirements();
3001        }
3002        self.supervisor.remove_spawn_subscribers(connection_id);
3003        self.activity_streams
3004            .lock()
3005            .unwrap_or_else(|p| p.into_inner())
3006            .retain(|(id, _), task| {
3007                if *id == connection_id {
3008                    task.abort();
3009                    false
3010                } else {
3011                    true
3012                }
3013            });
3014        // Sync authority dies with its connection, so the owner's next
3015        // connection can take it; the owner's scopes stay as they are.
3016        self.hello_launch_nonces
3017            .lock()
3018            .unwrap_or_else(|poisoned| poisoned.into_inner())
3019            .forget(connection_id);
3020        self.scopes
3021            .write()
3022            .unwrap_or_else(|poisoned| poisoned.into_inner())
3023            .release_connection(connection_id);
3024        registrations
3025    }
3026
3027    pub(crate) fn handle_route_goodbye(
3028        &self,
3029        connection_id: ConnectionId,
3030        route_channel: u16,
3031        route_epoch: u32,
3032    ) -> Result<bool, RouterError> {
3033        debug!(
3034            connection_id = connection_id.get(),
3035            route_channel, route_epoch, "handling route GOODBYE"
3036        );
3037        let RouteRelease::Removed(released_route) = self
3038            .forwarding
3039            .release_client_route(connection_id, route_channel, route_epoch)
3040            .map_err(RouterError::Forwarding)?
3041        else {
3042            return Ok(false);
3043        };
3044        self.emit_route_goodbyes(vec![released_route]);
3045        Ok(true)
3046    }
3047
3048    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
3049        scope_expiry::ScopeExpiryContext::new(self).emit_route_goodbyes(released_routes);
3050    }
3051
3052    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
3053    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
3054    /// own commit failed after the module had already accepted). Without this, a
3055    /// module that accepts late keeps a binding subc has torn down, so a later
3056    /// frame on that module channel could misdeliver if the channel is reused.
3057    ///
3058    /// Never closes the shared module connection on failure: a dropped notification
3059    /// only wastes a bounded amount of warm module-side state, which the module's
3060    /// own idle reaper reclaims. Only call this once the route.bind relay was
3061    /// actually enqueued to the module — if the relay send itself failed, the
3062    /// module never created a binding and there is nothing to tear down.
3063    fn send_abandoned_route_bind_goodbye(
3064        &self,
3065        module_sink: &crate::FrameSink,
3066        negotiated_ver: u8,
3067        module_channel: u16,
3068        module_epoch: u32,
3069    ) {
3070        let frame = match Frame::build_with_version(
3071            negotiated_ver,
3072            FrameType::Goodbye,
3073            control_flags(),
3074            module_channel,
3075            module_epoch,
3076            0,
3077            Vec::new(),
3078        ) {
3079            Ok(frame) => frame,
3080            Err(err) => {
3081                warn!(
3082                    route_channel = module_channel,
3083                    error = %err,
3084                    "failed to build GOODBYE for abandoned route.bind"
3085                );
3086                return;
3087            }
3088        };
3089        crate::forwarding::send_module_route_goodbye(
3090            &self.counters,
3091            module_sink,
3092            frame,
3093            None,
3094            "abandoned route.bind",
3095        );
3096    }
3097
3098    #[cfg(test)]
3099    fn handle_hello(
3100        &self,
3101        connection_id: ConnectionId,
3102        egress: Option<crate::router::FrameSink>,
3103        frame: Frame,
3104    ) -> Result<Vec<Frame>, RouterError> {
3105        let binding = self.claims.binding(connection_id);
3106        let channel_spawn = self.supervisor.registration_spawn(connection_id);
3107        self.handle_hello_checked(connection_id, egress, frame, false, binding, channel_spawn)
3108    }
3109
3110    fn handle_hello_checked(
3111        &self,
3112        connection_id: ConnectionId,
3113        sink: Option<crate::FrameSink>,
3114        frame: Frame,
3115        wait_for_gate: bool,
3116        prefix_binding: Option<claims::ClaimBinding>,
3117        channel_spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3118    ) -> Result<Vec<Frame>, RouterError> {
3119        let primary_sink = channel_spawn.as_ref().and_then(|_| sink.clone());
3120        #[cfg(test)]
3121        if prefix_binding.is_some() {
3122            self.claims.pause_prefix_hello_for_test(5);
3123            match self.claims.take_prefix_hello_failure_for_test() {
3124                Some("error") => {
3125                    return Err(RouterError::backend(
3126                        0,
3127                        frame.header.corr,
3128                        "injected HELLO failure",
3129                    ))
3130                }
3131                Some("panic") => panic!("injected HELLO task panic"),
3132                _ => {}
3133            }
3134        }
3135        if let Some(binding) = &prefix_binding {
3136            let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3137            let module = value
3138                .as_ref()
3139                .and_then(|value| value["manifest"]["module_id"].as_str());
3140            if !binding.admit_prefix_hello(module)
3141                || !self.prefix_binding_live(binding, connection_id)
3142                || !module.is_some_and(|module| {
3143                    self.supervisor
3144                        .prefix_claim_authorized(&binding.registration.module_id, module)
3145                })
3146                || binding.registration.check().is_err()
3147            {
3148                return Ok(vec![control_error_frame(
3149                    &frame,
3150                    "claim_prefix_refused",
3151                    "HELLO must name the prefix claim's exact module id once",
3152                )?]);
3153            }
3154        }
3155        struct RefuseUnregisteredChannel {
3156            spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3157            sink: Option<crate::FrameSink>,
3158        }
3159        impl Drop for RefuseUnregisteredChannel {
3160            fn drop(&mut self) {
3161                if let Some(spawn) = &self.spawn {
3162                    if !spawn.registered() {
3163                        if let Some(sink) = &self.sink {
3164                            sink.terminate_ingress();
3165                        }
3166                        spawn.retire_registration_channel();
3167                    }
3168                }
3169            }
3170        }
3171        let _channel_attempt = RefuseUnregisteredChannel {
3172            spawn: channel_spawn.clone(),
3173            sink: sink.clone(),
3174        };
3175        if let Some(spawn) = &channel_spawn {
3176            let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3177            if let Err(code) = spawn.admit_hello(
3178                value
3179                    .as_ref()
3180                    .and_then(|value| value["manifest"]["module_id"].as_str()),
3181            ) {
3182                return Ok(vec![control_error_frame(
3183                    &frame,
3184                    code,
3185                    "registration channel HELLO refused",
3186                )?]);
3187            }
3188        }
3189        debug!(
3190            connection_id = connection_id.get(),
3191            corr = frame.header.corr,
3192            "handling HELLO"
3193        );
3194        // A module connection has one identity for its entire lifetime. A second
3195        // registration would leave the old registry owner behind while replacing
3196        // its forwarding endpoint and launch nonce.
3197        if self
3198            .registry
3199            .get_module_by_connection(connection_id)
3200            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3201            .is_some()
3202        {
3203            return Ok(vec![control_error_frame(
3204                &frame,
3205                "invalid_hello",
3206                "connection is already registered as a module",
3207            )?]);
3208        }
3209        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
3210            Ok(value) => value,
3211            Err(err) => {
3212                return Ok(vec![control_error_frame(
3213                    &frame,
3214                    "invalid_hello",
3215                    format!("malformed HELLO body: {err}"),
3216                )?])
3217            }
3218        };
3219        if let Err(detail) = catalog::validate_hello_declarations(&hello_value) {
3220            return Ok(vec![control_error_frame(
3221                &frame,
3222                error_codes::INVALID_PROVIDER_DECLARATIONS,
3223                detail,
3224            )?]);
3225        }
3226        if let Some(module) = hello_value
3227            .get("manifest")
3228            .and_then(|value| value.get("module_id"))
3229            .and_then(|value| value.as_str())
3230        {
3231            let recovering = self
3232                .config_runtime
3233                .as_ref()
3234                .is_some_and(|owner| owner.check_recovery().unwrap_or(true));
3235            // Defence in depth: RuntimeOwner::module_is_gated in
3236            // config_gate/admission.rs already gates every id during recovery.
3237            // A registration channel still never uses the nonce fallback.
3238            if recovering
3239                && channel_spawn.as_ref().is_none_or(|spawn| {
3240                    !self.supervisor.has_live_verified_channel_binding(spawn)
3241                        || spawn.check().is_err()
3242                })
3243                && !(channel_spawn.is_none()
3244                    && hello_value
3245                        .get("launch_nonce")
3246                        .and_then(|value| value.as_str())
3247                        .is_some_and(|nonce| {
3248                            self.supervisor.spawned_consumer_authorized(module, nonce)
3249                                && self
3250                                    .supervisor
3251                                    .has_live_verified_program_binding(module, nonce)
3252                        }))
3253            {
3254                return Ok(vec![control_error_frame(
3255                    &frame,
3256                    "config_gated",
3257                    "recovery admits only previously bound supervised launches",
3258                )?]);
3259            }
3260            if self
3261                .config_runtime
3262                .as_ref()
3263                .is_some_and(|owner| owner.module_is_gated(module))
3264                && (channel_spawn
3265                    .as_ref()
3266                    .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3267                    || self.config_runtime.as_ref().is_some_and(|owner| {
3268                        !recovering && owner.check_module_hold(module).is_err()
3269                    }))
3270            {
3271                return Ok(vec![control_error_frame(
3272                    &frame,
3273                    "reserved_module",
3274                    "a gated identity belongs only to its supervised child",
3275                )?]);
3276            }
3277        }
3278        if channel_spawn.is_none() {
3279            if let (Some(module), Some(nonce)) = (
3280                hello_value
3281                    .get("manifest")
3282                    .and_then(|value| value.get("module_id"))
3283                    .and_then(|value| value.as_str()),
3284                hello_value
3285                    .get("launch_nonce")
3286                    .and_then(|value| value.as_str()),
3287            ) {
3288                let binding = self.supervisor.has_program_binding_record(module, nonce);
3289                let required = self
3290                    .config_runtime
3291                    .as_ref()
3292                    .is_some_and(|owner| owner.module_is_gated(module));
3293                if let Some(error) = (binding || required)
3294                    .then(|| {
3295                        self.supervisor
3296                            .check_current_program_binding(module, nonce)
3297                            .err()
3298                    })
3299                    .flatten()
3300                {
3301                    let code = if error.code == "program_refused" {
3302                        "program_refused"
3303                    } else {
3304                        "config_delivery_deferred"
3305                    };
3306                    self.request_refusal_close(
3307                        connection_id,
3308                        sink.as_ref(),
3309                        CloseReason::new(code, error.detail.clone()),
3310                    );
3311                    return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3312                }
3313            }
3314        }
3315        if let Err(err) = validate_hello_display_name(&hello_value) {
3316            return Ok(vec![control_error_frame(
3317                &frame,
3318                "invalid_manifest",
3319                err.to_string(),
3320            )?]);
3321        }
3322        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
3323            return Ok(vec![control_error_frame(
3324                &frame,
3325                "invalid_capability_grammar",
3326                err.to_string(),
3327            )?]);
3328        }
3329        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
3330            return Ok(vec![control_error_frame(
3331                &frame,
3332                "invalid_manifest",
3333                err.to_string(),
3334            )?]);
3335        }
3336        if let Err(err) = validate_hello_event_declarations(&hello_value) {
3337            return Ok(vec![control_error_frame(
3338                &frame,
3339                "invalid_event_declaration",
3340                err.to_string(),
3341            )?]);
3342        }
3343        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
3344            return Ok(vec![control_error_frame(
3345                &frame,
3346                "invalid_manifest",
3347                err.to_string(),
3348            )?]);
3349        }
3350        if let Some(provenance) = hello_value
3351            .get("manifest")
3352            .and_then(|manifest| manifest.get("provenance"))
3353        {
3354            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
3355                return Ok(vec![control_error_frame(
3356                    &frame,
3357                    "invalid_manifest",
3358                    format!("malformed manifest provenance: {err}"),
3359                )?]);
3360            }
3361        }
3362        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
3363            let module_id = hello_value["manifest"]["module_id"]
3364                .as_str()
3365                .unwrap_or("<unknown>");
3366            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
3367        }
3368        if let Err(reason) = validate_hello_operation_deferral(&hello_value) {
3369            let module_id = hello_value
3370                .get("manifest")
3371                .and_then(|m| m.get("module_id"))
3372                .and_then(serde_json::Value::as_str)
3373                .unwrap_or("unknown");
3374            warn!(module_id, %reason, "dropping operation_deferral from HELLO");
3375        }
3376        if let Err(reason) = validate_hello_flow_operations(&hello_value) {
3377            let module_id = hello_value["manifest"]["module_id"]
3378                .as_str()
3379                .unwrap_or("<unknown>");
3380            warn!(module_id, %reason, "dropping flow_operations from HELLO");
3381        }
3382        if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
3383            let module_id = hello_value["manifest"]["module_id"]
3384                .as_str()
3385                .unwrap_or("<unknown>");
3386            warn!(module_id, %reason, "dropping confirm_operations from HELLO");
3387        }
3388        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
3389            Ok(hello) => hello,
3390            Err(err) => {
3391                return Ok(vec![control_error_frame(
3392                    &frame,
3393                    "invalid_hello",
3394                    format!("malformed HELLO body: {err}"),
3395                )?])
3396            }
3397        };
3398
3399        if hello.protocol_ver != hello.manifest.protocol_ver {
3400            return Ok(vec![control_error_frame(
3401                &frame,
3402                "invalid_manifest",
3403                format!(
3404                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
3405                    hello.protocol_ver, hello.manifest.protocol_ver
3406                ),
3407            )?]);
3408        }
3409
3410        if hello.manifest.module_id.trim().is_empty() {
3411            return Ok(vec![control_error_frame(
3412                &frame,
3413                "invalid_manifest",
3414                "manifest module_id must not be empty",
3415            )?]);
3416        }
3417
3418        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
3419            Ok(negotiated_ver) => negotiated_ver,
3420            Err(message) => {
3421                return Ok(vec![control_error_frame(
3422                    &frame,
3423                    "version_unsupported",
3424                    message,
3425                )?])
3426            }
3427        };
3428
3429        // Registration and forwarding publication use the same module gate as
3430        // stamp, unstamp and spawning. Recheck after taking it; parsing may
3431        // precede those transitions. A program-refusal hold blocks future
3432        // spawns, but does not take this gate or revoke a HELLO whose supervised
3433        // child's image has already passed the check.
3434        let _publication_gate = if let Some(owner) = &self.config_runtime {
3435            let gate = owner
3436                .module_gate(&hello.manifest.module_id)
3437                .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3438            Some(if wait_for_gate {
3439                gate.blocking_lock_owned()
3440            } else {
3441                match gate.try_lock_owned() {
3442                    Ok(guard) => guard,
3443                    Err(_) => {
3444                        return Ok(vec![control_error_frame(
3445                            &frame,
3446                            "config_delivery_deferred",
3447                            "module admission is busy",
3448                        )?])
3449                    }
3450                }
3451            })
3452        } else {
3453            None
3454        };
3455        #[cfg(test)]
3456        if let Some(hold) = &self.hello_before_publication_hold {
3457            hold.wait();
3458            hold.wait();
3459        }
3460        let module = &hello.manifest.module_id;
3461        let required = self
3462            .config_runtime
3463            .as_ref()
3464            .is_some_and(|owner| owner.module_is_gated(module));
3465        let recovering = self
3466            .config_runtime
3467            .as_ref()
3468            .is_some_and(|owner| owner.is_recovering());
3469        // Both entry paths (handle_control -> handle_hello and
3470        // handle_control_frame_timed -> spawn_blocking) enter this same
3471        // handle_hello_checked function. Its fresh, fail-closed store check
3472        // precedes parsing and this publication-gate check; this cached read
3473        // also observes any recovery transition published during that work.
3474        // The nonce-connection-only fallback is defence in depth alongside
3475        // RuntimeOwner::module_is_gated's all-id recovery gate.
3476        if recovering
3477            && channel_spawn.as_ref().is_none_or(|spawn| {
3478                !self.supervisor.has_live_verified_channel_binding(spawn) || spawn.check().is_err()
3479            })
3480            && !(channel_spawn.is_none()
3481                && hello.launch_nonce.as_deref().is_some_and(|nonce| {
3482                    self.supervisor.spawned_consumer_authorized(module, nonce)
3483                        && self
3484                            .supervisor
3485                            .has_live_verified_program_binding(module, nonce)
3486                }))
3487        {
3488            return Ok(vec![control_error_frame(
3489                &frame,
3490                "config_gated",
3491                "recovery admits only previously bound supervised launches",
3492            )?]);
3493        }
3494        if required
3495            && (channel_spawn
3496                .as_ref()
3497                .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3498                || self
3499                    .config_runtime
3500                    .as_ref()
3501                    .is_some_and(|owner| !recovering && owner.check_module_hold(module).is_err()))
3502        {
3503            return Ok(vec![control_error_frame(
3504                &frame,
3505                "reserved_module",
3506                "gated admission requires the current supervised child's registration channel and verified image binding",
3507            )?]);
3508        }
3509        if let Some(spawn) = &channel_spawn {
3510            if let Err(code) = spawn.check() {
3511                return Ok(vec![control_error_frame(
3512                    &frame,
3513                    code,
3514                    "registration launch is no longer live",
3515                )?]);
3516            }
3517        } else if let Some(nonce) = hello.launch_nonce.as_deref() {
3518            if required || self.supervisor.has_program_binding_record(module, nonce) {
3519                if let Err(error) = self.supervisor.check_current_program_binding(module, nonce) {
3520                    let code = if error.code == "program_refused" {
3521                        "program_refused"
3522                    } else {
3523                        "config_delivery_deferred"
3524                    };
3525                    self.request_refusal_close(
3526                        connection_id,
3527                        sink.as_ref(),
3528                        CloseReason::new(code, error.detail.clone()),
3529                    );
3530                    return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3531                }
3532            }
3533        }
3534
3535        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
3536        // swap is open for this id, the only HELLO admitted as a second process
3537        // is the one carrying the candidate's launch nonce (the swap token), and
3538        // it registers into the candidate slot rather than being refused as a
3539        // duplicate. Run after the reserved gate, a reserved module's candidate
3540        // would be refused `reserved_module` for presenting a nonce that gate
3541        // does not know. See `SupervisorHandle::swap_hello_admission`.
3542        let swap_admission = match &channel_spawn {
3543            Some(spawn) if spawn.is_candidate() => SwapHelloAdmission::Candidate,
3544            Some(_) => SwapHelloAdmission::NotSwapping,
3545            None if prefix_binding.is_some() => SwapHelloAdmission::NotSwapping,
3546            None => self
3547                .supervisor
3548                .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref()),
3549        };
3550        if swap_admission == SwapHelloAdmission::ProtocolNone {
3551            warn!(
3552                module_id = %hello.manifest.module_id,
3553                connection_id = connection_id.get(),
3554                "HELLO refused: the module_id belongs to a protocol:none module, which never registers"
3555            );
3556            // The code stays `swap_token_invalid` for existing callers; only the
3557            // explanation names the actual reason.
3558            return Ok(vec![control_error_frame(
3559                &frame,
3560                "swap_token_invalid",
3561                format!(
3562                    "module_id '{}' is a protocol:none module; a protocol:none module may not register",
3563                    hello.manifest.module_id
3564                ),
3565            )?]);
3566        }
3567        if swap_admission == SwapHelloAdmission::ChannelRequired {
3568            warn!(
3569                module_id = %hello.manifest.module_id,
3570                connection_id = connection_id.get(),
3571                "HELLO refused: the module_id is gated and the HELLO did not arrive over its registration channel"
3572            );
3573            // The code stays `swap_token_invalid` for existing callers; only the
3574            // explanation names the actual reason.
3575            return Ok(vec![control_error_frame(
3576                &frame,
3577                "swap_token_invalid",
3578                format!(
3579                    "module_id '{}' is gated; it must register over its supervised registration channel",
3580                    hello.manifest.module_id
3581                ),
3582            )?]);
3583        }
3584        if swap_admission == SwapHelloAdmission::Refused {
3585            warn!(
3586                module_id = %hello.manifest.module_id,
3587                connection_id = connection_id.get(),
3588                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
3589            );
3590            return Ok(vec![control_error_frame(
3591                &frame,
3592                "swap_token_invalid",
3593                format!(
3594                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
3595                    hello.manifest.module_id
3596                ),
3597            )?]);
3598        }
3599        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
3600
3601        // A swap candidate proves its identity with the candidate nonce above.
3602        // Other registrations receive this catalog fact only when reserved-module
3603        // policy required and accepted a nonce; unreserved registrations require
3604        // no nonce check.
3605        let verification = if swap_candidate && channel_spawn.is_none() {
3606            ReservedHelloVerification {
3607                nonce_verified: true,
3608                prefix_owner: None,
3609            }
3610        } else {
3611            let verification = if let Some(binding) = &prefix_binding {
3612                if !self.supervisor.prefix_claim_authorized(
3613                    &binding.registration.module_id,
3614                    &hello.manifest.module_id,
3615                ) {
3616                    return Ok(vec![control_error_frame(
3617                        &frame,
3618                        "claim_prefix_refused",
3619                        "prefix ownership changed",
3620                    )?]);
3621                }
3622                Ok(ReservedHelloVerification {
3623                    nonce_verified: true,
3624                    prefix_owner: Some(binding.registration.module_id.clone()),
3625                })
3626            } else if channel_spawn.is_some() {
3627                self.supervisor
3628                    .channel_hello_verification(&hello.manifest.module_id)
3629            } else {
3630                self.supervisor.reserved_hello_verification(
3631                    &hello.manifest.module_id,
3632                    hello.launch_nonce.as_deref(),
3633                )
3634            };
3635            match verification {
3636                Ok(verified) => verified,
3637                Err(rejection) => {
3638                    let message = match rejection {
3639                        ReservedHelloRejection::Exact { module_id } => format!(
3640                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
3641                        ),
3642                        ReservedHelloRejection::Prefix {
3643                            prefix,
3644                            owner_module_id,
3645                        } => format!(
3646                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
3647                            hello.manifest.module_id
3648                        ),
3649                    };
3650                    return Ok(vec![control_error_frame(
3651                        &frame,
3652                        "reserved_module",
3653                        message,
3654                    )?]);
3655                }
3656            }
3657        };
3658
3659        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
3660            &hello.manifest.module_id,
3661            hello.manifest.capabilities.as_ref(),
3662        );
3663        if let Some(refusal) = reserved_capability_refusals.first() {
3664            let capability = refusal.capability.clone();
3665            let bound_module = refusal.claimants[0].clone();
3666            log_duplicate_claim_events(reserved_capability_refusals);
3667            return Ok(vec![control_error_frame(
3668                &frame,
3669                "reserved_capability",
3670                format!(
3671                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3672                    capability, bound_module, hello.manifest.module_id
3673                ),
3674            )?]);
3675        }
3676
3677        // A connection that already opened client routes must not also register as
3678        // a module: cleanup would then release only one side and leak the other.
3679        if self
3680            .forwarding
3681            .connection_has_client_routes(connection_id)
3682            .map_err(RouterError::Forwarding)?
3683        {
3684            return Ok(vec![control_error_frame(
3685                &frame,
3686                "invalid_hello",
3687                "connection has open client routes and cannot also register as a module",
3688            )?]);
3689        }
3690
3691        // Kept for scope sync authority, which goes only to the connection that
3692        // presented the module's current launch nonce. Recorded before the
3693        // registration is attempted: a connection whose registration then fails
3694        // has no registration, so it cannot sync anyway, and cleanup forgets it.
3695        // A registration channel's authority is the channel itself, never a
3696        // nonce it also presented, and a prefix registration's is its claim:
3697        // neither records one, so neither can fall back to nonce authority.
3698        self.hello_launch_nonces
3699            .lock()
3700            .unwrap_or_else(|poisoned| poisoned.into_inner())
3701            .record(
3702                connection_id,
3703                if prefix_binding.is_some() || channel_spawn.is_some() {
3704                    None
3705                } else {
3706                    hello.launch_nonce.as_deref()
3707                },
3708            );
3709        let control_ops = effective_module_control_ops(hello.control_ops);
3710        let prefix_submodule = verification.prefix_owner.is_some();
3711        // Built before anything is registered so an encoding failure leaves no
3712        // registry or forwarding state behind.
3713        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
3714        #[cfg(test)]
3715        struct PublicationFinished(Arc<std::sync::Barrier>);
3716        #[cfg(test)]
3717        impl Drop for PublicationFinished {
3718            fn drop(&mut self) {
3719                self.0.wait();
3720            }
3721        }
3722        #[cfg(test)]
3723        let _publication_finished = self.hello_after_launch_check_hold.as_ref().map(|hold| {
3724            hold.wait();
3725            hold.wait();
3726            PublicationFinished(hold.clone())
3727        });
3728        let publication_spawn = channel_spawn
3729            .as_ref()
3730            .or_else(|| prefix_binding.as_ref().map(|binding| &binding.registration));
3731        #[cfg(test)]
3732        if prefix_binding.is_some() {
3733            self.claims.pause_prefix_hello_for_test(7);
3734        }
3735        let _registration_publication = match publication_spawn {
3736            Some(spawn) => Some(match spawn.begin_publication() {
3737                Ok(guard) => guard,
3738                Err(code) => {
3739                    return Ok(vec![control_error_frame(
3740                        &frame,
3741                        code,
3742                        "registration ended before HELLO publication",
3743                    )?])
3744                }
3745            }),
3746            None => None,
3747        };
3748        let (_prefix_live, _prefix_ownership) = if let Some(binding) = &prefix_binding {
3749            let live = self.claims.lock_prefix_binding(binding, connection_id);
3750            let ownership = self.supervisor.prefix_claim_publication(
3751                &binding.registration.module_id,
3752                &hello.manifest.module_id,
3753            );
3754            if live.is_none()
3755                || ownership.is_none()
3756                || !self.claim_primary_registered(binding.spawn, &binding.registration)
3757                || binding.prefix_module_id() != Some(hello.manifest.module_id.as_str())
3758            {
3759                return Ok(vec![control_error_frame(
3760                    &frame,
3761                    "claim_prefix_refused",
3762                    "prefix authority ended before publication",
3763                )?]);
3764            }
3765            (live, ownership)
3766        } else {
3767            (None, None)
3768        };
3769        if let Some(spawn) = publication_spawn {
3770            self.forwarding
3771                .bind_spawn(connection_id, spawn)
3772                .map_err(RouterError::Forwarding)?;
3773        }
3774        // The spawn whose own registration this HELLO is, for `target` on
3775        // the route.open reply: its channel, or the launch its nonce was
3776        // minted for. A prefix child's HELLO names no spawn of its own. It is
3777        // only looked up here. The connection is bound to it and its
3778        // requested declarations are frozen once the registry has accepted
3779        // this HELLO (`admit_hello_authority`), so a HELLO refused as a
3780        // duplicate, or for any other reason, leaves the spawn untouched: its
3781        // closing connection is then no registration of the spawn and ends
3782        // none of the spawn's routes.
3783        let hello_authority = match (&channel_spawn, &prefix_binding) {
3784            (Some(spawn), _) => Some(spawn.authority.clone()),
3785            (None, None) => hello.launch_nonce.as_deref().and_then(|nonce| {
3786                self.supervisor
3787                    .spawned_consumer_record(&hello.manifest.module_id, nonce)
3788            }),
3789            (None, Some(_)) => None,
3790        }
3791        .map(|record| HelloAuthority {
3792            record,
3793            requested: hello
3794                .manifest
3795                .provider_declarations
3796                .clone()
3797                .unwrap_or_default(),
3798            froze: Default::default(),
3799        });
3800        #[cfg(test)]
3801        if prefix_binding.is_some() {
3802            self.claims.pause_prefix_hello_for_test(8);
3803        }
3804        if swap_candidate {
3805            let module_id = hello.manifest.module_id.clone();
3806            let mut differing_declarations = Default::default();
3807            let reply = self.register_swap_candidate(
3808                connection_id,
3809                sink,
3810                &frame,
3811                hello.manifest,
3812                negotiated_ver,
3813                control_ops,
3814                hello_ack,
3815                hello_authority.as_ref(),
3816                &mut differing_declarations,
3817                #[cfg(windows)]
3818                hello.launch_nonce.as_deref(),
3819            )?;
3820            if let Some(spawn) = &channel_spawn {
3821                if self
3822                    .registry
3823                    .get_module_by_connection(connection_id)
3824                    .ok()
3825                    .flatten()
3826                    .is_some()
3827                {
3828                    if let Some(sink) = primary_sink {
3829                        spawn.bind_primary_sink(sink);
3830                    }
3831                    spawn.hello_accepted();
3832                }
3833            } else if self
3834                .registry
3835                .get_module_by_connection(connection_id)
3836                .ok()
3837                .flatten()
3838                .is_some()
3839            {
3840                self.supervisor
3841                    .retire_unused_registration_channel(&module_id, true);
3842            }
3843            if self
3844                .registry
3845                .get_module_by_connection(connection_id)
3846                .ok()
3847                .flatten()
3848                .is_some()
3849            {
3850                self.file_development_widening(
3851                    hello_authority.map(|authority| authority.record),
3852                    differing_declarations,
3853                );
3854            }
3855            return Ok(reply);
3856        }
3857        // Registered unpublished: the module id is claimed, so a duplicate HELLO
3858        // is refused, but `catalog.list` does not show the module and the
3859        // catalog generation does not move until it is published below, once
3860        // its forwarding endpoint exists. Listing it now would advertise a
3861        // module that `route.open` still refuses for having no forwarding
3862        // connection.
3863        let registration = match self
3864            .registry
3865            .register_unpublished_verified_with_control_ops(
3866                hello.manifest,
3867                negotiated_ver,
3868                connection_id,
3869                control_ops,
3870                verification.nonce_verified,
3871            ) {
3872            Ok(registration) => registration,
3873            Err(RegistryError::DuplicateModuleId { module_id }) => {
3874                return Ok(vec![control_error_frame(
3875                    &frame,
3876                    "duplicate_module_id",
3877                    format!(
3878                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
3879                    ),
3880                )?])
3881            }
3882            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3883                return Ok(vec![control_error_frame(
3884                    &frame,
3885                    "invalid_module_id",
3886                    err.to_string(),
3887                )?])
3888            }
3889            Err(err) => {
3890                return Ok(vec![control_error_frame(
3891                    &frame,
3892                    "registry_error",
3893                    err.to_string(),
3894                )?])
3895            }
3896        };
3897        let differing_declarations =
3898            match self.admit_hello_authority(connection_id, hello_authority.as_ref()) {
3899                Ok(differing) => differing,
3900                Err(err) => {
3901                    if matches!(
3902                        self.rollback_unpublished_hello(connection_id),
3903                        Ok(r) if !r.is_empty()
3904                    ) {
3905                        crate::supervise::notify_registration_release();
3906                    }
3907                    return Err(RouterError::Forwarding(err));
3908                }
3909            };
3910        #[cfg(test)]
3911        self.hold_between_registry_and_forwarding();
3912
3913        let mut reply = Vec::new();
3914        // Registry readers must wait from before the ACK can be read until
3915        // publication. Nest forwarding inside the registry lock, never reverse.
3916        let publication = self
3917            .registry
3918            .publish_registration_after(connection_id, || {
3919                if let Some(sink) = sink {
3920                    // The forwarding table's module store is also the daemon-to-module
3921                    // control-RPC lane, so every HELLO gets a live endpoint even when the
3922                    // manifest has no routable provider role. Non-routable modules still
3923                    // cannot receive route.bind in production: `handle_route_open` checks
3924                    // the registry manifest with `target_has_required_role` before the
3925                    // only production call to `begin_route_bind_relay_for` below that
3926                    // route.open path. The remaining direct relay callers are unit tests
3927                    // and benchmark harnesses that construct forwarding state explicitly.
3928                    //
3929                    // The HELLO_ACK is queued by the forwarding table itself, before the
3930                    // endpoint becomes visible, and is NOT returned as a reply. A module
3931                    // reads HELLO_ACK first and exits on anything else; a reply is only
3932                    // written after this handler returns, by which time a route.open on
3933                    // another connection could already have queued a route.bind request
3934                    // for this module ahead of it.
3935                    let concurrency = manifest_concurrency(&registration.manifest);
3936                    self.forwarding.register_module_connection_acked(
3937                        connection_id,
3938                        registration.manifest.module_id.clone(),
3939                        negotiated_ver,
3940                        concurrency,
3941                        sink,
3942                        crate::forwarding::ModuleHelloAck {
3943                            frame: hello_ack,
3944                            verified_prefix_owner: verification.prefix_owner,
3945                        },
3946                    )?;
3947                } else {
3948                    // No sink means no forwarding endpoint, so nothing can be routed
3949                    // ahead of the ack; it goes out as the reply.
3950                    reply.push(hello_ack);
3951                }
3952                #[cfg(test)]
3953                self.hold_between_ack_and_publication();
3954                Ok(())
3955            })
3956            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3957        let published = match publication {
3958            Ok(published) => published,
3959            Err(err) => {
3960                // The registry lock has been released, so rollback can take it
3961                // again. No forwarding endpoint or route was published. Keep
3962                // forwarding's weak connection-to-spawn association until the
3963                // connection's read loop ends and removes its remaining routes.
3964                // Removing this unpublished entry does not advance the catalog
3965                // generation.
3966                if matches!(
3967                    self.rollback_unpublished_hello(connection_id),
3968                    Ok(r) if !r.is_empty()
3969                ) {
3970                    crate::supervise::notify_registration_release();
3971                }
3972                self.refuse_hello_authority(connection_id, hello_authority.as_ref());
3973                return Ok(vec![control_error_frame(
3974                    &frame,
3975                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
3976                        "invalid_hello"
3977                    } else {
3978                        forwarding_error_code(&err)
3979                    },
3980                    err.to_string(),
3981                )?]);
3982            }
3983        };
3984        if !published {
3985            debug!(
3986                module_id = %registration.manifest.module_id,
3987                connection_id = connection_id.get(),
3988                "registration was removed before it could be published"
3989            );
3990            self.refuse_hello_authority(connection_id, hello_authority.as_ref());
3991            return Ok(reply);
3992        }
3993
3994        drop(_prefix_ownership);
3995        drop(_prefix_live);
3996
3997        // Exposure over assumption: Concurrency's serde default is pinned to the
3998        // pre-field behavior (ModuleManaged), so a management surface that is
3999        // genuinely Serial and just never declared it inherits concurrent
4000        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
4001        // turns "no module has been bitten yet" into the checkable claim "no
4002        // module is exposed" -- one read of the boot log instead of a fleet
4003        // audit. Detected from the raw HELLO bytes because the serde default
4004        // deliberately erases the absent/declared distinction from the type.
4005        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
4006            info!(
4007                module_id = %registration.manifest.module_id,
4008                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
4009            );
4010        }
4011        if registration.manifest.registration_channel == Some(false)
4012            && self
4013                .supervisor
4014                .module_is_gated(&registration.manifest.module_id)
4015            && self
4016                .config_runtime
4017                .as_ref()
4018                .and_then(|owner| owner.registration_declaration(&registration.manifest.module_id))
4019                .as_deref()
4020                == Some("declared")
4021        {
4022            warn!(module_id = %registration.manifest.module_id,
4023                "HELLO registration_channel is false despite the approved channel declaration; registration remains admitted");
4024        }
4025
4026        self.apply_registration_capabilities(&registration);
4027
4028        self.supervisor
4029            .record_activity_registration(&registration.manifest.module_id);
4030
4031        info!(
4032            module_id = %registration.manifest.module_id,
4033            module_version = %registration.manifest.module_version,
4034            negotiated_ver,
4035            routable_provider = manifest_provides_routable_role(&registration.manifest),
4036            connection_id = connection_id.get(),
4037            "module registered"
4038        );
4039
4040        if let Some(spawn) = &channel_spawn {
4041            if let Some(sink) = primary_sink {
4042                spawn.bind_primary_sink(sink);
4043            }
4044            spawn.hello_accepted();
4045        } else if !prefix_submodule {
4046            self.supervisor
4047                .retire_unused_registration_channel(&registration.manifest.module_id, false);
4048        }
4049        self.file_development_widening(
4050            hello_authority.map(|authority| authority.record),
4051            differing_declarations,
4052        );
4053        #[cfg(windows)]
4054        if let Some(nonce) = hello.launch_nonce.as_deref() {
4055            self.supervisor
4056                .program_hello_accepted(&registration.manifest.module_id, nonce);
4057        }
4058
4059        #[cfg(test)]
4060        if let Some(hold) = &self.hello_inside_publication_hold {
4061            hold.wait();
4062            hold.wait();
4063        }
4064
4065        Ok(reply)
4066    }
4067
4068    /// Register a HELLO the swap gate admitted into the candidate slot of the
4069    /// registry and of forwarding, where it is reachable over its own
4070    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
4071    /// nothing routes to it until the supervisor cuts over.
4072    ///
4073    /// Registry first (unpublished), then forwarding, then publication, the
4074    /// same order as an ordinary HELLO; a forwarding failure removes the
4075    /// registry entry again. The swap supervisor treats a published, ready
4076    /// candidate as registered and promotes it, and promotion needs the
4077    /// candidate's forwarding endpoint, so the candidate is published only
4078    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
4079    /// recompute capability verdicts or run the capability deny census
4080    /// (`enforce_capability_denies`): both are about the module in the active
4081    /// slot, which `route.open` can target. A candidate sits in the candidate
4082    /// slot, so it neither provides capabilities to other modules nor can be
4083    /// routed to until promotion moves it into the active slot.
4084    #[allow(clippy::too_many_arguments)]
4085    fn register_swap_candidate(
4086        &self,
4087        connection_id: ConnectionId,
4088        sink: Option<crate::FrameSink>,
4089        frame: &Frame,
4090        manifest: ModuleManifest,
4091        negotiated_ver: u8,
4092        control_ops: Vec<String>,
4093        hello_ack: Frame,
4094        hello_authority: Option<&HelloAuthority>,
4095        differing_declarations: &mut ProviderDeclarations,
4096        #[cfg(windows)] launch_nonce: Option<&str>,
4097    ) -> Result<Vec<Frame>, RouterError> {
4098        let module_id = manifest.module_id.clone();
4099        // Closing a swap and publishing its candidate must be ordered. The
4100        // spawn role proves which slot is intended, not that the slot is open.
4101        self.supervisor
4102            .with_swap_candidate_admission(&module_id, || {
4103                let reply = self.publish_swap_candidate(
4104                    connection_id,
4105                    sink,
4106                    frame,
4107                    manifest,
4108                    negotiated_ver,
4109                    control_ops,
4110                    hello_ack,
4111                    hello_authority,
4112                    differing_declarations,
4113                    #[cfg(windows)]
4114                    launch_nonce,
4115                );
4116                let admitted = self
4117                    .registry
4118                    .get_candidate(&module_id)
4119                    .ok()
4120                    .flatten()
4121                    .is_some_and(|candidate| candidate.connection_id == connection_id);
4122                (reply, admitted)
4123            })
4124            .unwrap_or_else(|| {
4125                Ok(vec![control_error_frame(
4126                    frame,
4127                    "swap_token_invalid",
4128                    "swap ended before candidate HELLO publication",
4129                )?])
4130            })
4131    }
4132
4133    #[allow(clippy::too_many_arguments)]
4134    fn publish_swap_candidate(
4135        &self,
4136        connection_id: ConnectionId,
4137        sink: Option<crate::FrameSink>,
4138        frame: &Frame,
4139        manifest: ModuleManifest,
4140        negotiated_ver: u8,
4141        control_ops: Vec<String>,
4142        hello_ack: Frame,
4143        hello_authority: Option<&HelloAuthority>,
4144        differing_declarations: &mut ProviderDeclarations,
4145        #[cfg(windows)] launch_nonce: Option<&str>,
4146    ) -> Result<Vec<Frame>, RouterError> {
4147        let module_id = manifest.module_id.clone();
4148        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
4149            manifest,
4150            negotiated_ver,
4151            connection_id,
4152            control_ops,
4153        ) {
4154            Ok(registration) => registration,
4155            Err(RegistryError::DuplicateModuleId { module_id }) => {
4156                return Ok(vec![control_error_frame(
4157                    frame,
4158                    "duplicate_module_id",
4159                    format!(
4160                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
4161                    ),
4162                )?])
4163            }
4164            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
4165                return Ok(vec![control_error_frame(
4166                    frame,
4167                    "invalid_module_id",
4168                    err.to_string(),
4169                )?])
4170            }
4171            Err(err) => {
4172                return Ok(vec![control_error_frame(
4173                    frame,
4174                    "registry_error",
4175                    err.to_string(),
4176                )?])
4177            }
4178        };
4179        *differing_declarations = match self.admit_hello_authority(connection_id, hello_authority) {
4180            Ok(differing) => differing,
4181            Err(err) => {
4182                if matches!(
4183                    self.rollback_unpublished_hello(connection_id),
4184                    Ok(r) if !r.is_empty()
4185                ) {
4186                    crate::supervise::notify_registration_release();
4187                }
4188                return Err(RouterError::Forwarding(err));
4189            }
4190        };
4191        #[cfg(test)]
4192        self.hold_between_registry_and_forwarding();
4193        let mut reply = Vec::new();
4194        let publication = self
4195            .registry
4196            .publish_registration_after(connection_id, || {
4197                if let Some(sink) = sink {
4198                    // Same ordering as an ordinary HELLO: queue the ACK before
4199                    // inserting the endpoint, and keep registry readers excluded
4200                    // until the candidate is published.
4201                    let concurrency = manifest_concurrency(&registration.manifest);
4202                    self.forwarding.register_candidate_module_connection_acked(
4203                        connection_id,
4204                        module_id.clone(),
4205                        negotiated_ver,
4206                        concurrency,
4207                        sink,
4208                        hello_ack,
4209                    )?;
4210                } else {
4211                    reply.push(hello_ack);
4212                }
4213                #[cfg(test)]
4214                self.hold_between_ack_and_publication();
4215                Ok(())
4216            })
4217            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4218        let published = match publication {
4219            Ok(published) => published,
4220            Err(err) => {
4221                if matches!(
4222                    self.rollback_unpublished_hello(connection_id),
4223                    Ok(r) if !r.is_empty()
4224                ) {
4225                    crate::supervise::notify_registration_release();
4226                }
4227                self.refuse_hello_authority(connection_id, hello_authority);
4228                return Ok(vec![control_error_frame(
4229                    frame,
4230                    forwarding_error_code(&err),
4231                    err.to_string(),
4232                )?]);
4233            }
4234        };
4235        if !published {
4236            debug!(
4237                module_id = %module_id,
4238                connection_id = connection_id.get(),
4239                "swap candidate registration was removed before it could be published"
4240            );
4241            self.refuse_hello_authority(connection_id, hello_authority);
4242            return Ok(reply);
4243        }
4244        info!(
4245            module_id = %module_id,
4246            module_version = %registration.manifest.module_version,
4247            negotiated_ver,
4248            ready = registration.ready,
4249            connection_id = connection_id.get(),
4250            "swap candidate registered; not routable until cutover"
4251        );
4252        #[cfg(windows)]
4253        if let Some(nonce) = launch_nonce {
4254            self.supervisor.program_hello_accepted(&module_id, nonce);
4255        }
4256        Ok(reply)
4257    }
4258
4259    /// Bind `connection` to the spawn its HELLO registers as and freeze the
4260    /// spawn's requested provider declarations. Runs only once the registry
4261    /// has accepted the HELLO, before its endpoint is published, so a route
4262    /// reserved on the endpoint always finds the spawn. Returns the requested
4263    /// entries that differ from the saved approvals, for a widening request.
4264    fn admit_hello_authority(
4265        &self,
4266        connection: ConnectionId,
4267        authority: Option<&HelloAuthority>,
4268    ) -> Result<ProviderDeclarations, ForwardingError> {
4269        let Some(authority) = authority else {
4270            return Ok(ProviderDeclarations::new());
4271        };
4272        let froze = authority
4273            .record
4274            .freeze_declarations(authority.requested.clone());
4275        authority
4276            .froze
4277            .store(froze.is_some(), std::sync::atomic::Ordering::Relaxed);
4278        self.forwarding
4279            .bind_target_authority(connection, authority.record.clone())
4280            .inspect_err(|_| self.refuse_hello_authority(connection, Some(authority)))?;
4281        Ok(froze.unwrap_or_default())
4282    }
4283
4284    /// Undo [`Self::admit_hello_authority`] for a HELLO refused after the
4285    /// registry accepted it, so the refused connection is no registration of
4286    /// the spawn and the spawn's declarations are left for the HELLO that
4287    /// does register it to freeze.
4288    fn refuse_hello_authority(&self, connection: ConnectionId, authority: Option<&HelloAuthority>) {
4289        let Some(authority) = authority else {
4290            return;
4291        };
4292        if let Err(err) = self.forwarding.unbind_target_authority(connection) {
4293            warn!(error = %err, "failed to unbind a refused HELLO from its spawn");
4294        }
4295        if authority
4296            .froze
4297            .swap(false, std::sync::atomic::Ordering::Relaxed)
4298        {
4299            authority.record.unfreeze_refused_declarations();
4300        }
4301    }
4302
4303    fn build_hello_ack(
4304        &self,
4305        frame: &Frame,
4306        negotiated_ver: u8,
4307        module_id: &str,
4308    ) -> Result<Frame, RouterError> {
4309        let ack = ModuleHelloAckBody {
4310            negotiated_ver,
4311            subc_ops: module_subc_ops(),
4312            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
4313            storage: self
4314                .storage_config
4315                .as_ref()
4316                .map(|cfg| cfg.descriptor_for(module_id)),
4317            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4318        };
4319        let body = serde_json::to_vec(&ack).map_err(|err| {
4320            RouterError::backend(
4321                0,
4322                frame.header.corr,
4323                format!("failed to encode HELLO_ACK: {err}"),
4324            )
4325        })?;
4326
4327        Frame::build_with_version(
4328            negotiated_ver,
4329            FrameType::HelloAck,
4330            control_flags(),
4331            0,
4332            0,
4333            frame.header.corr,
4334            body,
4335        )
4336        .map_err(RouterError::FrameBuild)
4337    }
4338
4339    async fn handle_client_control_request(
4340        &self,
4341        ctx: &RouteCtx,
4342        frame: Frame,
4343        request: ClientControlRequest,
4344        route_open_timings: &mut Option<RouteOpenTimings>,
4345    ) -> Result<Vec<Frame>, RouterError> {
4346        if let Some(owner) = &self.config_runtime {
4347            let module = match &request {
4348                ClientControlRequest::SupervisorRestart { module_id, .. }
4349                | ClientControlRequest::SupervisorReload { module_id }
4350                | ClientControlRequest::SupervisorSwap { module_id, .. }
4351                | ClientControlRequest::SupervisorSetEnabled {
4352                    module_id,
4353                    enabled: true,
4354                } => Some(module_id.as_str()),
4355                _ => None,
4356            };
4357            let rescan = matches!(
4358                &request,
4359                ClientControlRequest::SupervisorRescan { preview: false }
4360                    | ClientControlRequest::SupervisorRescanChecked { preview: false, .. }
4361            );
4362            let held = if module.is_some() || rescan {
4363                let module = module.map(str::to_owned);
4364                owner
4365                    .on_blocking(move |owner| owner.checked_admission_hold(module.as_deref()))
4366                    .await
4367            } else {
4368                Ok(())
4369            };
4370            if let Err(error) = held {
4371                if error.code != "config_delivery_deferred" {
4372                    let code = if error.code == "store_tampered" {
4373                        "store_tampered"
4374                    } else {
4375                        "config_gated"
4376                    };
4377                    return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
4378                }
4379            }
4380        }
4381        if let Some(operation) = privileged_operation(&request) {
4382            let caller = self.resolve_caller(ctx.connection_id).await;
4383            let op = client_control_request_op(&request);
4384            let text = prompt_text(operation, &caller);
4385            match self.privileged_operator_prompt {
4386                PrivilegedOperatorPrompt::Log => {
4387                    info!(target: "subc_daemon::privileged_prompt", operation = op,
4388                        target = ?operation, caller = %caller.description(), summary = %text,
4389                        "privileged request would prompt the operator");
4390                }
4391                PrivilegedOperatorPrompt::Enforce => {
4392                    let key =
4393                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
4394                    let stamp = self
4395                        .operator_apps
4396                        .lock()
4397                        .unwrap_or_else(|p| p.into_inner())
4398                        .get(&ctx.connection_id)
4399                        .cloned();
4400                    let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
4401                        &self.operator_app_policy,
4402                        &self.launch_table,
4403                        stamp.as_ref(),
4404                    )
4405                    .then(|| self.shortcut_context(stamp, None, None));
4406                    let outcome = self
4407                        .forwarding
4408                        .operator_confirms()
4409                        .approve_privileged_with_shortcut(
4410                            ctx,
4411                            &frame,
4412                            op,
4413                            text,
4414                            caller.description(),
4415                            key,
4416                            shortcut,
4417                        )
4418                        .await;
4419                    if outcome.code != "confirmed" {
4420                        return Ok(vec![outcome.frame(&frame)]);
4421                    }
4422                }
4423            }
4424        }
4425        match request {
4426            ClientControlRequest::DaemonShutdownIntent { intent } => {
4427                if intent.as_str() != Some("restart") {
4428                    return Ok(vec![control_error_frame(
4429                        &frame,
4430                        "invalid_params",
4431                        "intent must be 'restart'",
4432                    )?]);
4433                }
4434                // A stamped daemon boots a newer approved generation only from a
4435                // durable restart intent, written here before the daemon exits.
4436                // Failing to write it never blocks the restart: the next start
4437                // then boots the last active generation and the newer one waits.
4438                if let Some(owner) = &self.config_runtime {
4439                    match owner
4440                        .on_blocking(|owner| owner.assign_daemon_restart())
4441                        .await
4442                    {
4443                        Ok(Some(generation)) => {
4444                            info!(
4445                                generation,
4446                                "daemon restart will warm up approved generation"
4447                            )
4448                        }
4449                        Ok(None) => {}
4450                        Err(error) => warn!(
4451                            %error,
4452                            "daemon restart intent not recorded; the next start boots the last active generation"
4453                        ),
4454                    }
4455                }
4456                let response = ClientControlResponse::DaemonShutdownIntent {
4457                    intent: "restart".to_owned(),
4458                    expires_at_ms: self.supervisor.announce_daemon_restart(),
4459                };
4460                Ok(vec![control_response_body_frame(
4461                    &frame,
4462                    &response,
4463                    "ClientControlResponse::DaemonShutdownIntent",
4464                )?])
4465            }
4466            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
4467            ClientControlRequest::CatalogList { module_id } => {
4468                self.handle_catalog_list(frame, module_id)
4469            }
4470            ClientControlRequest::RouteOpen {
4471                target,
4472                identity,
4473                consumer_identity,
4474                consumer_capabilities,
4475                role_versions,
4476                busy_refusals,
4477                admission_facts,
4478                scope,
4479                ..
4480            } => {
4481                let extensions = match serde_json::from_slice::<subc_control::RouteOpenRequestExt>(
4482                    &frame.body,
4483                ) {
4484                    Ok(extensions) => extensions,
4485                    Err(err) => {
4486                        return Ok(vec![control_error_frame(
4487                            &frame,
4488                            "invalid_control_body",
4489                            format!("malformed route.open extensions: {err}"),
4490                        )?]);
4491                    }
4492                };
4493                self.handle_route_open(
4494                    ctx,
4495                    frame,
4496                    RouteOpenRequest {
4497                        target,
4498                        identity,
4499                        consumer_identity,
4500                        consumer_capabilities,
4501                        role_versions,
4502                        busy_refusals,
4503                        admission_facts,
4504                        scope,
4505                        accepts_close_reasons: extensions.accepts_close_reasons,
4506                    },
4507                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
4508                )
4509                .await
4510            }
4511            ClientControlRequest::RoutePoll {
4512                route_channel,
4513                route_epoch,
4514                kind,
4515            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
4516            ClientControlRequest::SupervisorList {} => {
4517                let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
4518                    self.agent_declarations.refuse(
4519                        self,
4520                        ctx,
4521                        control_error_frame(
4522                            &frame,
4523                            "busy",
4524                            "supervisor list work is at capacity; retry",
4525                        )?,
4526                    );
4527                    return Ok(Vec::new());
4528                };
4529                // Filesystem-backed diagnostics must not hold up this connection's reader.
4530                let handler = self.clone();
4531                let sink = ctx.egress.clone();
4532                tokio::spawn(
4533                    async move {
4534                        let _permit = permit;
4535                        let result = handler.handle_supervisor_list(frame.clone()).await;
4536                        let replies = match result {
4537                            Ok(replies) => replies,
4538                            Err(error) => {
4539                                control_error_frame(&frame, "backend_error", error.to_string())
4540                                    .into_iter()
4541                                    .collect()
4542                            }
4543                        };
4544                        for reply in replies {
4545                            if !matches!(
4546                                tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
4547                                    .await,
4548                                Ok(Ok(()))
4549                            ) {
4550                                break;
4551                            }
4552                        }
4553                    }
4554                    .with_current_subscriber(),
4555                );
4556                Ok(Vec::new())
4557            }
4558            ClientControlRequest::SupervisorSpawnSnapshot {} => {
4559                self.handle_supervisor_spawn_snapshot(frame)
4560            }
4561            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
4562                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
4563            }
4564            ClientControlRequest::SupervisorRestart {
4565                module_id,
4566                drain_timeout_ms,
4567            } => {
4568                self.log_supervisor_request_received(
4569                    ctx,
4570                    frame.header.corr,
4571                    ops::SUPERVISOR_RESTART,
4572                    Some(&module_id),
4573                    None,
4574                )?;
4575                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4576                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
4577                    .await
4578            }
4579            ClientControlRequest::SupervisorSwap {
4580                module_id,
4581                ready_timeout_ms,
4582            } => {
4583                self.log_supervisor_request_received(
4584                    ctx,
4585                    frame.header.corr,
4586                    ops::SUPERVISOR_SWAP,
4587                    Some(&module_id),
4588                    None,
4589                )?;
4590                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4591                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
4592                    .await
4593            }
4594            ClientControlRequest::SupervisorReload { module_id } => {
4595                self.log_supervisor_request_received(
4596                    ctx,
4597                    frame.header.corr,
4598                    ops::SUPERVISOR_RELOAD,
4599                    Some(&module_id),
4600                    None,
4601                )?;
4602                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4603                self.handle_supervisor_reload(frame, module_id, requester)
4604                    .await
4605            }
4606            ClientControlRequest::SupervisorRescan { preview } => {
4607                if !preview {
4608                    self.log_supervisor_request_received(
4609                        ctx,
4610                        frame.header.corr,
4611                        ops::SUPERVISOR_RESCAN,
4612                        None,
4613                        None,
4614                    )?;
4615                }
4616                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4617                let operation = self.handle_supervisor_rescan(
4618                    frame,
4619                    if preview {
4620                        RescanMode::Preview
4621                    } else {
4622                        RescanMode::Execute
4623                    },
4624                    requester,
4625                );
4626                self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
4627                    .await
4628            }
4629            ClientControlRequest::SupervisorRescanChecked {
4630                preview,
4631                candidate,
4632                expect,
4633            } => {
4634                self.observe_control_wait(
4635                    ctx,
4636                    None,
4637                    ops::SUPERVISOR_RESCAN,
4638                    self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
4639                )
4640                .await
4641            }
4642            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
4643                self.log_supervisor_request_received(
4644                    ctx,
4645                    frame.header.corr,
4646                    ops::SUPERVISOR_RELEASE_RESERVED,
4647                    Some(&module_id),
4648                    None,
4649                )?;
4650                self.handle_supervisor_release_reserved(frame, module_id)
4651                    .await
4652            }
4653            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
4654                self.log_supervisor_request_received(
4655                    ctx,
4656                    frame.header.corr,
4657                    ops::SUPERVISOR_SET_ENABLED,
4658                    Some(&module_id),
4659                    Some(enabled),
4660                )?;
4661                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4662                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
4663                    .await
4664            }
4665            ClientControlRequest::SupervisorHealthProbe { module_id } => {
4666                self.handle_supervisor_health_probe(ctx, frame, module_id)
4667                    .await
4668            }
4669            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
4670            ClientControlRequest::SupervisorRoutes { module_id } => {
4671                self.handle_supervisor_routes(frame, module_id)
4672            }
4673            ClientControlRequest::SupervisorProvenance { module_id } => {
4674                self.handle_supervisor_provenance(ctx, frame, module_id)
4675                    .await
4676            }
4677            ClientControlRequest::SupervisorStderrTail {
4678                module_id,
4679                max_lines,
4680                max_bytes,
4681            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
4682            ClientControlRequest::SupervisorTerminals { module_id } => {
4683                self.handle_supervisor_terminals(ctx, frame, module_id)
4684                    .await
4685            }
4686        }
4687    }
4688
4689    fn log_supervisor_request_received(
4690        &self,
4691        ctx: &RouteCtx,
4692        corr: u64,
4693        op: &'static str,
4694        module_id: Option<&str>,
4695        enabled: Option<bool>,
4696    ) -> Result<(), RouterError> {
4697        let caller = self.supervisor_requester(ctx, corr)?;
4698        let caller = principal_label(&caller);
4699        // `Option` fields are recorded only when present, so a request with no
4700        // module id (rescan) or no enabled flag simply omits that field.
4701        info!(
4702            target: "control",
4703            op,
4704            module_id,
4705            enabled,
4706            connection_id = ctx.connection_id.get(),
4707            caller = %caller,
4708            "supervisor request received"
4709        );
4710        Ok(())
4711    }
4712
4713    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
4714        Ok(self
4715            .registry
4716            .get_module_by_connection(ctx.connection_id)
4717            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4718            .map(|registration| Principal::Reserved {
4719                module_id: registration.manifest.module_id,
4720            })
4721            .unwrap_or(Principal::Direct))
4722    }
4723
4724    fn handle_module_control_request(
4725        &self,
4726        connection_id: ConnectionId,
4727        frame: Frame,
4728        request: ModuleControlRequestFromModule,
4729    ) -> Result<Vec<Frame>, RouterError> {
4730        match request {
4731            ModuleControlRequestFromModule::RouteAuthorize {
4732                principal,
4733                scope,
4734                target,
4735                expected,
4736            } => self.handle_route_authorize(
4737                connection_id,
4738                frame,
4739                principal,
4740                scope,
4741                target,
4742                expected,
4743            ),
4744            ModuleControlRequestFromModule::CatalogUpdate {
4745                provides,
4746                capabilities,
4747                ready,
4748            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
4749            ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { role_declarations } => {
4750                self.handle_catalog_update_role_declarations(
4751                    connection_id,
4752                    frame,
4753                    role_declarations,
4754                )
4755            }
4756            ModuleControlRequestFromModule::LiveRoots {} => {
4757                let registered = self
4758                    .registry
4759                    .get_module_by_connection(connection_id)
4760                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4761                let Some(registration) = registered else {
4762                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
4763                };
4764                let response = self
4765                    .forwarding
4766                    .live_roots(&registration.manifest.module_id)
4767                    .map_err(RouterError::Forwarding)?;
4768                Ok(vec![control_response_body_frame(
4769                    &frame,
4770                    &response,
4771                    "ModuleControlResponseToModule::LiveRoots",
4772                )?])
4773            }
4774            ModuleControlRequestFromModule::AgentCommands {
4775                started,
4776                finished,
4777                known,
4778            } => self.handle_agent_commands(connection_id, frame, started, finished, known),
4779            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
4780                self.handle_scope_sync(connection_id, frame, generation, scopes)
4781            }
4782            ModuleControlRequestFromModule::ScopeApply {
4783                generation,
4784                upsert,
4785                end,
4786            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
4787            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
4788                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
4789            }
4790            _ => Ok(vec![control_error_frame(
4791                &frame,
4792                "unsupported_control_frame",
4793                "unsupported module control operation",
4794            )?]),
4795        }
4796    }
4797
4798    fn handle_scope_served_by(
4799        &self,
4800        connection_id: ConnectionId,
4801        frame: Frame,
4802    ) -> Result<Vec<Frame>, RouterError> {
4803        let registered = self
4804            .registry
4805            .get_module_by_connection(connection_id)
4806            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4807        let Some(registered) = registered else {
4808            return Ok(vec![control_error_frame(
4809                &frame,
4810                "not_registered",
4811                "scope.served_by requires an active module registration owned by this connection",
4812            )?]);
4813        };
4814        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
4815        {
4816            Ok(request) => request,
4817            Err(err) => {
4818                return Ok(vec![control_error_frame(
4819                    &frame,
4820                    "invalid_control_body",
4821                    format!("malformed scope.served_by body: {err}"),
4822                )?])
4823            }
4824        };
4825        let answer = self
4826            .scopes
4827            .read()
4828            .unwrap_or_else(|poisoned| poisoned.into_inner())
4829            .served_by(
4830                &registered.manifest.module_id,
4831                &request.scope_ref,
4832                request.scope_epoch,
4833                &request.module_id,
4834            );
4835        let reply = subc_control::ScopeServedByReply::new(
4836            answer,
4837            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
4838        );
4839        Ok(vec![control_response_body_frame(
4840            &frame,
4841            &reply,
4842            "ScopeServedByReply",
4843        )?])
4844    }
4845
4846    fn handle_operator_confirm(
4847        &self,
4848        ctx: &RouteCtx,
4849        frame: Frame,
4850    ) -> Result<Vec<Frame>, RouterError> {
4851        use crate::operator_confirm::{audit, Outcome};
4852        let registration = self
4853            .registry
4854            .get_module_by_connection(ctx.connection_id)
4855            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4856        let Some(registration) = registration else {
4857            let outcome = Outcome::refusal("not_registered");
4858            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4859            return Ok(vec![outcome.frame(&frame)]);
4860        };
4861        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
4862            Ok(request) => request,
4863            Err(_) => {
4864                let outcome = Outcome::refusal("invalid_control_body");
4865                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4866                return Ok(vec![outcome.frame(&frame)]);
4867            }
4868        };
4869        let declared = matches!(
4870            request.kind,
4871            Some(subc_protocol::session::OperatorConfirmKind::Operation)
4872        ) && request.operation.as_ref().is_some_and(|operation| {
4873            registration
4874                .manifest
4875                .confirm_operations
4876                .as_ref()
4877                .is_some_and(|operations| operations.contains(operation))
4878        });
4879        let module_id = registration.manifest.module_id;
4880        // Read the launch nonce (under its own lock) before taking the forwarding
4881        // table's lock below: holding forwarding while waiting on another daemon
4882        // lock risks a lock-order deadlock with paths that take them the other way.
4883        let nonce = self
4884            .hello_launch_nonces
4885            .lock()
4886            .unwrap_or_else(|p| p.into_inner())
4887            .nonce(ctx.connection_id)
4888            .map(str::to_owned);
4889        // A registration channel proves its own spawn; only a connection that
4890        // registered over the launch nonce falls back to the nonce.
4891        let nonce_proven = self
4892            .confirm_channel_proven(ctx.connection_id, &module_id)
4893            .unwrap_or_else(|| {
4894                nonce.as_deref().is_some_and(|nonce| {
4895                    self.supervisor
4896                        .spawned_consumer_authorized(&module_id, nonce)
4897                })
4898            });
4899        let confirms = self.forwarding.operator_confirms();
4900        self.forwarding
4901            .with_operator_route(
4902                ctx.connection_id,
4903                request.route_channel,
4904                request.route_epoch,
4905                |binding| {
4906                    let shortcut = binding
4907                        .filter(|binding| {
4908                            nonce_proven
4909                                && crate::operator_confirm::valid_summary(&request.summary)
4910                                && declared
4911                                && crate::operator_confirm::ShortcutContext::route_eligible(
4912                                    &self.operator_app_policy,
4913                                    &self.launch_table,
4914                                    binding,
4915                                )
4916                        })
4917                        .map(|binding| {
4918                            let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
4919                                &self.operator_app_policy,
4920                                binding,
4921                            );
4922                            let stamp = if phone.is_some() {
4923                                None
4924                            } else {
4925                                binding.operator_app.clone()
4926                            };
4927                            self.shortcut_context(
4928                                stamp,
4929                                phone,
4930                                Some((request.route_channel, request.route_epoch)),
4931                            )
4932                        });
4933                    confirms.admit_with_shortcut(
4934                        ctx,
4935                        frame,
4936                        module_id,
4937                        nonce_proven,
4938                        request,
4939                        binding,
4940                        shortcut,
4941                    )
4942                },
4943            )
4944            .map_err(RouterError::Forwarding)
4945    }
4946
4947    fn shortcut_context(
4948        &self,
4949        stamp: Option<crate::operator_apps::OperatorAppStamp>,
4950        phone_fingerprint: Option<String>,
4951        route: Option<(u16, u32)>,
4952    ) -> crate::operator_confirm::ShortcutContext {
4953        crate::operator_confirm::ShortcutContext {
4954            ledger: self.activity.clone(),
4955            forwarding: self.forwarding.clone(),
4956            launches: self.launch_table.clone(),
4957            stamp,
4958            phone_fingerprint,
4959            route,
4960            connections: self.operator_apps.clone(),
4961            #[cfg(test)]
4962            hooks: Default::default(),
4963        }
4964    }
4965
4966    /// Explicit test-only phone policy; daemon configuration cannot select it.
4967    #[cfg(any(test, feature = "test-support"))]
4968    pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
4969        self.operator_app_policy.phone_enabled = enabled;
4970        self
4971    }
4972
4973    /// Resolve folders before taking the scope-table lock: a slow filesystem
4974    /// must not hold up admissions for unrelated scoped routes.
4975    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
4976        let mut scopes = ScopeRecords::from(scopes);
4977        for (index, record) in scopes.records.iter_mut().enumerate() {
4978            if let Some(refusal) = Self::canonicalize_workdir(record) {
4979                scopes.refusals.insert(index, refusal);
4980            }
4981        }
4982        scopes
4983    }
4984
4985    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
4986        let workdir = record.workdir.as_mut()?;
4987        let path = std::path::Path::new(workdir);
4988        if !path.is_absolute() {
4989            return Some((
4990                error_codes::INVALID_WORKDIR,
4991                "workdir must be absolute".to_string(),
4992            ));
4993        }
4994        match ProjectRootId::from_path_allowing_missing(path) {
4995            Ok(root) => match root.as_path().to_str() {
4996                Some(canonical) => {
4997                    *workdir = canonical.to_string();
4998                    None
4999                }
5000                None => Some((
5001                    error_codes::INVALID_WORKDIR,
5002                    "canonical workdir is not UTF-8".to_string(),
5003                )),
5004            },
5005            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
5006        }
5007    }
5008
5009    /// `scope.sync`: the owner is the module registered on this connection.
5010    /// A connection with no registration (every client connection, `direct`
5011    /// included) is refused `not_registered` before the table is consulted.
5012    fn handle_scope_sync(
5013        &self,
5014        connection_id: ConnectionId,
5015        frame: Frame,
5016        generation: u64,
5017        scopes: Vec<ScopeRecord>,
5018    ) -> Result<Vec<Frame>, RouterError> {
5019        self.handle_scope_change(connection_id, frame, generation, scopes, None)
5020    }
5021
5022    fn handle_scope_change(
5023        &self,
5024        connection_id: ConnectionId,
5025        frame: Frame,
5026        generation: u64,
5027        scopes: Vec<ScopeRecord>,
5028        end: Option<Vec<ScopeEnd>>,
5029    ) -> Result<Vec<Frame>, RouterError> {
5030        let is_apply = end.is_some();
5031        let Some(registration) = self
5032            .registry
5033            .get_module_by_connection(connection_id)
5034            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5035        else {
5036            return Ok(vec![control_error_frame(
5037                &frame,
5038                "not_registered",
5039                "scope.sync and scope.apply require an active module registration owned by this connection",
5040            )?]);
5041        };
5042        let owner = registration.manifest.module_id;
5043        let scopes = Self::prepare_scope_records(scopes);
5044        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
5045        let is_current_launch = |connection: ConnectionId| {
5046            self.connection_is_current_launch(&owner, connection, current_nonce.as_deref())
5047        };
5048        // Lock order is the scope table, then the forwarding table: the new
5049        // tags are published, and the routes the change closes are selected,
5050        // while the scope table is still write-locked, so no admission can read
5051        // a record whose tag is not yet published.
5052        let mut table = self
5053            .scopes
5054            .write()
5055            .unwrap_or_else(|poisoned| poisoned.into_inner());
5056        let outcome = match end {
5057            Some(end) => table.apply(
5058                &owner,
5059                connection_id,
5060                is_current_launch,
5061                generation,
5062                scopes,
5063                end,
5064            ),
5065            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
5066        };
5067        let drained = match &outcome {
5068            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
5069            Err(_) => Vec::new(),
5070        };
5071        let has_deadlines = outcome.is_ok() && table.has_deadlines();
5072        drop(table);
5073        if has_deadlines {
5074            self.scope_deadline_added.notify_one();
5075        }
5076        match outcome {
5077            Ok(applied) => {
5078                let counts = ScopeOutcomeCounts::of(&applied.results);
5079                info!(
5080                    owner = %owner,
5081                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5082                    generation,
5083                    records = applied.results.len(),
5084                    created = counts.created,
5085                    replaced = counts.replaced,
5086                    updated = counts.updated,
5087                    unchanged = counts.unchanged,
5088                    refused = counts.refused,
5089                    ended = applied.ended.len(),
5090                    tag_changes = applied.tag_changes.len(),
5091                    routes_closed = drained.len(),
5092                    "scope change accepted"
5093                );
5094                // An accepted scope change can still refuse individual records, and the
5095                // owner is the only party that sees the reply. Name them here so
5096                // an operator can tell a refused session from a missing one
5097                // without the owner's logs. Capped so a call that refuses
5098                // thousands cannot flood the log; the count above is complete.
5099                for refused in applied
5100                    .results
5101                    .iter()
5102                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
5103                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
5104                {
5105                    warn!(
5106                        owner = %owner,
5107                        generation,
5108                        scope_ref = %refused.scope_ref,
5109                        scope_epoch = refused.scope_epoch,
5110                        code = refused.code.as_deref().unwrap_or(""),
5111                        "scope record refused"
5112                    );
5113                }
5114                self.close_scope_drained_routes(drained);
5115                let response = if is_apply {
5116                    ModuleControlResponseToModule::ScopeApply {
5117                        generation,
5118                        results: applied.results,
5119                        end_results: applied.end_results,
5120                        ended: applied.ended,
5121                    }
5122                } else {
5123                    ModuleControlResponseToModule::ScopeSync {
5124                        generation,
5125                        results: applied.results,
5126                        ended: applied.ended,
5127                    }
5128                };
5129                Ok(vec![control_response_body_frame(
5130                    &frame,
5131                    &response,
5132                    "scope change response",
5133                )?])
5134            }
5135            Err(refusal) => {
5136                info!(
5137                    owner = %owner,
5138                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5139                    generation,
5140                    code = refusal.code,
5141                    "scope change refused"
5142                );
5143                Ok(vec![control_error_frame(
5144                    &frame,
5145                    refusal.code,
5146                    refusal.message,
5147                )?])
5148            }
5149        }
5150    }
5151
5152    /// Notify both endpoints of routes drained by a scope change.
5153    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
5154        scope_expiry::ScopeExpiryContext::new(self).close_scope_drained_routes(drained);
5155    }
5156
5157    /// Re-authorize a stored source on a principal the module supplies.
5158    ///
5159    /// That principal carries no spawn or authority epoch, so this reply never
5160    /// conveys a caller attestation or a first-party mark, and a provider must
5161    /// not apply automatic grants on anything it authorizes.
5162    fn handle_route_authorize(
5163        &self,
5164        connection_id: ConnectionId,
5165        frame: Frame,
5166        principal: Principal,
5167        scope: subc_protocol::scope::RouteAuthorizeScope,
5168        target: String,
5169        expected: subc_protocol::scope::RouteAuthorizeExpected,
5170    ) -> Result<Vec<Frame>, RouterError> {
5171        use crate::scopes::admission::{admit_route, AdmissionMode};
5172        use subc_protocol::scope::{
5173            RouteAuthorizeOutcome, RouteAuthorizeResponse, ROUTE_AUTHORIZE_CAPABILITY,
5174        };
5175        let registry = self
5176            .registry
5177            .route_admission_snapshot()
5178            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5179        let owner_configured = match &scope.owner {
5180            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5181            _ => false,
5182        };
5183        let selector = ScopeSelector::new(scope.owner, scope.scope_ref, Some(scope.epoch));
5184        let answer = {
5185            let scopes = self.scopes.read().unwrap_or_else(|p| p.into_inner());
5186            let permitted = registry
5187                .iter()
5188                .find(|r| r.connection_id == connection_id)
5189                .is_some_and(|r| {
5190                    scopes.is_federated_owner(&Principal::Reserved {
5191                        module_id: r.manifest.module_id.clone(),
5192                    }) && r.manifest.capabilities.as_ref().is_some_and(|caps| {
5193                        caps.provides
5194                            .iter()
5195                            .any(|cap| cap == ROUTE_AUTHORIZE_CAPABILITY)
5196                    })
5197                });
5198            if !permitted {
5199                drop(scopes);
5200                return Ok(vec![control_error_frame(&frame, error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED,
5201                    "route.authorize requires the configured federated owner's route-authorize/v1 registration")?]);
5202            }
5203            admit_route(
5204                &scopes,
5205                &registry,
5206                &principal,
5207                &target,
5208                Some(&selector),
5209                owner_configured,
5210                AdmissionMode::Authorize(&expected),
5211            )
5212        };
5213        let response = match answer {
5214            Ok(admission) => {
5215                let mut response = RouteAuthorizeResponse::new(
5216                    RouteAuthorizeOutcome::Authorized,
5217                    "route admission authorized",
5218                );
5219                response.content_version = admission.content_version;
5220                response
5221            }
5222            Err(refusal) => refusal.response(),
5223        };
5224        Ok(vec![control_response_body_frame(
5225            &frame,
5226            &response,
5227            "RouteAuthorizeResponse",
5228        )?])
5229    }
5230
5231    /// `scope.describe`: any registered module may read any scope, because a
5232    /// provider must read the scope a route it serves is stamped with.
5233    fn handle_scope_describe(
5234        &self,
5235        connection_id: ConnectionId,
5236        frame: Frame,
5237        owner: Principal,
5238        scope_ref: String,
5239    ) -> Result<Vec<Frame>, RouterError> {
5240        let registered = self
5241            .registry
5242            .get_module_by_connection(connection_id)
5243            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5244        let Some(registered) = registered else {
5245            return Ok(vec![control_error_frame(
5246                &frame,
5247                "not_registered",
5248                "scope.describe requires an active module registration owned by this connection",
5249            )?]);
5250        };
5251        let scopes = self
5252            .scopes
5253            .read()
5254            .unwrap_or_else(|poisoned| poisoned.into_inner());
5255        if scopes.is_federated_owner(&owner)
5256            && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
5257        {
5258            return Ok(vec![control_error_frame(
5259                &frame,
5260                error_codes::SCOPE_FEDERATED_UNSUPPORTED,
5261                "scope.describe of the federated owner requires scope-federated/v1",
5262            )?]);
5263        }
5264        let mut description = scopes.describe(&owner, &scope_ref);
5265        drop(scopes);
5266        if owner
5267            != (Principal::Reserved {
5268                module_id: registered.manifest.module_id.clone(),
5269            })
5270        {
5271            if let Some(facts) = description
5272                .stamp
5273                .as_mut()
5274                .and_then(|stamp| stamp.attributes.federated.as_mut())
5275            {
5276                facts.stamp_required = false;
5277            }
5278        }
5279        // Describe readers also decode strict ScopeAttributes. Attribution is
5280        // safe to omit unless the requesting module declares it understands it.
5281        if !registered
5282            .manifest
5283            .capabilities
5284            .as_ref()
5285            .is_some_and(|caps| {
5286                caps.provides
5287                    .iter()
5288                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5289            })
5290        {
5291            if let Some(stamp) = description.stamp.as_mut() {
5292                stamp.attributes.on_behalf_of_agent = None;
5293            }
5294        }
5295        if !registered
5296            .manifest
5297            .capabilities
5298            .as_ref()
5299            .is_some_and(|caps| {
5300                caps.provides
5301                    .iter()
5302                    .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5303            })
5304        {
5305            if let Some(stamp) = description.stamp.as_mut() {
5306                stamp.attributes.flow_version = None;
5307                stamp.attributes.flow_code_hash = None;
5308                stamp.attributes.flow_install_id = None;
5309            }
5310        }
5311        let owner_configured = match &owner {
5312            // Ask whether the owner is configured (`is_configured`), not
5313            // whether it is on the roster (`get(..).is_some()`): a supervised
5314            // module's process can register and describe a scope before the
5315            // supervisor has put it on the roster.
5316            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5317            _ => false,
5318        };
5319        let response = ModuleControlResponseToModule::ScopeDescribe {
5320            status: description.status,
5321            scope_epoch: description.scope_epoch,
5322            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
5323            owner_synced: description.owner_synced,
5324            owner_configured,
5325            scope: description.stamp,
5326        };
5327        Ok(vec![control_response_body_frame(
5328            &frame,
5329            &response,
5330            "ModuleControlResponseToModule::ScopeDescribe",
5331        )?])
5332    }
5333
5334    fn handle_catalog_update_role_declarations(
5335        &self,
5336        connection_id: ConnectionId,
5337        frame: Frame,
5338        role_declarations: serde_json::Value,
5339    ) -> Result<Vec<Frame>, RouterError> {
5340        if let Err(err) = validate_role_declarations_value(Some(&role_declarations)) {
5341            return Ok(vec![control_error_frame(
5342                &frame,
5343                "invalid_role_declarations",
5344                err.to_string(),
5345            )?]);
5346        }
5347        let declarations = serde_json::from_value(role_declarations)
5348            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5349        if self
5350            .registry
5351            .replace_role_declarations_for_connection(connection_id, declarations)
5352            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5353            .is_none()
5354        {
5355            return Ok(vec![control_error_frame(
5356                &frame,
5357                "not_registered",
5358                "catalog.update_role_declarations requires an active module registration owned by this connection",
5359            )?]);
5360        }
5361        control_response_body_frame(
5362            &frame,
5363            &ModuleControlResponseToModule::CatalogUpdateRoleDeclarations {},
5364            "ModuleControlResponseToModule::CatalogUpdateRoleDeclarations",
5365        )
5366        .map(|frame| vec![frame])
5367    }
5368
5369    fn handle_catalog_update(
5370        &self,
5371        connection_id: ConnectionId,
5372        frame: Frame,
5373        provides: Vec<ProviderRole>,
5374        capabilities: Option<CapabilityDeclarations>,
5375        ready: Option<bool>,
5376    ) -> Result<Vec<Frame>, RouterError> {
5377        self.refresh_capability_requirements();
5378        let Some(registration) = self
5379            .registry
5380            .get_module_by_connection(connection_id)
5381            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5382        else {
5383            return Ok(vec![control_error_frame(
5384                &frame,
5385                "not_registered",
5386                "catalog.update requires an active module registration owned by this connection",
5387            )?]);
5388        };
5389
5390        if let Some(message) =
5391            catalog_update_frozen_field_message(&registration.manifest, &provides)
5392        {
5393            return Ok(vec![control_error_frame(
5394                &frame,
5395                "catalog_update_frozen_field",
5396                message,
5397            )?]);
5398        }
5399
5400        let mut candidate = registration.manifest.clone();
5401        candidate.provides = provides.clone();
5402        candidate.capabilities = capabilities
5403            .clone()
5404            .or_else(|| registration.manifest.capabilities.clone());
5405        if let Some(rich_ui) = &candidate.rich_ui {
5406            if let Err(err) = rich_ui.validate() {
5407                return Ok(vec![control_error_frame(
5408                    &frame,
5409                    "invalid_manifest",
5410                    err.to_string(),
5411                )?]);
5412            }
5413        }
5414        if let Err(err) = candidate.validate_capability_grammar() {
5415            return Ok(vec![control_error_frame(
5416                &frame,
5417                "invalid_capability_grammar",
5418                err.to_string(),
5419            )?]);
5420        }
5421
5422        // Updates must honor the same reserved owner as initial registration;
5423        // otherwise an empty HELLO could acquire the claim after admission.
5424        let mut conflicts = self
5425            .capability_evaluator
5426            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
5427        if let Some(conflict) = conflicts.first() {
5428            let message = format!(
5429                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
5430                conflict.capability, conflict.claimants[0], candidate.module_id
5431            );
5432            for conflict in &mut conflicts {
5433                conflict.source = DuplicateClaimSource::CatalogUpdate;
5434            }
5435            log_duplicate_claim_events(conflicts);
5436            return Ok(vec![control_error_frame(
5437                &frame,
5438                "reserved_capability",
5439                message,
5440            )?]);
5441        }
5442
5443        let updated = match self.registry.replace_catalog_for_connection(
5444            connection_id,
5445            provides,
5446            capabilities,
5447            ready,
5448        ) {
5449            Err(crate::registry::RegistryError::FederatedCapabilityChanged) => {
5450                return Ok(vec![control_error_frame(
5451                    &frame,
5452                    error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5453                    "scope-federated/v1 is fixed at HELLO; restart the module to change it",
5454                )?])
5455            }
5456            Err(crate::registry::RegistryError::CallerAttestationCapabilityChanged) => {
5457                return Ok(vec![control_error_frame(
5458                    &frame,
5459                    error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5460                    "caller-attestation/v1 is fixed at HELLO; restart the module to change it",
5461                )?])
5462            }
5463            result => {
5464                result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5465            }
5466        };
5467        if updated.is_none() {
5468            return Ok(vec![control_error_frame(
5469                &frame,
5470                "not_registered",
5471                "catalog.update requires an active module registration owned by this connection",
5472            )?]);
5473        }
5474        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
5475            log_duplicate_claim_events(
5476                self.capability_evaluator
5477                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
5478            );
5479        }
5480        if capability_census_trigger(
5481            registration.manifest.capabilities.as_ref(),
5482            updated
5483                .as_ref()
5484                .and_then(|entry| entry.manifest.capabilities.as_ref()),
5485        ) {
5486            self.enforce_capability_denies();
5487        }
5488        self.refresh_capability_requirements();
5489
5490        let response = ModuleControlResponseToModule::CatalogUpdate {};
5491        control_response_body_frame(
5492            &frame,
5493            &response,
5494            "ModuleControlResponseToModule::CatalogUpdate",
5495        )
5496        .map(|frame| vec![frame])
5497    }
5498
5499    pub(crate) fn with_activity_ledger_config(
5500        mut self,
5501        run_dir: &Path,
5502        incarnation: String,
5503        config: crate::activity::ActivityConfig,
5504    ) -> Self {
5505        self.activity = Some(crate::activity::ActivityLedger::open_config(
5506            run_dir.join("activity.jsonl"),
5507            incarnation,
5508            config,
5509        ));
5510        self.supervisor.set_activity(self.activity.clone());
5511        self
5512    }
5513
5514    pub(crate) async fn configure_activity(
5515        &self,
5516        config: crate::activity::ActivityConfig,
5517    ) -> Result<(), &'static str> {
5518        match &self.activity {
5519            Some(ledger) => ledger.configure(config).await,
5520            None => Ok(()),
5521        }
5522    }
5523
5524    fn cancel_activity_subscription(
5525        &self,
5526        ctx: &RouteCtx,
5527        frame: &Frame,
5528    ) -> Result<bool, RouterError> {
5529        let task = self
5530            .activity_streams
5531            .lock()
5532            .unwrap_or_else(|p| p.into_inner())
5533            .remove(&(ctx.connection_id, frame.header.corr));
5534        let Some(task) = task else {
5535            return Ok(false);
5536        };
5537        task.abort();
5538        let end = Frame::build_with_version(
5539            response_version(frame),
5540            FrameType::StreamEnd,
5541            control_flags(),
5542            0,
5543            0,
5544            frame.header.corr,
5545            Vec::new(),
5546        )
5547        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
5548        let sink = ctx.egress.clone();
5549        tokio::spawn(async move {
5550            let _ = sink.send(end).await;
5551        });
5552        Ok(true)
5553    }
5554
5555    async fn handle_activity_read(
5556        &self,
5557        ctx: &RouteCtx,
5558        frame: Frame,
5559    ) -> Result<Vec<Frame>, RouterError> {
5560        #[cfg(test)]
5561        {
5562            let pause = self
5563                .activity_request_pause
5564                .lock()
5565                .unwrap()
5566                .clone()
5567                .filter(|pause| pause.connection == ctx.connection_id);
5568            if let Some(pause) = pause {
5569                let _cancelled = ActivityRequestCancellation(pause.clone());
5570                pause.started.notify_one();
5571                std::future::pending::<()>().await;
5572            }
5573        }
5574        use subc_control::{
5575            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
5576            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
5577        };
5578        #[derive(Deserialize)]
5579        struct Identity {
5580            #[serde(default)]
5581            consumer_identity: Option<ConsumerIdentity>,
5582        }
5583        // A registration channel or claim-bound connection carries its module in
5584        // its binding, so no request field can name another principal, and its
5585        // subscription ends when the bound spawn's authority does.
5586        let mut bound_spawn = None;
5587        let legacy = match self.activity_reader(ctx.connection_id) {
5588            channel_operations::ActivityReader::Module { module_id, spawn } => {
5589                bound_spawn = Some(spawn);
5590                Some(Ok(Principal::Reserved { module_id }))
5591            }
5592            // Refused outright: falling back to `direct` would admit it.
5593            channel_operations::ActivityReader::Refused => {
5594                return Ok(vec![control_error_frame(
5595                    &frame,
5596                    ACTIVITY_READ_FORBIDDEN,
5597                    "helper, relay and retired connections cannot read activity",
5598                )?]);
5599            }
5600            channel_operations::ActivityReader::Legacy => None,
5601        };
5602        // A registered module connection already identifies its caller. Ignore
5603        // consumer_identity there so a module cannot borrow a mirror's identity.
5604        // Only separate consumer connections use the Direct/nonce trust boundary.
5605        let principal = match legacy {
5606            Some(principal) => principal,
5607            None => match self
5608                .forwarding
5609                .module_id_for_connection(ctx.connection_id)
5610                .map_err(RouterError::Forwarding)?
5611            {
5612                Some(module_id) => Ok(Principal::Reserved { module_id }),
5613                None => {
5614                    let identity: Identity = match serde_json::from_slice(&frame.body) {
5615                        Ok(identity) => identity,
5616                        Err(_) => {
5617                            return Ok(vec![control_error_frame(
5618                                &frame,
5619                                ACTIVITY_MALFORMED,
5620                                "malformed activity read",
5621                            )?])
5622                        }
5623                    };
5624                    self.route_open_principal(
5625                        ctx.connection_id,
5626                        &frame,
5627                        identity.consumer_identity,
5628                    )?
5629                }
5630            },
5631        };
5632        let admitted = matches!(principal, Ok(Principal::Direct))
5633            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
5634        if !admitted {
5635            return Ok(vec![control_error_frame(
5636                &frame,
5637                ACTIVITY_READ_FORBIDDEN,
5638                "activity reads require an operator or trusted mirror principal",
5639            )?]);
5640        }
5641        let Some(ledger) = &self.activity else {
5642            return Ok(vec![control_error_frame(
5643                &frame,
5644                ACTIVITY_UNAVAILABLE,
5645                "activity ledger unavailable",
5646            )?]);
5647        };
5648        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
5649            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
5650        {
5651            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
5652                Ok(request) => ledger.query(request).await,
5653                Err(_) => Err(ACTIVITY_MALFORMED),
5654            };
5655            return match result {
5656                Ok(reply) => Ok(vec![control_response_body_frame(
5657                    &frame,
5658                    &reply,
5659                    ops::ACTIVITY_QUERY,
5660                )?]),
5661                Err(code) => Ok(vec![control_error_frame(
5662                    &frame,
5663                    code,
5664                    "activity query refused",
5665                )?]),
5666            };
5667        }
5668        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
5669            Ok(request) => request,
5670            Err(_) => {
5671                return Ok(vec![control_error_frame(
5672                    &frame,
5673                    ACTIVITY_MALFORMED,
5674                    "malformed activity subscription",
5675                )?])
5676            }
5677        };
5678        let mut subscription = match ledger.subscribe(request.since).await {
5679            Ok(subscription) => subscription,
5680            Err(code) => {
5681                return Ok(vec![control_error_frame(
5682                    &frame,
5683                    code,
5684                    "activity subscription refused",
5685                )?])
5686            }
5687        };
5688        let key = (ctx.connection_id, frame.header.corr);
5689        let sink = ctx.egress.clone();
5690        let mut streams = self
5691            .activity_streams
5692            .lock()
5693            .unwrap_or_else(|p| p.into_inner());
5694        streams.retain(|_, task| !task.is_finished());
5695        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
5696            return Ok(vec![control_error_frame(
5697                &frame,
5698                "duplicate_subscription",
5699                "activity subscription correlation id already in use",
5700            )?]);
5701        }
5702        let stream = async move {
5703            loop {
5704                let entry = if let Some(entry) = subscription.replay.pop_front() {
5705                    Some(entry)
5706                } else {
5707                    subscription.receiver.recv().await
5708                };
5709                let Some(entry) = entry else {
5710                    break;
5711                };
5712                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
5713                    return;
5714                };
5715                let Ok(next) = Frame::build_with_version(
5716                    response_version(&frame),
5717                    FrameType::StreamData,
5718                    control_flags(),
5719                    0,
5720                    0,
5721                    frame.header.corr,
5722                    body,
5723                ) else {
5724                    return;
5725                };
5726                if sink.send(next).await.is_err() {
5727                    return;
5728                }
5729            }
5730            // Dropping a full subscriber's sender drains its accepted entries
5731            // before the terminal error, preserving their order on the wire.
5732            if let Some(code) = subscription.terminal_code() {
5733                if let Ok(end) =
5734                    control_error_frame(&frame, code, "activity subscriber fell behind")
5735                {
5736                    let _ = sink.send(end).await;
5737                }
5738            }
5739        };
5740        // The spawn's retirement revokes this connection's authority. End the
5741        // subscription then, rather than when the connection's close is
5742        // eventually processed or the next entry fails to send.
5743        let task = tokio::spawn(async move {
5744            match bound_spawn {
5745                Some(spawn) => {
5746                    tokio::select! {
5747                        () = stream => {}
5748                        () = spawn.wait_closed() => {}
5749                    }
5750                }
5751                None => stream.await,
5752            }
5753        });
5754        streams.insert(key, task.abort_handle());
5755        Ok(Vec::new())
5756    }
5757
5758    async fn handle_activity_record(
5759        &self,
5760        ctx: &RouteCtx,
5761        frame: Frame,
5762    ) -> Result<Vec<Frame>, RouterError> {
5763        use subc_control::{
5764            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
5765            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
5766        };
5767        let result = async {
5768            let module = self
5769                .forwarding
5770                .module_id_for_connection(ctx.connection_id)
5771                .ok()
5772                .flatten()
5773                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
5774            let request: ActivityRecordRequest =
5775                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
5776            crate::activity::validate(&module, &request)?;
5777            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
5778            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
5779            let at_ms = std::time::SystemTime::now()
5780                .duration_since(std::time::UNIX_EPOCH)
5781                .unwrap_or_default()
5782                .as_millis() as u64;
5783            ledger.record(module, request, actor, at_ms).await
5784        }
5785        .await;
5786        match result {
5787            Ok(entry) => Ok(vec![control_response_body_frame(
5788                &frame,
5789                &ActivityRecordResponse { seq: entry.seq },
5790                "activity.record",
5791            )?]),
5792            Err(code) => Ok(vec![control_error_frame(
5793                &frame,
5794                code,
5795                "activity recording refused",
5796            )?]),
5797        }
5798    }
5799
5800    fn activity_actor(
5801        &self,
5802        connection: ConnectionId,
5803        module: &str,
5804        on: Option<&subc_control::ActivityOn>,
5805    ) -> Result<subc_control::ActivityActor, &'static str> {
5806        use crate::forwarding::{DataRoute, DataRouteState};
5807        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
5808        let principal = |id: &str| Principal::Reserved {
5809            module_id: id.to_owned(),
5810        };
5811        let mut actor = ActivityActor {
5812            principal: principal(module),
5813            scope_owner: None,
5814            agent_id: None,
5815            run_id: None,
5816            flow_id: None,
5817            project_root: None,
5818            workdir: None,
5819            project_id: None,
5820        };
5821        let scope = match on {
5822            None => return Ok(actor),
5823            Some(ActivityOn::Scope(scope)) => {
5824                let selector = subc_protocol::scope::ScopeSelector::new(
5825                    principal(module),
5826                    scope.scope_ref.clone(),
5827                    Some(scope.epoch),
5828                );
5829                self.scopes
5830                    .read()
5831                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5832                    .attribute(&principal(module), module, &selector, true)
5833                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5834            }
5835            Some(ActivityOn::Route(route)) => {
5836                let DataRoute::Module(DataRouteState::Bound(binding)) = self
5837                    .forwarding
5838                    .lookup_data_route(connection, route.channel, route.epoch)
5839                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5840                else {
5841                    return Err(ACTIVITY_ACTOR_UNKNOWN);
5842                };
5843                if binding.module_id != module {
5844                    return Err(ACTIVITY_ACTOR_UNKNOWN);
5845                }
5846                actor.principal = binding.principal.clone();
5847                actor.project_root = binding
5848                    .project_root
5849                    .as_ref()
5850                    .map(|root| root.as_path().to_path_buf());
5851                let Some(bound) = &binding.scope else {
5852                    return Ok(actor);
5853                };
5854                let owner = principal(&bound.owner);
5855                let selector = subc_protocol::scope::ScopeSelector::new(
5856                    owner.clone(),
5857                    bound.scope_ref.clone(),
5858                    Some(bound.tag.scope_epoch),
5859                );
5860                let admitted = self
5861                    .scopes
5862                    .read()
5863                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5864                    .attribute(&owner, module, &selector, true)
5865                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
5866                if admitted.tag != bound.tag {
5867                    return Err(ACTIVITY_ACTOR_UNKNOWN);
5868                }
5869                admitted
5870            }
5871        };
5872        actor.scope_owner = Some(scope.stamp.owner);
5873        actor.agent_id = scope.stamp.attributes.agent_id;
5874        actor.run_id = scope.stamp.attributes.run_id;
5875        actor.flow_id = scope.stamp.attributes.flow_id;
5876        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
5877        actor.project_id = scope.stamp.project_id;
5878        Ok(actor)
5879    }
5880
5881    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5882        self.refresh_capability_requirements();
5883        // A bare connection count is ambiguous between many clients holding a
5884        // route each and one client accumulating hundreds, so publish the
5885        // concentration alongside it. Route state is best-effort here: a
5886        // diagnostic endpoint must still answer if the forwarding lock is
5887        // contended.
5888        let mut counters = self.counters.snapshot();
5889        let losses = self.resource_tracker.loss_diagnostics();
5890        if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
5891            if let Some(obj) = counters.as_object_mut() {
5892                obj.insert("agent_command_declaration_loss_by_module".into(), losses);
5893            }
5894        }
5895        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
5896            obj.insert(
5897                "activity_unparseable_lines".into(),
5898                ledger.read_errors().into(),
5899            );
5900            obj.insert(
5901                "activity_lifecycle_append_failures".into(),
5902                ledger.lifecycle_failures().into(),
5903            );
5904            obj.insert(
5905                "activity_started_digest_skips".into(),
5906                ledger.lifecycle_digest_skips().into(),
5907            );
5908        }
5909        if let (Ok((connections_with_routes, max)), Some(obj)) = (
5910            self.forwarding.client_route_concentration(),
5911            counters.as_object_mut(),
5912        ) {
5913            obj.insert(
5914                "client_connections_with_routes".into(),
5915                connections_with_routes.into(),
5916            );
5917            obj.insert("max_routes_on_one_connection".into(), max.into());
5918        }
5919        // A module that is being fast-refused and a module that is fine look
5920        // identical from a client that retries and succeeds, so name the open
5921        // breakers here. This rides the existing free-form counters object
5922        // rather than a new wire field, so no sibling that deserializes
5923        // `ServerDescribe` has to be rebuilt to keep reading it.
5924        if let (Some(open_breakers), Some(obj)) = (
5925            self.route_bind_breakers.open_snapshot(),
5926            counters.as_object_mut(),
5927        ) {
5928            obj.insert("route_bind_breakers_open".into(), open_breakers);
5929        }
5930        // A daemon whose local endpoint could not be bound keeps serving TCP,
5931        // and the connection file names no endpoint. Name the reason here, in
5932        // the free-form counters object, so an operator reading daemon facts
5933        // sees it without searching the log, and no wire field changes.
5934        if let (Some(reason), Some(obj)) = (
5935            self.local_endpoint_unavailable.clone(),
5936            counters.as_object_mut(),
5937        ) {
5938            obj.insert("local_endpoint_unavailable".into(), reason.into());
5939        }
5940        let response = ClientControlResponse::ServerDescribe {
5941            protocol_ver: PROTOCOL_VERSION,
5942            subc_ops: subc_ops(),
5943            capabilities: self.describe_capabilities(),
5944            connected_clients: self.connected_clients.count(),
5945            counters: Some(counters),
5946            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
5947            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
5948            capability_requirements: self.capability_requirement_statuses(),
5949            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
5950            config_path: self
5951                .rescan
5952                .as_ref()
5953                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
5954        };
5955        Ok(vec![control_response_body_frame(
5956            &frame,
5957            &response,
5958            "ClientControlResponse::ServerDescribe",
5959        )?])
5960    }
5961
5962    fn handle_catalog_list(
5963        &self,
5964        frame: Frame,
5965        module_id: Option<String>,
5966    ) -> Result<Vec<Frame>, RouterError> {
5967        let (generation, modules) = self
5968            .registry
5969            .list_modules_with_nonce_verification()
5970            .map_err(|err| {
5971                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
5972            })?;
5973        let entries = modules
5974            .into_iter()
5975            .filter(|(registration, _)| {
5976                module_id
5977                    .as_deref()
5978                    .map(|wanted| registration.manifest.module_id == wanted)
5979                    .unwrap_or(true)
5980            })
5981            .map(|(registration, nonce_verified)| {
5982                let entry = self
5983                    .catalog_discovery_entry(&registration, module_id.is_some())
5984                    .map_err(|err| RouterError::backend(0, frame.header.corr, err))?;
5985                let not_ready = self.not_ready_reason(&registration);
5986                let mut entry = entry;
5987                entry.roles = registration.manifest.provides;
5988                entry.control_ops = registration.control_ops;
5989                Ok(entry
5990                    .with_ready(not_ready.is_none())
5991                    .with_display_name(registration.manifest.display_name)
5992                    .with_not_ready(not_ready)
5993                    .with_module_version(Some(registration.manifest.module_version))
5994                    .with_provenance(registration.manifest.provenance)
5995                    .with_capabilities(registration.manifest.capabilities)
5996                    .with_self_signals(registration.manifest.self_signals)
5997                    .with_events(registration.manifest.events)
5998                    .with_operation_delivery(registration.manifest.operation_delivery)
5999                    .with_operation_payload(registration.manifest.operation_payload)
6000                    .with_flow_operations(registration.manifest.flow_operations)
6001                    .with_operation_deferral(registration.manifest.operation_deferral)
6002                    .with_confirm_operations(registration.manifest.confirm_operations)
6003                    .with_rich_ui(registration.manifest.rich_ui)
6004                    .with_role_declarations(registration.manifest.role_declarations)
6005                    .with_nonce_verified(nonce_verified))
6006            })
6007            .collect::<Result<Vec<_>, RouterError>>()?;
6008        let response = ClientControlResponse::CatalogList {
6009            generation,
6010            modules: entries,
6011            subc_ops: subc_ops(),
6012        };
6013        Ok(vec![control_response_body_frame(
6014            &frame,
6015            &response,
6016            "ClientControlResponse::CatalogList",
6017        )?])
6018    }
6019
6020    /// The caller fields for a refusal that only has the raw request frame,
6021    /// such as the connection-level admission refusal sent before the request
6022    /// reaches [`Self::handle_route_open`].
6023    fn route_open_caller_from_frame(&self, ctx: &RouteCtx, frame: &Frame) -> RouteOpenCaller {
6024        let Ok(ClientControlRequest::RouteOpen {
6025            identity,
6026            consumer_identity,
6027            scope,
6028            ..
6029        }) = parse_client_control_request(&frame.body)
6030        else {
6031            return RouteOpenCaller::unknown();
6032        };
6033        let principal = self
6034            .attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref())
6035            .map(|attested| attested.principal);
6036        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
6037    }
6038
6039    fn route_open_principal(
6040        &self,
6041        connection_id: ConnectionId,
6042        frame: &Frame,
6043        consumer_identity: Option<ConsumerIdentity>,
6044    ) -> Result<Result<Principal, Frame>, RouterError> {
6045        let attested = self
6046            .attest_route_open_consumer(connection_id, consumer_identity.as_ref())
6047            .map(|attested| attested.principal);
6048        self.route_open_principal_attested(frame, consumer_identity, attested)
6049    }
6050
6051    /// [`Self::route_open_principal`] with `attested` already taken by
6052    /// [`Self::attest_route_open_consumer`], so one open resolves its connection
6053    /// claim or launch nonce once.
6054    fn route_open_principal_attested(
6055        &self,
6056        frame: &Frame,
6057        consumer_identity: Option<ConsumerIdentity>,
6058        attested: Option<Principal>,
6059    ) -> Result<Result<Principal, Frame>, RouterError> {
6060        if let Some(principal) = attested {
6061            return Ok(Ok(principal));
6062        }
6063        Ok(Err(control_error_frame(
6064            frame,
6065            "bad_consumer_identity",
6066            consumer_identity.map_or_else(
6067                || "connection claim is no longer authorized for routes".to_string(),
6068                |identity| format!(
6069                    "consumer_identity for module_id '{}' conflicts with its connection claim or did not match a supervised launch nonce",
6070                    identity.module_id
6071                ),
6072            ),
6073        )?))
6074    }
6075
6076    /// Ordinary `route.open` refusals go through here; admission and breaker
6077    /// refusals log separately with their capacity or breaker state. The daemon can
6078    /// attest which code it sent: without the event, a client's "the daemon
6079    /// refused me" and the daemon's own view could only be reconciled by
6080    /// argument. Malformed input (`invalid_project_root`) does not come here;
6081    /// rejecting a request that was never a valid open is not a refusal of one.
6082    #[allow(clippy::too_many_arguments)]
6083    fn route_open_refusal_frame(
6084        &self,
6085        ctx: &RouteCtx,
6086        caller: &RouteOpenCaller,
6087        frame: &Frame,
6088        module_id: &str,
6089        reason: &'static str,
6090        code: &'static str,
6091        message: impl Into<String>,
6092    ) -> Result<Frame, RouterError> {
6093        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
6094        let detail = if matches!(
6095            reason,
6096            "registration_not_active"
6097                | "supervisor_not_live"
6098                | "no_forwarding_connection"
6099                | "reloading"
6100        ) {
6101            // A planned restart, reload or swap is already replacing the
6102            // process; it does not need a slot in the crash-restart budget.
6103            let expected_back = (reason == "reloading").then_some(true);
6104            self.route_open_recovery_detail(module_id, frame.header.corr, None, expected_back)?
6105        } else {
6106            None
6107        };
6108        control_error_body_frame(
6109            frame,
6110            ErrorBody {
6111                code: code.to_string(),
6112                message: message.into(),
6113                detail,
6114            },
6115        )
6116    }
6117
6118    fn route_open_recovery_detail(
6119        &self,
6120        module_id: &str,
6121        corr: u64,
6122        detail: Option<serde_json::Value>,
6123        expected_back: Option<bool>,
6124    ) -> Result<Option<serde_json::Value>, RouterError> {
6125        let Some((status, _)) = self.supervisor_status(module_id, corr)? else {
6126            return Ok(detail);
6127        };
6128        let mut detail = detail.unwrap_or_else(|| serde_json::json!({}));
6129        detail["state"] = serde_json::json!(status.state.to_string());
6130        if let Some(expected_back) = expected_back {
6131            detail["expected_back"] = serde_json::json!(expected_back);
6132        } else if let Some(module) = self.supervisor.get(module_id) {
6133            match module.will_recover_after_connection_loss() {
6134                Ok(expected_back) => detail["expected_back"] = serde_json::json!(expected_back),
6135                Err(err) => warn!(
6136                    %module_id,
6137                    error = %err,
6138                    "failed to read route.open recovery verdict; omitting expected_back"
6139                ),
6140            }
6141        }
6142        Ok(Some(detail))
6143    }
6144
6145    /// Answer a waiting `route.open` whose relayed bind was refused.
6146    ///
6147    /// `daemon_authored` is provenance, not a code match: the daemon
6148    /// synthesized the refusal itself (a drain, a late reply, a scope that
6149    /// ended mid-bind) or the module refused the bind with its own body. A
6150    /// daemon refusal carries the daemon-origin bit; a module's own
6151    /// rejection is relayed with the bit clear, so a module that refuses
6152    /// with `module_reloading` still cannot mint a daemon refusal.
6153    #[allow(clippy::too_many_arguments)]
6154    fn route_bind_rejection(
6155        &self,
6156        ctx: &RouteCtx,
6157        caller: &RouteOpenCaller,
6158        frame: &Frame,
6159        target_module_id: &str,
6160        body: ErrorBody,
6161        daemon_authored: bool,
6162    ) -> Result<Vec<Frame>, RouterError> {
6163        // The daemon's own commit re-check refused the bind because the
6164        // scope ended or changed after admission. The module accepted;
6165        // counting it as a module rejection would blame the module. This
6166        // branch only runs for daemon-synthesized bodies: a module that
6167        // refuses with a scope code is answered as a module rejection below.
6168        if daemon_authored {
6169            let scope_code = match body.code.as_str() {
6170                error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
6171                error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
6172                _ => None,
6173            };
6174            if let Some(code) = scope_code {
6175                self.observe_route_open_refusal(
6176                    ctx,
6177                    caller,
6178                    target_module_id,
6179                    "scope_changed_before_commit",
6180                    code,
6181                );
6182                return Ok(vec![control_error_body_frame(frame, body)?]);
6183            }
6184        }
6185        self.counters
6186            .increment_route_open_refused("module_rejected");
6187        info!(
6188            target: "control",
6189            code = "module_rejected",
6190            module_code = ?body.code,
6191            module_id = ?target_module_id,
6192            connection_id = ctx.connection_id.get(),
6193            principal = %caller.principal,
6194            scope = %caller.scope,
6195            harness = %caller.harness,
6196            "route.open refused"
6197        );
6198        let refusal = if daemon_authored {
6199            control_error_body_frame(frame, body)?
6200        } else {
6201            control_relayed_error_body_frame(frame, body)?
6202        };
6203        Ok(vec![refusal])
6204    }
6205
6206    /// Refuse a `route.open` because the target module's bind-relay breaker is
6207    /// open, without attempting the relay.
6208    ///
6209    /// The wire code is `module_timeout`, which is the truth (the module has
6210    /// not been answering binds) and which both SDKs already classify as
6211    /// retryable with capped backoff. Reusing it is what keeps this change out
6212    /// of both SDKs; the daemon-side distinction lives in the counter key
6213    /// instead.
6214    ///
6215    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
6216    /// While a breaker is open this fires on every open to that module, and the
6217    /// stall written up in `docs/designs/route-open-head-of-line.md` already
6218    /// produced 261 lines about a single module inside 3000 lines of daemon
6219    /// log. The rare transitions are logged at warn/info instead and the volume
6220    /// is carried by the counter, so the evidence survives without the flood.
6221    /// The debug line keeps a per-refusal record reachable for whoever turns
6222    /// the level up.
6223    #[allow(clippy::too_many_arguments)]
6224    fn route_open_breaker_refusal_frame(
6225        &self,
6226        ctx: &RouteCtx,
6227        caller: &RouteOpenCaller,
6228        frame: &Frame,
6229        module_id: &str,
6230        consecutive_timeouts: u32,
6231        retry_in: Duration,
6232        probe_in_flight: bool,
6233    ) -> Result<Frame, RouterError> {
6234        self.counters
6235            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
6236        debug!(
6237            target: "control",
6238            code = "module_timeout",
6239            module_id = ?module_id,
6240            connection_id = ctx.connection_id.get(),
6241            principal = %caller.principal,
6242            scope = %caller.scope,
6243            harness = %caller.harness,
6244            consecutive_timeouts,
6245            retry_in_ms = retry_in.as_millis() as u64,
6246            probe_in_flight,
6247            "route.open refused by open bind-relay breaker"
6248        );
6249        // Say what a caller can act on. An open bind-relay breaker means the
6250        // module timed out accepting several new routes in a row. The module
6251        // is still running and its established routes keep working; only new
6252        // route.open requests are refused until the cooldown ends and one
6253        // test route (the probe) gets through. A message that only counts
6254        // failed relays reads as "the module is down" to a worker that sees it.
6255        let detail = if probe_in_flight {
6256            "one test route is already being tried; retry once it settles".to_string()
6257        } else {
6258            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
6259        };
6260        control_error_frame(
6261            frame,
6262            "module_timeout",
6263            format!(
6264                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
6265                 timed out in a row); {detail}; its established routes are unaffected"
6266            ),
6267        )
6268    }
6269
6270    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
6271    /// requester's bytes (an unknown target is whatever the client sent) and
6272    /// is Debug-formatted so control characters land in the log escaped
6273    /// rather than as terminal sequences for whoever tails it.
6274    ///
6275    /// `reason` names the check that refused, because one wire code has
6276    /// several senders: after a module registers, `target_unavailable` can
6277    /// come from a missing role, an inactive registration, a supervisor that
6278    /// has not marked the process live, a missing forwarding connection, or a
6279    /// failed relay, and a log that records only the code cannot say which of
6280    /// them fired. It is a static, daemon-chosen label per branch, so it is
6281    /// safe to print plainly and stays a closed set.
6282    fn observe_route_open_refusal(
6283        &self,
6284        ctx: &RouteCtx,
6285        caller: &RouteOpenCaller,
6286        module_id: &str,
6287        reason: &'static str,
6288        code: &'static str,
6289    ) {
6290        self.counters.increment_route_open_refused(code);
6291        info!(
6292            target: "control",
6293            code,
6294            reason,
6295            module_id = ?module_id,
6296            connection_id = ctx.connection_id.get(),
6297            principal = %caller.principal,
6298            scope = %caller.scope,
6299            harness = %caller.harness,
6300            "route.open refused"
6301        );
6302        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
6303            self.route_outages.record_not_serving(module_id, reason);
6304        }
6305    }
6306
6307    /// Record an ACCEPTED route.open.
6308    ///
6309    /// Refusals have been logged and counted since the attestation work; accepts
6310    /// were invisible, so the daemon knew every principal it stamped and wrote
6311    /// none of them down. The party that attests the identity was the only party
6312    /// not recording it, which left a credential vault unable to name the sender
6313    /// of a call that reached it (claustrum #43) and left the launch-nonce
6314    /// concurrency question unanswerable from the outside.
6315    ///
6316    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
6317    /// `code`/`module_id`/`connection_id` returns both directions of the same
6318    /// decision rather than two shapes a reader has to join by hand.
6319    ///
6320    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
6321    /// and the difference carries information rather than being an
6322    /// inconsistency. This line is only reachable after a successful bind to a
6323    /// REGISTERED module, so the value has already passed HELLO validation
6324    /// including the path-hazard refusal and cannot contain control bytes. A
6325    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
6326    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
6327    ///
6328    /// Bare is also what every other daemon line already emits (`module
6329    /// registered`, `configured module supervised`). Shipping `?module_id` here
6330    /// made this instrument the only one in the file whose ids did not answer
6331    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
6332    /// line whose whole purpose is being grepped beside its sibling.
6333    ///
6334    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
6335    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
6336    /// forwards every field type through it and a bare `&str` and a `?`-escaped
6337    /// one are recorded identically. A test written against that harness passes
6338    /// either way -- I wrote one, measured it, and deleted it rather than ship a
6339    /// green assertion that cannot fail. The same limit applies to the escaping
6340    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
6341    /// it reads as a guard on the Debug escaping and cannot detect its removal.
6342    /// Fencing either needs the real formatter, not the capture layer.
6343    ///
6344    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
6345    /// unix-socket options and subc is loopback TCP, so there is no peer identity
6346    /// to record. The ephemeral port would decay within minutes and answer only a
6347    /// live question. The identity question is instead answered by counting
6348    /// distinct live connections presenting one module's `consumer_identity` --
6349    /// "is anyone else holding this secret" rather than "is this the right
6350    /// process".
6351    fn observe_route_open_accept(
6352        &self,
6353        ctx: &RouteCtx,
6354        module_id: &str,
6355        principal: &str,
6356        scope: &str,
6357    ) {
6358        self.route_outages.record_accepted(module_id);
6359        self.counters.increment_route_open_accepted(principal);
6360        info!(
6361            target: "control",
6362            principal,
6363            module_id,
6364            connection_id = ctx.connection_id.get(),
6365            scope = %scope,
6366            "route.open accepted"
6367        );
6368    }
6369
6370    fn supervised_absent_route_open_refusal_frame(
6371        &self,
6372        ctx: &RouteCtx,
6373        caller: &RouteOpenCaller,
6374        frame: &Frame,
6375        module_id: &str,
6376        code: &'static str,
6377        status: &crate::supervise::ModuleStatus,
6378    ) -> Result<Frame, RouterError> {
6379        self.counters.increment_route_open_refused(code);
6380        info!(
6381            target: "control",
6382            code,
6383            reason = "supervised_not_registered",
6384            module_id = ?module_id,
6385            connection_id = ctx.connection_id.get(),
6386            principal = %caller.principal,
6387            scope = %caller.scope,
6388            harness = %caller.harness,
6389            state = %status.state,
6390            enabled = status.enabled,
6391            live = status.live,
6392            "route.open refused"
6393        );
6394        // A supervised module whose process has not registered is not
6395        // serving, whatever the reason; the supervisor knows this id, so it is
6396        // safe to track.
6397        self.route_outages
6398            .record_not_serving(module_id, "supervised_not_registered");
6399        let message = if let Some(pid) = status.waiting_for_previous_pid {
6400            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
6401                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
6402            })
6403        } else {
6404            format!(
6405                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
6406                status.state, status.enabled, status.live
6407            )
6408        };
6409        // `module_warming` here means the process is starting, running or
6410        // being restarted and has not registered yet. Nothing has to be
6411        // restarted for it to serve, so the crash budget does not decide it.
6412        let expected_back = (code == error_codes::MODULE_WARMING).then_some(true);
6413        control_error_body_frame(
6414            frame,
6415            ErrorBody {
6416                code: code.to_string(),
6417                message,
6418                detail: self.route_open_recovery_detail(
6419                    module_id,
6420                    frame.header.corr,
6421                    None,
6422                    expected_back,
6423                )?,
6424            },
6425        )
6426    }
6427
6428    async fn handle_route_open(
6429        &self,
6430        ctx: &RouteCtx,
6431        frame: Frame,
6432        request: RouteOpenRequest,
6433        timings: &mut RouteOpenTimings,
6434    ) -> Result<Vec<Frame>, RouterError> {
6435        let RouteOpenRequest {
6436            target,
6437            mut identity,
6438            consumer_identity,
6439            consumer_capabilities,
6440            role_versions,
6441            busy_refusals,
6442            admission_facts,
6443            scope,
6444            accepts_close_reasons,
6445        } = request;
6446        let target_module_id = target_module_id(&target).to_string();
6447        timings.module_id.clone_from(&target_module_id);
6448        if self
6449            .registry
6450            .get_module_by_connection(ctx.connection_id)
6451            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6452            .is_some()
6453        {
6454            return Ok(vec![control_error_frame(
6455                &frame,
6456                "invalid_request",
6457                "module connections cannot open client routes",
6458            )?]);
6459        }
6460        debug!(
6461            connection_id = ctx.connection_id.get(),
6462            corr = frame.header.corr,
6463            module_id = %target_module_id,
6464            "handling route.open"
6465        );
6466
6467        // The caller's attested principal, requested scope and harness, for
6468        // every `route.open refused` line below. The connection's claim
6469        // binding or launch nonce is checked here, once, and the answer is
6470        // reused where the principal is enforced further down, so the order
6471        // of refusals is unchanged.
6472        let attested =
6473            self.attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref());
6474        let mut caller = RouteOpenCaller::new(
6475            attested.as_ref().map(|attested| &attested.principal),
6476            scope.as_ref(),
6477            &identity,
6478        );
6479
6480        // A malformed declaration is refused first, before anything about the
6481        // target is looked up: the same body would be refused against any
6482        // module, so the caller learns nothing by retrying or waiting. An empty
6483        // map declares nothing and travels as no field at all, so a provider
6484        // only ever sees a missing field or a non-empty one.
6485        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
6486        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
6487            self.observe_route_open_refusal(
6488                ctx,
6489                &caller,
6490                &target_module_id,
6491                "invalid_role_versions",
6492                error_codes::INVALID_REQUEST,
6493            );
6494            return Ok(vec![control_error_body_frame(
6495                &frame,
6496                ErrorBody {
6497                    code: error_codes::INVALID_REQUEST.to_string(),
6498                    message: error.to_string(),
6499                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
6500                },
6501            )?]);
6502        }
6503
6504        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
6505            if status.waiting_for_previous_pid.is_some() {
6506                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6507                    ctx,
6508                    &caller,
6509                    &frame,
6510                    &target_module_id,
6511                    "module_warming",
6512                    &status,
6513                )?]);
6514            }
6515        }
6516
6517        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
6518        // opposite. Below, a caller learns whether a module is unregistered,
6519        // supervised-but-down (with state/enabled/live), or registered without the
6520        // requested role. Elsewhere that is an enumeration leak: a probe learning
6521        // the shape of a fleet it cannot otherwise see.
6522        //
6523        // It is not one here, and the reason is the ACCESS MODEL rather than
6524        // anything about these errors. Reaching route.open requires the
6525        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
6526        // connection file, so any caller who completes it already runs as this
6527        // user -- and can read subc.jsonc for the module list and `ck module
6528        // status` for live state. The reply discloses nothing the caller cannot
6529        // read more easily from disk, while the precision is load-bearing:
6530        // `unknown_module` is retryable and a missing role is not.
6531        //
6532        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
6533        // remote transport, a sandboxed caller, a shared-host mode -- THAT
6534        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
6535        let admission_registry = self
6536            .registry
6537            .route_admission_snapshot()
6538            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6539        let Some(registration) =
6540            crate::scopes::admission::registered_target(&admission_registry, &target_module_id)
6541        else {
6542            if let Some((status, warming)) =
6543                self.supervisor_status(&target_module_id, frame.header.corr)?
6544            {
6545                // BEFORE the two availability codes below, because for a module
6546                // that speaks no subc wire both of them are false comfort: they
6547                // say "not right now" and are retried, and this module will
6548                // never register no matter how long the caller waits. The
6549                // absence here is the declaration being honoured, not a module
6550                // that is late.
6551                if status.protocol == ModuleProtocol::None {
6552                    return Ok(vec![self.route_open_refusal_frame(
6553                        ctx,
6554                        &caller,
6555                        &frame,
6556                        &target_module_id,
6557                        "protocol_none",
6558                        error_codes::MODULE_NO_PROTOCOL,
6559                        format!(
6560                            "module_id '{target_module_id}' is declared protocol: none; \
6561                             it speaks no subc wire and serves no routes"
6562                        ),
6563                    )?]);
6564                }
6565                let code = if warming {
6566                    "module_warming"
6567                } else {
6568                    "target_unavailable"
6569                };
6570                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6571                    ctx,
6572                    &caller,
6573                    &frame,
6574                    &target_module_id,
6575                    code,
6576                    &status,
6577                )?]);
6578            }
6579            if let Some(removed_ago_ms) =
6580                self.supervisor.removal_tombstone_age_ms(&target_module_id)
6581            {
6582                return Ok(vec![self.route_open_refusal_frame(
6583                    ctx,
6584                    &caller,
6585                    &frame,
6586                    &target_module_id,
6587                    "removed",
6588                    error_codes::MODULE_REMOVED,
6589                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
6590                )?]);
6591            }
6592            return Ok(vec![self.route_open_refusal_frame(
6593                ctx,
6594                &caller,
6595                &frame,
6596                &target_module_id,
6597                "not_registered",
6598                error_codes::UNKNOWN_MODULE,
6599                format!("module_id '{target_module_id}' is not registered"),
6600            )?]);
6601        };
6602
6603        // Best-effort only: registry readiness and forwarding reservation use
6604        // different locks, so a module can flip readiness between this read and
6605        // the relay. Modules must still tolerate an `on_bind` while not ready.
6606        if crate::scopes::admission::target_registration(&admission_registry, &target_module_id)
6607            .is_err()
6608        {
6609            self.counters
6610                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
6611            info!(
6612                target: "control",
6613                code = error_codes::MODULE_WARMING,
6614                module_id = ?target_module_id,
6615                connection_id = ctx.connection_id.get(),
6616                principal = %caller.principal,
6617                scope = %caller.scope,
6618                harness = %caller.harness,
6619                reason = "declared_not_ready",
6620                "route.open refused"
6621            );
6622            // The module is registered but says it cannot take work, which is
6623            // an outage from the caller's side even though its process is up.
6624            self.route_outages
6625                .record_not_serving(&target_module_id, "declared_not_ready");
6626            // The registered process can declare itself ready without a
6627            // restart, so an exhausted crash budget does not prevent serving.
6628            return Ok(vec![control_error_body_frame(
6629                &frame,
6630                ErrorBody {
6631                    code: error_codes::MODULE_WARMING.to_string(),
6632                    message: format!(
6633                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
6634                    ),
6635                    detail: self.route_open_recovery_detail(
6636                        &target_module_id,
6637                        frame.header.corr,
6638                        Some(serde_json::json!({ "reason": "declared_not_ready" })),
6639                        Some(true),
6640                    )?,
6641                },
6642            )?]);
6643        }
6644
6645        // Effective readiness, second half: a module that declares a capability
6646        // `need: required` is not routable while that capability has no
6647        // registered provider. It is enforced HERE, as a retryable routing
6648        // refusal, and deliberately not as spawn ordering or a boot block. The
6649        // module is still started and registered and can make its own calls;
6650        // spawn ordering is a promise that cannot be kept once a provider
6651        // crashes at runtime, and refusing to boot would stop the whole
6652        // machine, including the tools needed to fix its configuration.
6653        //
6654        // "Provided" is the evaluator's verdict, which counts a provider as
6655        // soon as it has REGISTERED, not once it is ready. Two modules that
6656        // require each other's capabilities are therefore both routable once
6657        // both register; counting readiness instead would deadlock them.
6658        //
6659        // Only new opens are refused. Routes already bound when a provider
6660        // goes away stay bound: nothing here tears them down, and the module
6661        // answers them as it can. Like the readiness read above this is
6662        // best-effort against a provider registering or leaving concurrently.
6663        if let Some(capability) = self.first_unprovided_required_capability(registration) {
6664            self.counters
6665                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
6666            info!(
6667                target: "control",
6668                code = error_codes::MODULE_WARMING,
6669                module_id = ?target_module_id,
6670                connection_id = ctx.connection_id.get(),
6671                principal = %caller.principal,
6672                scope = %caller.scope,
6673                harness = %caller.harness,
6674                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6675                capability = %capability,
6676                "route.open refused"
6677            );
6678            return Ok(vec![control_error_body_frame(
6679                &frame,
6680                ErrorBody {
6681                    code: error_codes::MODULE_WARMING.to_string(),
6682                    message: format!(
6683                        "module_id '{target_module_id}' requires capability '{capability}', \
6684                         which no registered module provides; retry"
6685                    ),
6686                    detail: Some(serde_json::json!({
6687                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6688                        "capability": capability,
6689                    })),
6690                },
6691            )?]);
6692        }
6693
6694        if !target_has_required_role(&target, &registration.manifest.provides) {
6695            return Ok(vec![self.route_open_refusal_frame(
6696                ctx,
6697                &caller,
6698                &frame,
6699                &target_module_id,
6700                "role_not_provided",
6701                "target_unavailable",
6702                format!("module_id '{target_module_id}' does not provide the requested target"),
6703            )?]);
6704        }
6705
6706        if registration.state != ChannelState::Active {
6707            return Ok(vec![self.route_open_refusal_frame(
6708                ctx,
6709                &caller,
6710                &frame,
6711                &target_module_id,
6712                "registration_not_active",
6713                "target_unavailable",
6714                format!("module_id '{target_module_id}' is not active"),
6715            )?]);
6716        }
6717
6718        if self
6719            .forwarding
6720            .module_is_draining(&target_module_id)
6721            .map_err(RouterError::Forwarding)?
6722        {
6723            return Ok(vec![self.route_open_refusal_frame(
6724                ctx,
6725                &caller,
6726                &frame,
6727                &target_module_id,
6728                "reloading",
6729                "module_reloading",
6730                format!("module_id '{target_module_id}' is reloading"),
6731            )?]);
6732        }
6733
6734        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
6735            process_liveness.process_live(&target_module_id) == Some(false)
6736        }) {
6737            // A module the supervisor is restarting or reloading can still hold
6738            // a registration: the old process before its connection closes, or
6739            // a new one that registered while the supervisor was draining. The
6740            // forwarding table does not see that as draining, but the consumer
6741            // should still be told to retry soon, exactly as for the drain
6742            // above, rather than that the target is unavailable.
6743            if process_liveness.process_replacing(&target_module_id) {
6744                return Ok(vec![self.route_open_refusal_frame(
6745                    ctx,
6746                    &caller,
6747                    &frame,
6748                    &target_module_id,
6749                    "reloading",
6750                    "module_reloading",
6751                    format!("module_id '{target_module_id}' is reloading"),
6752                )?]);
6753            }
6754            return Ok(vec![self.route_open_refusal_frame(
6755                ctx,
6756                &caller,
6757                &frame,
6758                &target_module_id,
6759                "supervisor_not_live",
6760                "target_unavailable",
6761                format!("module_id '{target_module_id}' is not live"),
6762            )?]);
6763        }
6764
6765        if !self
6766            .forwarding
6767            .has_live_module_connection(&target_module_id)
6768            .map_err(RouterError::Forwarding)?
6769        {
6770            return Ok(vec![self.route_open_refusal_frame(
6771                ctx,
6772                &caller,
6773                &frame,
6774                &target_module_id,
6775                "no_forwarding_connection",
6776                "target_unavailable",
6777                format!("module_id '{target_module_id}' has no live forwarding connection"),
6778            )?]);
6779        }
6780
6781        if let Some(error) =
6782            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
6783        {
6784            self.observe_route_open_refusal(
6785                ctx,
6786                &caller,
6787                &target_module_id,
6788                "op_not_allowed",
6789                "op_not_allowed",
6790            );
6791            return Ok(vec![error]);
6792        }
6793
6794        let principal = match (&attested, consumer_identity.is_some()) {
6795            // A claim-bound connection whose launch ended attests nothing,
6796            // and it must never fall back to opening as `direct`.
6797            (None, false) => {
6798                self.observe_route_open_refusal(
6799                    ctx,
6800                    &caller,
6801                    &target_module_id,
6802                    "registration_channel_closed",
6803                    crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6804                );
6805                return Ok(vec![control_error_frame(
6806                    &frame,
6807                    crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6808                    "the claim binding's launch has ended",
6809                )?]);
6810            }
6811            _ => match self.route_open_principal_attested(
6812                &frame,
6813                consumer_identity,
6814                attested.as_ref().map(|attested| attested.principal.clone()),
6815            )? {
6816                Ok(principal) => principal,
6817                Err(error) => {
6818                    self.observe_route_open_refusal(
6819                        ctx,
6820                        &caller,
6821                        &target_module_id,
6822                        "bad_consumer_identity",
6823                        "bad_consumer_identity",
6824                    );
6825                    return Ok(vec![error]);
6826                }
6827            },
6828        };
6829        let attested = attested.expect("an attested principal was resolved above");
6830        timings.principal = principal_label(&principal);
6831
6832        // The opener's spawn authority, read once here and carried with the
6833        // relay; nothing below reads its record again.
6834        let Ok(mut caller_capture) = attested.capture() else {
6835            self.observe_route_open_refusal(
6836                ctx,
6837                &caller,
6838                &target_module_id,
6839                "registration_channel_closed",
6840                crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6841            );
6842            return Ok(vec![control_error_frame(
6843                &frame,
6844                crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6845                "the opener's launch has ended",
6846            )?]);
6847        };
6848        // The caller-side checks below read the opener's own spawn: its
6849        // module from its record and its manifest from its own registration,
6850        // never another launch's active registration under the same id.
6851        let caller_module_id = match &attested.spawn {
6852            Some((record, _)) => Some(record.module_id().to_owned()),
6853            None => match &principal {
6854                Principal::Reserved { module_id } => Some(module_id.clone()),
6855                _ => None,
6856            },
6857        };
6858
6859        // Only Callosum can attest a production phone build. Consume the claim
6860        // here so it can be kept on the route without reaching any module.
6861        let build_class = identity.build_class.take().filter(|build_class| {
6862            if caller_module_id.as_deref() == Some("callosum") {
6863                build_class == subc_protocol::BUILD_CLASS_PRODUCTION
6864            } else {
6865                if self
6866                    .build_class_warned
6867                    .lock()
6868                    .unwrap_or_else(|p| p.into_inner())
6869                    .insert(ctx.connection_id)
6870                {
6871                    warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
6872                        "ignoring build_class from a non-Callosum opener");
6873                }
6874                false
6875            }
6876        });
6877
6878        // This is attested, control-plane policy for supervised module origins.
6879        // Keep it before route reservation and out of the opaque forwarding hot
6880        // path: data frames must never acquire a per-frame capability check.
6881        if let Some(opening_registration) = self.caller_registration(&attested, &admission_registry)
6882        {
6883            let opening_module_id = opening_registration.manifest.module_id.as_str();
6884            if let Some(capability) =
6885                denied_capability(&opening_registration.manifest, &registration.manifest)
6886            {
6887                warn!(
6888                    opening_module_id,
6889                    target_module_id,
6890                    capability,
6891                    "refusing route.open because an attested capability deny edge matches"
6892                );
6893                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6894                    &frame,
6895                    &target_module_id,
6896                    "capability_deny_edge",
6897                    "capability_forbidden",
6898                    format!(
6899                        "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
6900                    ),
6901                )?]);
6902            }
6903        }
6904
6905        if admission_facts.is_some() {
6906            let carrier_matches = caller_module_id.is_some()
6907                && self.admission_facts_carrier_module_id.as_deref() == caller_module_id.as_deref();
6908            if !carrier_matches {
6909                return Ok(vec![self.route_open_refusal_frame(
6910                    ctx,
6911                    &caller,
6912                    &frame,
6913                    &target_module_id,
6914                    "admission_facts_carrier_not_permitted",
6915                    "admission_facts_not_permitted",
6916                    "admission facts may only be carried by the configured reserved module",
6917                )?]);
6918            }
6919
6920            let target_allowed = self
6921                .admission_facts_targets
6922                .as_ref()
6923                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
6924            if !target_allowed {
6925                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6926                    &frame,
6927                    &target_module_id,
6928                    "admission_facts_target_not_listed",
6929                    "admission_facts_target_not_allowed",
6930                    format!(
6931                        "admission facts are not permitted for target module_id '{target_module_id}'"
6932                    ),
6933                )?]);
6934            }
6935
6936            // Keep the value opaque to subc. The downstream admission validator owns
6937            // schema and semantic checks; this daemon only enforces carrier authority
6938            // and the configured destination allowlist.
6939        }
6940
6941        // Scope admission, on the attested principal above and never on the
6942        // request body. The tag read here travels with the pending bind and is
6943        // compared with the published one at commit, so a sync between here
6944        // and the module's ack refuses the open instead of binding a stamp
6945        // that is no longer true.
6946        let (bound_scope, mut scope_stamp, stamp_required) = match &scope {
6947            None => (None, None, false),
6948            Some(selector) => {
6949                let owner_configured = match &selector.owner {
6950                    // A reserved owner counts as configured from before its
6951                    // process is spawned (see `SupervisorHandle::is_configured`).
6952                    // So an owner that has not synced its scopes yet is refused
6953                    // as retryable (`scope_not_synced`), not as one that will
6954                    // never sync.
6955                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
6956                    _ => false,
6957                };
6958                let mut scopes = self
6959                    .scopes
6960                    .write()
6961                    .unwrap_or_else(|poisoned| poisoned.into_inner());
6962                let admitted = crate::scopes::admission::admit_route(
6963                    &scopes,
6964                    &admission_registry,
6965                    &principal,
6966                    &target_module_id,
6967                    Some(selector),
6968                    owner_configured,
6969                    crate::scopes::admission::AdmissionMode::Open,
6970                )
6971                .map(|route| {
6972                    let admission = route.scope.expect("scoped admission");
6973                    let bound = scopes.reserve_bind(
6974                        &admission,
6975                        &principal,
6976                        ctx.connection_id,
6977                        selector.hold_presence,
6978                    );
6979                    (admission, bound)
6980                });
6981                match admitted {
6982                    Ok((admission, bound)) => {
6983                        (Some(bound), Some(admission.stamp), admission.stamp_required)
6984                    }
6985                    Err(refusal) => {
6986                        return Ok(vec![self.route_open_refusal_frame(
6987                            ctx,
6988                            &caller,
6989                            &frame,
6990                            &target_module_id,
6991                            refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
6992                            refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
6993                            refusal.message,
6994                        )?]);
6995                    }
6996                }
6997            }
6998        };
6999        // From here on a refusal names the scope by the stamp admission
7000        // issued, exactly as the `route.open accepted` line does.
7001        if let Some(stamp) = &scope_stamp {
7002            caller.scope = route_open_scope_label(stamp);
7003        }
7004        // Decided from the admitted stamp, before any field of it is stripped
7005        // for a target that lacks the capability to read it.
7006        if scope_stamp.as_ref().is_some_and(scope_acts_for_another) {
7007            caller_capture = caller_capture.map(crate::spawn_authority::CallerCapture::relayed);
7008        }
7009
7010        // Allow resolvable missing-tail paths so a vanished directory does not
7011        // prevent reopening an existing session for cleanup. Binding does not
7012        // authorize creation of durable provider state.
7013        //
7014        // Resolve real paths rather than lexical cleanup: canonicalize the
7015        // longest existing ancestor and append the missing tail, retaining the
7016        // path's real existing ancestry even when a leaf is missing.
7017        timings.enter(RouteOpenPhase::ProjectRoot);
7018        let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
7019            Ok(project) => project,
7020            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
7021        };
7022        timings.enter(RouteOpenPhase::Admission);
7023        let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
7024            Ok(root) => root,
7025            Err(err) => {
7026                return Ok(vec![control_error_frame(
7027                    &frame,
7028                    "invalid_project_root",
7029                    err.to_string(),
7030                )?])
7031            }
7032        };
7033        // Compute compatibility project fields before stripping optional stamp
7034        // members. Preserve the canonical opener root and its project ID,
7035        // including absence. For rootless openers, use stamp workdir or `/`, and
7036        // opener ID, else stamp ID, else no ID. Emit these identity fields only
7037        // to targets without bind-no-project/v1.
7038        let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
7039
7040        // Last gate before any relay work, and deliberately after the cheap
7041        // registry and availability checks above: those name a more precise
7042        // condition (unknown, removed, reloading) and a caller is better served
7043        // by the precise code than by this one.
7044        //
7045        // Everything below this point costs an egress permit, a reserved handle
7046        // pair and, if the module does not answer, the whole relay budget. The
7047        // reader no longer waits for that budget, so cap each target explicitly;
7048        // serial dispatch used to provide the accidental cap of one relay per
7049        // connection. Admission is a mutex-protected count and never waits.
7050        let _concurrency_guard = match self
7051            .route_bind_concurrency
7052            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
7053        {
7054            Ok(guard) => guard,
7055            Err(in_flight) => {
7056                return Ok(vec![self.route_open_target_capacity_refusal(
7057                    ctx,
7058                    &caller,
7059                    &frame,
7060                    &target_module_id,
7061                    in_flight,
7062                )?]);
7063            }
7064        };
7065
7066        // A module that has already burned the whole budget `threshold` times
7067        // in a row does not get to charge it again until a probe says it recovered.
7068        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
7069            RouteBindAdmission::Admitted { guard, probe } => {
7070                if probe {
7071                    info!(
7072                        module_id = %target_module_id,
7073                        connection_id = ctx.connection_id.get(),
7074                        "route.bind breaker half-open: admitting one probe"
7075                    );
7076                }
7077                guard
7078            }
7079            RouteBindAdmission::Refused {
7080                consecutive_timeouts,
7081                retry_in,
7082                probe_in_flight,
7083            } => {
7084                return Ok(vec![self.route_open_breaker_refusal_frame(
7085                    ctx,
7086                    &caller,
7087                    &frame,
7088                    &target_module_id,
7089                    consecutive_timeouts,
7090                    retry_in,
7091                    probe_in_flight,
7092                )?]);
7093            }
7094        };
7095
7096        // Resolve the per-module budget here so the wait matches the operator's
7097        // intent for this specific target. A per-module override in
7098        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
7099        // daemons) wins over the daemon-wide default.
7100        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
7101        let relay_deadline = Instant::now() + route_bind_relay_timeout;
7102        let operator_app = self
7103            .operator_apps
7104            .lock()
7105            .unwrap_or_else(|p| p.into_inner())
7106            .get(&ctx.connection_id)
7107            .cloned();
7108        timings.enter(RouteOpenPhase::EgressReservation);
7109        let pending = match self
7110            .forwarding
7111            .begin_route_bind_relay_for(
7112                ctx.connection_id,
7113                ctx.egress.clone(),
7114                response_version(&frame),
7115                frame.header.corr,
7116                &target_module_id,
7117                principal.clone(),
7118                bound_scope,
7119                project_root,
7120                identity.harness.clone(),
7121                build_class,
7122                operator_app.clone(),
7123                crate::forwarding::RouteAuthority {
7124                    incarnation: self.supervisor.daemon_incarnation(),
7125                    caller: caller_capture
7126                        .as_ref()
7127                        .map(|caller| caller.authority.pair()),
7128                },
7129                relay_deadline,
7130                busy_refusals == Some(true),
7131                accepts_close_reasons,
7132            )
7133            .await
7134        {
7135            Ok(pending) => pending,
7136            Err(err) => {
7137                return Ok(vec![self.route_open_refusal_frame(
7138                    ctx,
7139                    &caller,
7140                    &frame,
7141                    &target_module_id,
7142                    "relay_reservation_failed",
7143                    forwarding_error_code(&err),
7144                    err.to_string(),
7145                )?])
7146            }
7147        };
7148        timings.enter(RouteOpenPhase::Admission);
7149        let crate::forwarding::PendingRouteBindRelay {
7150            endpoint,
7151            module_sink,
7152            negotiated_ver,
7153            client_channel,
7154            client_epoch,
7155            module_channel,
7156            module_epoch,
7157            corr: relay_corr,
7158            receiver,
7159        } = pending;
7160        let mut reservation =
7161            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
7162
7163        // Reserving egress can wait while a module reconnects or a swap cuts
7164        // over. Check the relay's captured connection, not the earlier by-id
7165        // lookup: the original module's flow/run capabilities cannot authorize
7166        // its replacement. The captured sink stays bound to that connection.
7167        let relay_registration = self
7168            .registry
7169            .get_module_by_connection(endpoint.connection_id)
7170            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7171        let relay_capabilities = relay_registration
7172            .as_ref()
7173            .and_then(|registration| registration.manifest.capabilities.as_ref());
7174        if let Some(selector) = &scope {
7175            // Egress reservation may wait across a scope update. Recheck the
7176            // proof against the current record before sending the captured bind.
7177            if let Err(refusal) = self
7178                .scopes
7179                .read()
7180                .unwrap_or_else(|poisoned| poisoned.into_inner())
7181                .check_bind_proof(&principal, selector)
7182            {
7183                reservation.release_and_disarm();
7184                return Ok(vec![self.route_open_refusal_frame(
7185                    ctx,
7186                    &caller,
7187                    &frame,
7188                    &target_module_id,
7189                    refusal.code,
7190                    refusal.code,
7191                    refusal.message,
7192                )?]);
7193            }
7194        }
7195        if let Some(stamp) = scope_stamp.as_ref() {
7196            if let Err(refusal) = crate::scopes::check_target_required_stamp(
7197                stamp_required,
7198                &target_module_id,
7199                relay_capabilities,
7200            )
7201            .and_then(|()| {
7202                crate::scopes::check_target_flow_support(
7203                    stamp,
7204                    &target_module_id,
7205                    relay_capabilities,
7206                )
7207            })
7208            .and_then(|()| {
7209                crate::scopes::check_target_agent_run_support(
7210                    stamp,
7211                    &target_module_id,
7212                    relay_capabilities,
7213                )
7214            }) {
7215                reservation.release_and_disarm();
7216                return Ok(vec![self.route_open_refusal_frame(
7217                    ctx,
7218                    &caller,
7219                    &frame,
7220                    &target_module_id,
7221                    refusal.code,
7222                    refusal.code,
7223                    refusal.message,
7224                )?]);
7225            }
7226        }
7227
7228        // Older readers may reject unknown fields. Select optional fields from
7229        // the captured endpoint's declaration, including on unscoped binds.
7230        if scope_stamp
7231            .as_ref()
7232            .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
7233            && !crate::scopes::target_supports_federated(relay_capabilities)
7234        {
7235            // Keep bound_scope: even an unstamped legacy route belongs to this epoch.
7236            scope_stamp = None;
7237        }
7238        if !relay_capabilities.is_some_and(|caps| {
7239            caps.provides
7240                .iter()
7241                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
7242        }) {
7243            if let Some(stamp) = scope_stamp.as_mut() {
7244                stamp.workdir = None;
7245                stamp.project_id = None;
7246            }
7247        }
7248        // A worker run has no agent_id; on_behalf_of_agent is its only agent
7249        // attribution. The run gate above has already required the target to
7250        // declare `worker-runs/v1`, which promises to read both
7251        // `on_behalf_of_agent` and `call_mode`.
7252        let worker_run = scope_stamp
7253            .as_ref()
7254            .is_some_and(crate::scopes::is_worker_run)
7255            && relay_capabilities.is_some_and(|caps| {
7256                caps.provides
7257                    .iter()
7258                    .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
7259            });
7260        if !worker_run
7261            && !relay_capabilities.is_some_and(|caps| {
7262                caps.provides
7263                    .iter()
7264                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
7265            })
7266        {
7267            if let Some(stamp) = scope_stamp.as_mut() {
7268                stamp.attributes.on_behalf_of_agent = None;
7269            }
7270        }
7271        if !relay_capabilities.is_some_and(|caps| {
7272            caps.provides
7273                .iter()
7274                .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
7275        }) {
7276            if let Some(stamp) = scope_stamp.as_mut() {
7277                stamp.attributes.flow_version = None;
7278                stamp.attributes.flow_code_hash = None;
7279                stamp.attributes.flow_install_id = None;
7280            }
7281        }
7282        let call_mode = if worker_run
7283            || relay_capabilities
7284                .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
7285        {
7286            Some(derive_call_mode(scope_stamp.as_ref()))
7287        } else {
7288            if let Some(stamp) = scope_stamp.as_mut() {
7289                stamp.expires_at_ms = None;
7290            }
7291            None
7292        };
7293
7294        debug!(
7295            connection_id = ctx.connection_id.get(),
7296            client_channel,
7297            client_epoch,
7298            module_channel,
7299            module_epoch,
7300            "reserved route handle pair"
7301        );
7302        // Rendered BEFORE the move into the relay, because the accept arm below
7303        // is where it is logged and the principal is gone by then.
7304        let route_principal_label = principal_label(&principal);
7305        let relay = ModuleControlRequest::RouteBind {
7306            route_channel: module_channel,
7307            epoch: module_epoch,
7308            target,
7309            identity,
7310            principal: Some(principal),
7311            consumer_capabilities,
7312            role_versions,
7313            admission_facts,
7314            scope: scope_stamp,
7315            call_mode,
7316        };
7317        let peer = RouteBindPeer::new()
7318            .with_observed_peer(
7319                relay_capabilities
7320                    .is_some_and(|caps| {
7321                        caps.provides
7322                            .iter()
7323                            .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
7324                    })
7325                    .then(|| {
7326                        self.observed_peers
7327                            .lock()
7328                            .unwrap_or_else(|p| p.into_inner())
7329                            .get(&ctx.connection_id)
7330                            .cloned()
7331                            .unwrap_or_else(ObservedPeer::unobserved)
7332                    }),
7333            )
7334            .with_operator_app(
7335                self.connection_operator_app(operator_app.as_ref())
7336                    .filter(|_| {
7337                        relay_capabilities.is_some_and(|caps| {
7338                            caps.provides
7339                                .iter()
7340                                .any(|cap| cap == OPERATOR_APP_CAPABILITY)
7341                        })
7342                    }),
7343            )
7344            .with_caller(caller_for_target(
7345                caller_capture.as_ref(),
7346                relay_capabilities,
7347                &target_module_id,
7348            ));
7349        let relay_body = crate::bind_compat::encode_bind(
7350            &relay,
7351            &peer,
7352            &legacy_bind_project,
7353            relay_capabilities,
7354        )
7355        .map_err(|err| {
7356            RouterError::backend(
7357                0,
7358                frame.header.corr,
7359                format!("failed to encode route.bind request: {err}"),
7360            )
7361        })?;
7362        let relay_frame = Frame::build_with_version(
7363            negotiated_ver,
7364            FrameType::Request,
7365            control_flags(),
7366            0,
7367            0,
7368            relay_corr,
7369            relay_body,
7370        )
7371        .map_err(RouterError::FrameBuild)?;
7372
7373        timings.enter(RouteOpenPhase::BindSend);
7374        let sent = module_sink.send(relay_frame).await;
7375        timings.enter(RouteOpenPhase::Admission);
7376        if let Err(err) = sent {
7377            reservation.release_and_disarm();
7378            return Ok(vec![self.route_open_refusal_frame(
7379                ctx,
7380                &caller,
7381                &frame,
7382                &target_module_id,
7383                "relay_send_failed",
7384                "target_unavailable",
7385                err.to_string(),
7386            )?]);
7387        }
7388
7389        #[cfg(test)]
7390        if let Some(hold) = &self.route_bind_relay_sent_hold {
7391            // First wait: tell the test the bind is queued. Second wait: stay
7392            // here until the test has driven whatever it wants to happen
7393            // between queueing the bind and marking it enqueued.
7394            hold.wait().await;
7395            hold.wait().await;
7396        }
7397
7398        // A missing relay is not by itself an abandoned bind: the module may
7399        // already have acked it and the route may be committed. Only a relay
7400        // dropped without any module reply needs this GOODBYE; one a reply
7401        // settled has already had its GOODBYE decided by the reply path.
7402        if self
7403            .forwarding
7404            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
7405            .map_err(RouterError::Forwarding)?
7406            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
7407        {
7408            self.send_abandoned_route_bind_goodbye(
7409                &module_sink,
7410                negotiated_ver,
7411                module_channel,
7412                module_epoch,
7413            );
7414        }
7415
7416        timings.enter(RouteOpenPhase::BindAckWait);
7417        let outcome = timeout_at(relay_deadline, receiver).await;
7418        timings.enter(RouteOpenPhase::Admission);
7419        match outcome {
7420            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
7421                reservation.disarm();
7422                if breaker.record_accepted() {
7423                    info!(
7424                        module_id = %target_module_id,
7425                        "route.bind breaker closed: the probe was accepted"
7426                    );
7427                }
7428                self.observe_route_open_accept(
7429                    ctx,
7430                    &target_module_id,
7431                    &route_principal_label,
7432                    &caller.scope,
7433                );
7434                Ok(Vec::new())
7435            }
7436            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
7437                reservation.release_and_disarm();
7438                // A module that says no in microseconds is healthy. Rejection
7439                // is a different condition with its own refusal and must not
7440                // move the breaker.
7441                breaker.record_inconclusive();
7442                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
7443            }
7444            Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
7445                reservation.release_and_disarm();
7446                breaker.record_inconclusive();
7447                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
7448            }
7449            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
7450                reservation.release_and_disarm();
7451                breaker.record_inconclusive();
7452                // Fires when the module's connection closes while a relayed
7453                // bind is pending -- typically a caller racing a module restart
7454                // whose bind was relayed BEFORE the drain mark went up. Logged
7455                // because the caller sees only its own error and the fleet has
7456                // already spent one diagnosis round unable to tell this arm
7457                // from a relay timeout without daemon-side evidence.
7458                tracing::warn!(
7459                    module_id = %target_module_id,
7460                    "route.bind relay abandoned: {message}"
7461                );
7462                Ok(vec![self.route_open_refusal_frame(
7463                    ctx,
7464                    &caller,
7465                    &frame,
7466                    &target_module_id,
7467                    "relay_abandoned",
7468                    "target_unavailable",
7469                    message,
7470                )?])
7471            }
7472            Ok(Err(_)) => {
7473                reservation.release_and_disarm();
7474                breaker.record_inconclusive();
7475                Ok(vec![self.route_open_refusal_frame(
7476                    ctx,
7477                    &caller,
7478                    &frame,
7479                    &target_module_id,
7480                    "relay_waiter_canceled",
7481                    "target_unavailable",
7482                    "route.bind relay waiter was canceled before the module responded",
7483                )?])
7484            }
7485            Err(_) => {
7486                reservation.release_and_disarm();
7487                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
7488                // answer at all is the one condition a fast refusal can
7489                // usefully stand in for; every other arm already answered.
7490                if let Some(opened) = breaker.record_timeout(
7491                    self.route_bind_breaker_threshold,
7492                    self.route_bind_breaker_cooldown,
7493                ) {
7494                    warn!(
7495                        module_id = %target_module_id,
7496                        consecutive_timeouts = opened.consecutive_timeouts,
7497                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
7498                        reopened_after_probe = opened.reopened_after_probe,
7499                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
7500                    );
7501                }
7502                // The generous budget just burned to no answer: the module is
7503                // registered and its connection is up, but its bind handler sat
7504                // on the ack for the full budget (warm-on-bind, cold configure,
7505                // or a wedged handler). Every earlier unavailability shape
7506                // fast-refuses BEFORE the relay, so this arm firing means the
7507                // slowness is module-side -- log it so the per-module timeline
7508                // is reconstructable without client audit rows.
7509                tracing::warn!(
7510                    module_id = %target_module_id,
7511                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
7512                    "route.bind relay timed out: module did not ack within budget"
7513                );
7514                Ok(vec![self.route_open_refusal_frame(
7515                    ctx,
7516                    &caller,
7517                    &frame,
7518                    &target_module_id,
7519                    "relay_timed_out",
7520                    "module_timeout",
7521                    format!(
7522                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
7523                        route_bind_relay_timeout
7524                    ),
7525                )?])
7526            }
7527        }
7528    }
7529
7530    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7531        let response = ClientControlResponse::SupervisorSpawnSnapshot {
7532            snapshot: self.supervisor.spawn_snapshot(),
7533        };
7534        Ok(vec![control_response_body_frame(
7535            &frame,
7536            &response,
7537            "ClientControlResponse::SupervisorSpawnSnapshot",
7538        )?])
7539    }
7540
7541    fn handle_supervisor_spawn_subscribe(
7542        &self,
7543        ctx: &RouteCtx,
7544        frame: Frame,
7545        since: Option<SpawnCursor>,
7546    ) -> Result<Vec<Frame>, RouterError> {
7547        match self.supervisor.subscribe_spawns(
7548            ctx.connection_id,
7549            frame.header.corr,
7550            response_version(&frame),
7551            since,
7552            ctx.egress.clone(),
7553        ) {
7554            Ok(()) => Ok(Vec::new()),
7555            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
7556                Ok(vec![control_error_body_frame(
7557                    &frame,
7558                    ErrorBody {
7559                        code: "spawn_cursor_incarnation_mismatch".to_string(),
7560                        message: "spawn cursor belongs to a different daemon incarnation"
7561                            .to_string(),
7562                        detail: Some(serde_json::json!({
7563                            "current_daemon_incarnation": current
7564                        })),
7565                    },
7566                )?])
7567            }
7568            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
7569                &frame,
7570                ErrorBody {
7571                    code: "spawn_cursor_too_old".to_string(),
7572                    message: "spawn cursor predates the retained event ring".to_string(),
7573                    detail: Some(serde_json::json!({
7574                        "oldest_retained_cursor": oldest
7575                    })),
7576                },
7577            )?]),
7578            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
7579                0,
7580                frame.header.corr,
7581                format!("failed to open supervisor spawn subscription: {error}"),
7582            )),
7583        }
7584    }
7585
7586    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7587        // Status/configuration snapshots may wait on synchronous supervisor
7588        // locks. Keep that work off the connection's async runtime thread.
7589        let handler = self.clone();
7590        let runtime = tokio::runtime::Handle::current();
7591        // The blocking pool does not inherit the control request's tracing
7592        // subscriber. Preserve it so slow snapshot-lock warnings stay visible.
7593        let subscriber = tracing::dispatcher::get_default(Clone::clone);
7594        let corr = frame.header.corr;
7595        tokio::task::spawn_blocking(move || {
7596            runtime.block_on(
7597                list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
7598                    .with_subscriber(subscriber),
7599            )
7600        })
7601        .await
7602        .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
7603    }
7604
7605    async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7606        use crate::config_gate::runtime::ApprovedDaemonView;
7607        let publication = self
7608            .config_runtime
7609            .as_ref()
7610            .and_then(|owner| owner.polled_snapshot().ok());
7611        let approved = publication
7612            .as_ref()
7613            .map(|view| view.approved_daemon.clone())
7614            .unwrap_or_else(|| {
7615                if self.config_runtime.is_some() {
7616                    ApprovedDaemonView::Unavailable
7617                } else {
7618                    ApprovedDaemonView::NotStamped
7619                }
7620            });
7621        let generation = self
7622            .registry
7623            .generation()
7624            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7625        let supervised = self.supervisor.list();
7626        let mut statuses = BTreeMap::new();
7627        let mut not_running = BTreeSet::new();
7628        for module in &supervised {
7629            let status = module.status_for_control("list").map_err(|err| {
7630                RouterError::backend(
7631                    0,
7632                    frame.header.corr,
7633                    format!("failed to read supervisor status: {err}"),
7634                )
7635            })?;
7636            if !status.enabled || !status.process_alive {
7637                not_running.insert(module.module_id().to_owned());
7638            }
7639            statuses.insert(module.module_id().to_owned(), status);
7640        }
7641        // Legacy per-process diagnostics keep their shared optional budget.
7642        // The cached tree and accounting export below does not spend it.
7643        let mut process_resources = BTreeMap::new();
7644        for module in &supervised {
7645            process_resources.insert(
7646                module.module_id().to_owned(),
7647                crate::subtree_resources::read_process(module.clone()).await,
7648            );
7649        }
7650        // Image evidence uses the same optional deadline, but an exhausted
7651        // budget must not suppress the committed resource view.
7652        let mut image_readings = BTreeMap::new();
7653        for module in &supervised {
7654            image_readings.insert(
7655                module.module_id().to_owned(),
7656                module.running_image_agreement().await,
7657            );
7658        }
7659        let readings = self.resource_tracker.list(supervised.clone());
7660        let mut modules = Vec::new();
7661        for module in supervised {
7662            let status = statuses
7663                .remove(module.module_id())
7664                .expect("each supervised module has a status snapshot");
7665            let (configured, _) = module.configuration().map_err(|err| {
7666                RouterError::backend(
7667                    0,
7668                    frame.header.corr,
7669                    format!("failed to read module configuration: {err}"),
7670                )
7671            })?;
7672            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
7673                RouterError::backend(
7674                    0,
7675                    frame.header.corr,
7676                    format!("failed to read retry time: {err}"),
7677                )
7678            })?;
7679            let image = image_readings
7680                .remove(module.module_id())
7681                .expect("each supervised module has an image snapshot");
7682            let resources = process_resources.remove(module.module_id());
7683            let mut verdict =
7684                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
7685            verdict.spec = Some(launch_spec_agreement(
7686                &configured,
7687                status.process_alive,
7688                status.launch_spec_digest.as_ref(),
7689            ));
7690            let pending_reload = Some(verdict);
7691            let script = configured.script.as_ref().map(|path| {
7692                let approved = self
7693                    .config_runtime
7694                    .as_ref()
7695                    .and_then(|owner| owner.approved_script(&status.module_id, path));
7696                crate::supervise::script::status(path, approved.as_ref())
7697            });
7698            let configured_enabled = match &approved {
7699                ApprovedDaemonView::Approved {
7700                    config: approved, ..
7701                } => approved
7702                    .modules
7703                    .iter()
7704                    .find(|module| module.module_id == status.module_id)
7705                    .map(|module| module.enabled),
7706                ApprovedDaemonView::NotStamped => self
7707                    .capability_evaluator
7708                    .configured_enabled(&status.module_id),
7709                ApprovedDaemonView::Unavailable => None,
7710            };
7711            let admission_refused = status
7712                .admission_refused
7713                .clone()
7714                .filter(|_| status.enabled && !status.process_alive)
7715                .filter(|_| {
7716                    publication
7717                        .as_ref()
7718                        .and_then(|view| view.registration_declaration(&status.module_id))
7719                        .as_deref()
7720                        == Some("undeclared")
7721                });
7722            // Keep the retired policy field on the wire for one release so
7723            // existing status consumers still receive the platform policy.
7724            modules.push(
7725                SupervisorEntry::new(
7726                    status.module_id,
7727                    status.state.to_string(),
7728                    status.enabled,
7729                    status.live,
7730                    status.health.status,
7731                )
7732                .with_configured_enabled(configured_enabled)
7733                .with_script(script)
7734                .with_admission_refused(admission_refused)
7735                .with_next_retry_at_ms(next_retry_at_ms)
7736                .with_launch_nonce_env(Some(!cfg!(unix)))
7737                .with_launch_nonce_source(status.launch_nonce_source)
7738                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
7739                .with_protocol(status.protocol)
7740                .with_pending_reload(pending_reload)
7741                .with_last_probe_ms(status.health.last_probe_ms)
7742                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
7743                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
7744                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
7745                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
7746                .with_restart_count(Some(status.restart_count))
7747                .with_max_restarts(Some(status.max_restarts))
7748                .with_lifetime_restarts(Some(status.lifetime_restarts))
7749                .with_spawn_generation(Some(status.spawn_generation))
7750                .with_restart_window_secs(Some(status.restart_window.as_secs()))
7751                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
7752                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
7753                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
7754                .with_resources(resources),
7755            );
7756        }
7757        for module in &mut modules {
7758            if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
7759                module.tree = tree.clone();
7760                module.cpu_last_hour = hour.clone();
7761            }
7762            module.tree_unavailable = readings
7763                .reasons
7764                .get(&module.module_id)
7765                .map(|reason| (*reason).to_owned());
7766            if module.tree.is_none() {
7767                module.tree_unavailable = Some(
7768                    if not_running.contains(&module.module_id) {
7769                        "not_running"
7770                    } else if let Some(reason) = readings.reasons.get(&module.module_id) {
7771                        reason
7772                    } else {
7773                        "not_yet_measured"
7774                    }
7775                    .to_owned(),
7776                );
7777            }
7778        }
7779        let response = ClientControlResponse::SupervisorList {
7780            generation,
7781            modules,
7782            daemon: readings.daemon,
7783            agent_commands: readings.agent_commands,
7784        };
7785        Ok(vec![control_response_body_frame(
7786            &frame,
7787            &response,
7788            "ClientControlResponse::SupervisorList",
7789        )?])
7790    }
7791
7792    fn handle_supervisor_stderr_tail(
7793        &self,
7794        frame: Frame,
7795        module_id: String,
7796        max_lines: Option<u32>,
7797        max_bytes: Option<u32>,
7798    ) -> Result<Vec<Frame>, RouterError> {
7799        let Some(module) = self.supervisor.get(&module_id) else {
7800            return Ok(vec![control_error_frame(
7801                &frame,
7802                "unknown_module",
7803                format!("module_id '{module_id}' is not supervised"),
7804            )?]);
7805        };
7806
7807        let snapshot = module.stderr_tail(
7808            max_lines.map(|value| value as usize),
7809            max_bytes.map(|value| value as usize),
7810        );
7811
7812        let response = ClientControlResponse::SupervisorStderrTail {
7813            module_id,
7814            tail: StderrTail {
7815                capture: match snapshot.capture {
7816                    CaptureState::Captured => StderrCaptureState::Captured,
7817                    CaptureState::Incomplete { reason } => {
7818                        StderrCaptureState::Incomplete { reason }
7819                    }
7820                    CaptureState::NotCaptured { reason } => {
7821                        StderrCaptureState::NotCaptured { reason }
7822                    }
7823                },
7824                entries: snapshot
7825                    .entries
7826                    .into_iter()
7827                    .map(|entry| match entry {
7828                        TailEntry::Line {
7829                            text,
7830                            truncated,
7831                            at_ms,
7832                        } => StderrTailEntry::Line {
7833                            text,
7834                            truncated,
7835                            at_ms,
7836                        },
7837                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
7838                    })
7839                    .collect(),
7840                dropped_lines: snapshot.dropped_lines,
7841            },
7842        };
7843        Ok(vec![control_response_body_frame(
7844            &frame,
7845            &response,
7846            "ClientControlResponse::SupervisorStderrTail",
7847        )?])
7848    }
7849
7850    async fn observe_control_wait<F: std::future::Future>(
7851        &self,
7852        ctx: &RouteCtx,
7853        target: Option<&str>,
7854        kind: &'static str,
7855        future: F,
7856    ) -> F::Output {
7857        self.counters
7858            .observe_reader_wait(
7859                || {
7860                    let module = self
7861                        .forwarding
7862                        .module_id_for_connection(ctx.connection_id)
7863                        .ok()
7864                        .flatten();
7865                    let principal = if module.is_some() { "module" } else { "direct" };
7866                    (
7867                        self.resolve_reader_wait_target(kind, target)
7868                            .or(module)
7869                            .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
7870                        principal.into(),
7871                    )
7872                },
7873                kind,
7874                future,
7875            )
7876            .await
7877    }
7878
7879    /// Resolve the reader-wait module ID only from supervisor or registry
7880    /// records; never attribute a wait from the request selector alone.
7881    pub(crate) fn resolve_reader_wait_target(
7882        &self,
7883        op: &str,
7884        selector: Option<&str>,
7885    ) -> Option<String> {
7886        let selector = selector?;
7887        match op {
7888            ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
7889                .supervisor
7890                .get(selector)
7891                .map(|module| module.module_id().to_owned()),
7892            ops::SUPERVISOR_HEALTH_PROBE => self
7893                .registry
7894                .get_module(selector)
7895                .ok()
7896                .flatten()
7897                .map(|registration| registration.manifest.module_id),
7898            _ => None,
7899        }
7900    }
7901
7902    async fn handle_supervisor_terminals(
7903        &self,
7904        ctx: &RouteCtx,
7905        frame: Frame,
7906        module_id: String,
7907    ) -> Result<Vec<Frame>, RouterError> {
7908        let Some(module) = self.supervisor.get(&module_id) else {
7909            return Ok(vec![control_error_frame(
7910                &frame,
7911                "unknown_module",
7912                format!("module_id '{module_id}' is not supervised"),
7913            )?]);
7914        };
7915
7916        // The journal read runs on a blocking thread: it can be megabytes of
7917        // file I/O and must not occupy a runtime worker.
7918        let terminals = self
7919            .observe_control_wait(
7920                ctx,
7921                Some(module.module_id()),
7922                ops::SUPERVISOR_TERMINALS,
7923                module.read_durable_terminal_history(),
7924            )
7925            .await
7926            .map_err(|error| {
7927                RouterError::backend(
7928                    0,
7929                    frame.header.corr,
7930                    format!("failed to read terminal history: {error}"),
7931                )
7932            })?;
7933        let response = ClientControlResponse::SupervisorTerminals {
7934            module_id,
7935            terminals,
7936        };
7937        Ok(vec![control_response_body_frame(
7938            &frame,
7939            &response,
7940            "ClientControlResponse::SupervisorTerminals",
7941        )?])
7942    }
7943
7944    fn handle_supervisor_routes(
7945        &self,
7946        frame: Frame,
7947        module_id: Option<String>,
7948    ) -> Result<Vec<Frame>, RouterError> {
7949        let modules = self
7950            .forwarding
7951            .route_census(module_id.as_deref())
7952            .map_err(RouterError::Forwarding)?
7953            .into_iter()
7954            .map(|(module_id, routes)| SupervisorRouteModule {
7955                module_id,
7956                routes: routes
7957                    .into_iter()
7958                    .map(|route| SupervisorRoute {
7959                        consumer: match route.principal {
7960                            Principal::Reserved { module_id } => {
7961                                SupervisorRouteConsumer::Reserved { module_id }
7962                            }
7963                            Principal::Direct | Principal::Unverified => {
7964                                SupervisorRouteConsumer::Direct {
7965                                    connection_id: route.goodbye_target.connection_id.get(),
7966                                }
7967                            }
7968                        },
7969                        age_ms: Instant::now()
7970                            .saturating_duration_since(route.bound_at)
7971                            .as_millis()
7972                            .try_into()
7973                            .unwrap_or(u64::MAX),
7974                        draining: route.draining,
7975                        drain_reason: route.drain_reason,
7976                    })
7977                    .collect(),
7978            })
7979            .collect();
7980        let response = ClientControlResponse::SupervisorRoutes { modules };
7981        Ok(vec![control_response_body_frame(
7982            &frame,
7983            &response,
7984            "ClientControlResponse::SupervisorRoutes",
7985        )?])
7986    }
7987
7988    fn running_registration_method(
7989        &self,
7990        module: &str,
7991    ) -> Option<subc_control::RegistrationMethod> {
7992        let registration = self.registry.get_module(module).ok()??;
7993        let connection = registration.connection_id;
7994        let channel = self.supervisor.registration_spawn(connection).is_some();
7995        let nonce = if channel {
7996            None
7997        } else {
7998            self.hello_launch_nonces
7999                .lock()
8000                .ok()?
8001                .nonce(connection)
8002                .map(str::to_owned)
8003        };
8004        if !channel && nonce.is_none() {
8005            return None;
8006        }
8007        self.supervisor
8008            .registration_snapshot_is_current_live(
8009                &self.registry,
8010                module,
8011                connection,
8012                nonce.as_deref(),
8013            )
8014            .then_some(if channel {
8015                subc_control::RegistrationMethod::Channel
8016            } else {
8017                subc_control::RegistrationMethod::Nonce
8018            })
8019    }
8020
8021    async fn handle_supervisor_provenance(
8022        &self,
8023        ctx: &RouteCtx,
8024        frame: Frame,
8025        module_id: Option<String>,
8026    ) -> Result<Vec<Frame>, RouterError> {
8027        let targeted = module_id.is_some();
8028        let selected = if let Some(module_id) = module_id {
8029            let Some(module) = self.supervisor.get(&module_id) else {
8030                return Ok(vec![control_error_frame(
8031                    &frame,
8032                    "unknown_module",
8033                    format!("module_id '{module_id}' is not supervised"),
8034                )?]);
8035            };
8036            vec![module]
8037        } else {
8038            self.supervisor.list()
8039        };
8040
8041        let target = targeted.then(|| selected[0].module_id());
8042        let declarations = if let Some(owner) = &self.config_runtime {
8043            owner
8044                .on_blocking(|owner| owner.registration_declarations())
8045                .await
8046                .ok()
8047        } else {
8048            None
8049        };
8050        self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
8051            let mut modules = Vec::with_capacity(selected.len());
8052            let mut extensions = Vec::with_capacity(selected.len());
8053            for module in &selected {
8054                let (status, observed_image) = module
8055                    .status_and_running_image_agreement()
8056                    .await
8057                    .map_err(|err| {
8058                        RouterError::backend(
8059                            0,
8060                            frame.header.corr,
8061                            format!("failed to read supervisor status: {err}"),
8062                        )
8063                    })?;
8064                let module_declared = self
8065                    .registry
8066                    .get_module(&status.module_id)
8067                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
8068                    .and_then(|registration| registration.manifest.provenance)
8069                    .map(|build| ModuleDeclaredProvenance::Reported { build })
8070                    .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
8071                #[cfg(test)]
8072                let running_image = match &self.provenance_probe_override {
8073                    Some(result) => result.clone(),
8074                    None => observed_image,
8075                };
8076                #[cfg(not(test))]
8077                let running_image = observed_image;
8078                let registration = self.running_registration_method(&status.module_id);
8079                let mut extension = subc_control::SupervisorModuleProvenanceExt::new()
8080                    .with_registration(registration);
8081                extension.registration_declaration = declarations
8082                    .as_ref()
8083                    .and_then(|entries| entries.get(&status.module_id))
8084                    .cloned();
8085                if status.process_alive && status.registration_declaration.is_some() {
8086                    extension.registration_declaration = status.registration_declaration.clone();
8087                }
8088                extensions.push(extension);
8089                modules.push(SupervisorModuleProvenance {
8090                    module_id: status.module_id,
8091                    module_declared,
8092                    daemon_observed: SupervisorObservedProcess {
8093                        launch_nonce_source: status.launch_nonce_source,
8094                        pid: status.pid,
8095                        spawned_at_ms: status.spawned_at_ms,
8096                        spawned_from: status.spawned_from,
8097                        running_image,
8098                    },
8099                });
8100            }
8101            let daemon = SupervisorDaemonProvenance {
8102                daemon_build: self.daemon_provenance.build.clone(),
8103                daemon_observed: DaemonObservedProcess {
8104                    pid: self.daemon_provenance.pid,
8105                    started_at_ms: self
8106                        .daemon_provenance
8107                        .start_clock
8108                        .map(|clock| clock.started_at_ms())
8109                        .or(self.daemon_provenance.started_at_ms),
8110                    running_image: self
8111                        .daemon_provenance
8112                        .probe
8113                        .observe(
8114                            self.daemon_provenance.pid,
8115                            self.daemon_provenance.executable_path.as_deref(),
8116                            self.daemon_provenance.executable_identity.clone(),
8117                            self.daemon_provenance.process_start_time,
8118                        )
8119                        .await,
8120                },
8121            };
8122            let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
8123            let mut response = serde_json::to_value(response)
8124                .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
8125            for (entry, extension) in response["modules"]
8126                .as_array_mut()
8127                .expect("provenance modules are an array")
8128                .iter_mut()
8129                .zip(extensions)
8130            {
8131                extension.write_to(entry).map_err(|error| {
8132                    RouterError::backend(0, frame.header.corr, error.to_string())
8133                })?;
8134            }
8135            Ok(vec![control_response_body_frame(
8136                &frame,
8137                &response,
8138                "ClientControlResponse::SupervisorProvenance",
8139            )?])
8140        })
8141        .await
8142    }
8143
8144    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
8145        self.refresh_capability_requirements();
8146        let publication = self
8147            .config_runtime
8148            .as_ref()
8149            .and_then(|owner| owner.polled_snapshot().ok());
8150        let generation = self
8151            .registry
8152            .generation()
8153            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
8154        let modules = self
8155            .supervisor
8156            .list()
8157            .into_iter()
8158            .map(|module| {
8159                let status = module.status_for_control("health").map_err(|err| {
8160                    RouterError::backend(
8161                        0,
8162                        frame.header.corr,
8163                        format!("failed to read supervisor health: {err}"),
8164                    )
8165                })?;
8166                let module_id = status.module_id;
8167                let capability_detail = self
8168                    .capability_evaluator
8169                    .required_problem_detail(&module_id);
8170                let declaration_warning = self.registry.get_module(&module_id).ok().flatten()
8171                    .filter(|registration| registration.manifest.registration_channel == Some(false))
8172                    .filter(|_| publication.as_ref().and_then(|view| view.registration_declaration(&module_id))
8173                        .as_deref().map(|next| if status.process_alive { status.registration_declaration.as_deref().unwrap_or(next) } else { next }) == Some("declared"))
8174                    .map(|_| format!("{module_id}: HELLO registration_channel is false despite the approved channel declaration"));
8175                let capability_detail = append_capability_problem_detail(capability_detail, declaration_warning);
8176                Ok(SupervisorHealthEntry {
8177                    module_id,
8178                    status: status.health.status,
8179                    detail: append_capability_problem_detail(
8180                        status.health.detail,
8181                        capability_detail,
8182                    ),
8183                    metrics: status.health.metrics,
8184                    consecutive_failures: status.health.consecutive_failures,
8185                    late_answer_count: status.health.late_answer_count,
8186                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
8187                    last_action: status.health.last_action,
8188                    last_action_ms: status.health.last_action_ms,
8189                    last_probe_ms: status.health.last_probe_ms,
8190                })
8191            })
8192            .collect::<Result<Vec<_>, RouterError>>()?;
8193        let response = ClientControlResponse::SupervisorHealth {
8194            generation,
8195            modules,
8196        };
8197        Ok(vec![control_response_body_frame(
8198            &frame,
8199            &response,
8200            "ClientControlResponse::SupervisorHealth",
8201        )?])
8202    }
8203
8204    async fn handle_supervisor_restart(
8205        &self,
8206        frame: Frame,
8207        module_id: String,
8208        drain_timeout_ms: Option<u64>,
8209        requester: Principal,
8210    ) -> Result<Vec<Frame>, RouterError> {
8211        let operation_lock = self.supervisor.operation_lock();
8212        let _operation_guard = operation_lock.lock().await;
8213        let Some(module) = self.supervisor.get(&module_id) else {
8214            return Ok(vec![control_error_frame(
8215                &frame,
8216                "unknown_module",
8217                format!("module_id '{module_id}' is not supervised"),
8218            )?]);
8219        };
8220
8221        self.route_outages.mark_operator_action(&module_id);
8222        if let Err(err) = module
8223            .restart_requested(drain_timeout_ms, Some(requester))
8224            .await
8225        {
8226            self.route_outages
8227                .operator_action_ended_unrefused(&module_id);
8228            let (code, message) = match err {
8229                _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8230                crate::supervise::SuperviseError::Disabled { .. } => {
8231                    ("module_disabled", err.to_string())
8232                }
8233                crate::supervise::SuperviseError::SwapInProgress { .. } => {
8234                    ("swap_in_progress", err.to_string())
8235                }
8236                _ => (
8237                    "target_unavailable",
8238                    format!("failed to restart module_id '{module_id}': {err}"),
8239                ),
8240            };
8241            return Ok(vec![control_error_frame(&frame, code, message)?]);
8242        }
8243
8244        let response = ClientControlResponse::SupervisorAck {
8245            module_id,
8246            applied: true,
8247        };
8248        Ok(vec![control_response_body_frame(
8249            &frame,
8250            &response,
8251            "ClientControlResponse::SupervisorAck",
8252        )?])
8253    }
8254
8255    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
8256    /// when the old process has finished draining: a caller whose own lane
8257    /// rides the old process must get its reply before that drain waits on it.
8258    async fn handle_supervisor_swap(
8259        &self,
8260        frame: Frame,
8261        module_id: String,
8262        ready_timeout_ms: Option<u64>,
8263        requester: Principal,
8264    ) -> Result<Vec<Frame>, RouterError> {
8265        // The daemon-wide operation lock is held only to resolve the handle,
8266        // not across the swap. The swap can take its whole readiness budget,
8267        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
8268        // holding it here would park an operator's stop behind the swap it is
8269        // meant to abort. A rescan or stop that reaches the module during the
8270        // swap is served by the swap itself (see `supervise_swap`).
8271        let module = {
8272            let operation_lock = self.supervisor.operation_lock();
8273            let _operation_guard = operation_lock.lock().await;
8274            self.supervisor.get(&module_id)
8275        };
8276        let Some(module) = module else {
8277            return Ok(vec![control_error_frame(
8278                &frame,
8279                "unknown_module",
8280                format!("module_id '{module_id}' is not supervised"),
8281            )?]);
8282        };
8283
8284        self.route_outages.mark_operator_action(&module_id);
8285        if let Err(err) = module
8286            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
8287            .await
8288        {
8289            self.route_outages
8290                .operator_action_ended_unrefused(&module_id);
8291            use crate::supervise::SuperviseError;
8292            let message = err.to_string();
8293            let error = match err {
8294                SuperviseError::ConfigHeld { .. } => ErrorBody::new("config_gated", message),
8295                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
8296                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
8297                    code: "swap_refused".to_string(),
8298                    message,
8299                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
8300                },
8301                SuperviseError::SwapFailed {
8302                    arm,
8303                    candidate_exit,
8304                    ..
8305                } => ErrorBody {
8306                    code: "swap_failed".to_string(),
8307                    message,
8308                    detail: Some(serde_json::json!({
8309                        "arm": arm.as_str(),
8310                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
8311                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
8312                    })),
8313                },
8314                _ => ErrorBody::new(
8315                    "target_unavailable",
8316                    format!("failed to swap module_id '{module_id}': {message}"),
8317                ),
8318            };
8319            return Ok(vec![control_error_body_frame(&frame, error)?]);
8320        }
8321        // A completed swap kept the incumbent serving until cutover, so it
8322        // usually opened no outage; a mark left behind would make the next,
8323        // unrelated outage read as requested.
8324        self.route_outages
8325            .operator_action_ended_unrefused(&module_id);
8326
8327        let response = ClientControlResponse::SupervisorAck {
8328            module_id,
8329            applied: true,
8330        };
8331        Ok(vec![control_response_body_frame(
8332            &frame,
8333            &response,
8334            "ClientControlResponse::SupervisorAck",
8335        )?])
8336    }
8337
8338    async fn handle_supervisor_reload(
8339        &self,
8340        frame: Frame,
8341        module_id: String,
8342        requester: Principal,
8343    ) -> Result<Vec<Frame>, RouterError> {
8344        let operation_lock = self.supervisor.operation_lock();
8345        let _operation_guard = operation_lock.lock().await;
8346        let Some(module) = self.supervisor.get(&module_id) else {
8347            return Ok(vec![control_error_frame(
8348                &frame,
8349                "unknown_module",
8350                format!("module_id '{module_id}' is not supervised"),
8351            )?]);
8352        };
8353
8354        self.route_outages.mark_operator_action(&module_id);
8355        if let Err(err) = module.reload_requested(Some(requester)).await {
8356            self.route_outages
8357                .operator_action_ended_unrefused(&module_id);
8358            let (code, message) = match err {
8359                _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8360                crate::supervise::SuperviseError::Disabled { .. } => {
8361                    ("module_disabled", err.to_string())
8362                }
8363                crate::supervise::SuperviseError::SwapInProgress { .. } => {
8364                    ("swap_in_progress", err.to_string())
8365                }
8366                _ => (
8367                    "reload_failed",
8368                    format!("failed to reload module_id '{module_id}': {err}"),
8369                ),
8370            };
8371            return Ok(vec![control_error_frame(&frame, code, message)?]);
8372        }
8373
8374        let response = ClientControlResponse::SupervisorAck {
8375            module_id,
8376            applied: true,
8377        };
8378        Ok(vec![control_response_body_frame(
8379            &frame,
8380            &response,
8381            "ClientControlResponse::SupervisorAck",
8382        )?])
8383    }
8384
8385    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
8386    /// the configuration comes from and whether to apply it; every mode then
8387    /// runs the same comparison, so a preview describes the apply it previews.
8388    async fn handle_supervisor_rescan(
8389        &self,
8390        frame: Frame,
8391        mode: RescanMode,
8392        requester: Principal,
8393    ) -> Result<Vec<Frame>, RouterError> {
8394        let preview = mode.previews();
8395        let mut approved_preview = None;
8396        if matches!(mode, RescanMode::Preview) {
8397            if let Some(owner) = &self.config_runtime {
8398                match owner
8399                    .on_blocking(|owner| {
8400                        owner.observe_file_proposal()?;
8401                        owner.approved_daemon_configuration()
8402                    })
8403                    .await
8404                {
8405                    Ok(config) => approved_preview = config,
8406                    Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8407                }
8408            }
8409        }
8410        let Some(context) = self.rescan.clone() else {
8411            return Ok(vec![control_error_frame(
8412                &frame,
8413                "rescan_unavailable",
8414                "the daemon was not started with a reloadable config path".to_string(),
8415            )?]);
8416        };
8417
8418        let operation_lock = self.supervisor.operation_lock();
8419        let _operation_guard = operation_lock.lock().await;
8420        // Loaded under the operation lock, so no other rescan can change the
8421        // staged configuration between a checked apply's comparison and the
8422        // apply itself.
8423        let loaded = match self.load_rescan_config(&mode, &context) {
8424            Ok(config) => config,
8425            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
8426        };
8427        // `load` reports a missing file as Ok(None), which is correct at boot
8428        // (no config, nothing to supervise) and catastrophic here: rescan treats
8429        // "not in the config" as "remove it", so an absent file would read as an
8430        // empty module list and retire the entire running fleet. An editor
8431        // writing via write-new-then-rename, or a half-finished edit, is enough
8432        // to open that window. Refuse instead: a config that cannot be read
8433        // carries no instruction to remove anything.
8434        let Some((config, activity_config)) = loaded else {
8435            return Ok(vec![control_error_frame(
8436                &frame,
8437                "invalid_daemon_config",
8438                format!(
8439                    "daemon config not found at {}; refusing to rescan (an absent config would \
8440                     retire every supervised module)",
8441                    context.config_path.display()
8442                ),
8443            )?]);
8444        };
8445        let (
8446            configured_port,
8447            storage_config,
8448            admission_facts_carrier_module_id,
8449            admission_facts_targets,
8450            scope_authority_owners,
8451            federated_scope_owner,
8452            privileged_operator_prompt,
8453            modules,
8454            reserved_capabilities,
8455        ) = (
8456            config.port,
8457            config.storage,
8458            config.admission_facts_carrier_module_id,
8459            config.admission_facts_targets,
8460            config.scope_authority_owners,
8461            config.federated_scope_owner,
8462            config.privileged_operator_prompt,
8463            config.modules,
8464            config.reserved_capabilities,
8465        );
8466
8467        // Collect the sections rescan cannot apply, so the REPLY carries them.
8468        //
8469        // The warning below has always been correct and has always gone only to
8470        // the journal -- addressed to whoever reads logs, while the person who
8471        // just edited the config is looking at the CLI. Naming each section
8472        // individually rather than setting a flag: "something outside modules
8473        // changed" sends the operator back to diffing their own file, which is
8474        // the work this is meant to save.
8475        let mut restart_required = Vec::new();
8476        for section in RestartRequiredSection::ALL {
8477            let changed = match section {
8478                RestartRequiredSection::Port => configured_port != context.configured_port,
8479                RestartRequiredSection::Storage => storage_config != context.storage_config,
8480                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
8481                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
8482                }
8483                RestartRequiredSection::AdmissionFactsTargets => {
8484                    admission_facts_targets != context.admission_facts_targets
8485                }
8486                RestartRequiredSection::ScopeAuthorityOwners => {
8487                    scope_authority_owners != context.scope_authority_owners
8488                }
8489                RestartRequiredSection::FederatedScopeOwner => {
8490                    federated_scope_owner != context.federated_scope_owner
8491                }
8492                RestartRequiredSection::PrivilegedOperatorPrompt => {
8493                    privileged_operator_prompt != context.privileged_operator_prompt
8494                }
8495            };
8496            if changed {
8497                restart_required.push(section.label().to_string());
8498            }
8499        }
8500        if !restart_required.is_empty() {
8501            warn!(
8502                config_path = %context.config_path.display(),
8503                sections = %restart_required.join(", "),
8504                "daemon config changed outside the modules section; restart the daemon to apply those changes"
8505            );
8506        }
8507
8508        for configured in &modules {
8509            if let Err(err) = validate_spec(&configured.module_spec()) {
8510                return Ok(vec![control_error_frame(
8511                    &frame,
8512                    "invalid_daemon_config",
8513                    format!("supervisor rescan rejected daemon config: {err}"),
8514                )?]);
8515            }
8516        }
8517
8518        let configured_capabilities = modules
8519            .iter()
8520            .map(|module| (module.module_id.clone(), module.enabled))
8521            .collect::<Vec<_>>();
8522        let preview_capability_warnings = if preview {
8523            let (_, registrations) = self.runtime_capability_snapshot()?;
8524            let current_modules = self
8525                .supervisor
8526                .list()
8527                .into_iter()
8528                .map(|module| module.module_id().to_string())
8529                .collect::<BTreeSet<_>>();
8530            let resulting_modules = configured_capabilities.clone();
8531            let removed = current_modules
8532                .into_iter()
8533                .filter(|module_id| {
8534                    !resulting_modules
8535                        .iter()
8536                        .any(|(configured_id, _)| configured_id == module_id)
8537                })
8538                .collect::<Vec<_>>();
8539            // A candidate preview also names providers the candidate disables.
8540            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
8541            let disabled = match &mode {
8542                RescanMode::Candidate(_) => resulting_modules
8543                    .iter()
8544                    .filter(|(_, enabled)| !enabled)
8545                    .map(|(module_id, _)| module_id.clone())
8546                    .collect::<Vec<_>>(),
8547                _ => Vec::new(),
8548            };
8549            self.capability_evaluator.preview_warnings(
8550                resulting_modules,
8551                &removed,
8552                &disabled,
8553                &registrations,
8554            )
8555        } else {
8556            Vec::new()
8557        };
8558        let reconciled = match (&mode, approved_preview.as_ref()) {
8559            (RescanMode::Preview, Some(approved)) => Ok((
8560                config_gate::preview_approved_daemon(approved, &modules),
8561                Vec::new(),
8562            )),
8563            (RescanMode::Candidate(candidate), _) => self
8564                .preview_candidate(&context, modules, candidate)
8565                .map(|result| (result, Vec::new())),
8566            _ => {
8567                self.reconcile_supervised_modules(
8568                    &context.supervisor,
8569                    modules,
8570                    preview,
8571                    requester.clone(),
8572                )
8573                .await
8574            }
8575        };
8576        let (result, changed) = match reconciled {
8577            Ok(result) => result,
8578            Err(message) => {
8579                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
8580            }
8581        };
8582        if !preview {
8583            if let Some(ledger) = &self.activity {
8584                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
8585                ledger.config_changed(&result.added, &result.removed, &changed, requester);
8586            }
8587            if let Err(code) = self.configure_activity(activity_config).await {
8588                warn!(
8589                    code,
8590                    "activity retention could not be applied during rescan"
8591                );
8592            }
8593            self.capability_evaluator
8594                .configure(configured_capabilities, reserved_capabilities);
8595            self.capability_evaluator.wake_deadline_loop();
8596            self.refresh_capability_requirements();
8597        }
8598        let mut result = result;
8599        result.restart_required = restart_required;
8600        result.capability_warnings = preview_capability_warnings;
8601        let response = ClientControlResponse::SupervisorRescan { result };
8602        Ok(vec![control_response_body_frame(
8603            &frame,
8604            &response,
8605            "ClientControlResponse::SupervisorRescan",
8606        )?])
8607    }
8608
8609    async fn handle_supervisor_release_reserved(
8610        &self,
8611        frame: Frame,
8612        module_id: String,
8613    ) -> Result<Vec<Frame>, RouterError> {
8614        let Some(context) = self.rescan.clone() else {
8615            return Ok(vec![control_error_frame(
8616                &frame,
8617                "release_unavailable",
8618                "reserved-id release requires a daemon started with a reloadable config path",
8619            )?]);
8620        };
8621        let operation_lock = self.supervisor.operation_lock();
8622        let _operation_guard = operation_lock.lock().await;
8623        let approved = if let Some(owner) = &self.config_runtime {
8624            // Observe the file as a proposal, but decide membership solely from
8625            // approved bytes. A proposed re-add must not retain a removed id.
8626            match owner
8627                .on_blocking(|owner| {
8628                    owner.observe_file_proposal()?;
8629                    owner.approved_daemon_configuration()
8630                })
8631                .await
8632            {
8633                Ok(config) => config,
8634                Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8635            }
8636        } else {
8637            None
8638        };
8639        let loaded = if let Some(approved) = approved {
8640            Ok(Some(approved))
8641        } else {
8642            crate::daemon_config::load(&context.config_path)
8643        };
8644        let loaded = match loaded {
8645            Ok(Some(config)) => config,
8646            Ok(None) => {
8647                return Ok(vec![control_error_frame(
8648                    &frame,
8649                    "invalid_daemon_config",
8650                    format!(
8651                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
8652                        context.config_path.display()
8653                    ),
8654                )?])
8655            }
8656            Err(err) => {
8657                return Ok(vec![control_error_frame(
8658                    &frame,
8659                    "invalid_daemon_config",
8660                    format!("unable to verify reserved-id release against daemon config: {err}"),
8661                )?])
8662            }
8663        };
8664        if loaded
8665            .modules
8666            .iter()
8667            .any(|configured| configured.module_id == module_id)
8668        {
8669            return Ok(vec![control_error_frame(
8670                &frame,
8671                "reserved_module_configured",
8672                format!(
8673                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
8674                ),
8675            )?]);
8676        }
8677        if !self.supervisor.release_retained_reserved_gate(&module_id) {
8678            return Ok(vec![control_error_frame(
8679                &frame,
8680                "reserved_gate_not_retained",
8681                format!(
8682                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
8683                ),
8684            )?]);
8685        }
8686
8687        let response = ClientControlResponse::SupervisorAck {
8688            module_id,
8689            applied: true,
8690        };
8691        Ok(vec![control_response_body_frame(
8692            &frame,
8693            &response,
8694            "ClientControlResponse::SupervisorAck",
8695        )?])
8696    }
8697
8698    /// Compare the configured module set with the supervised one, changing
8699    /// nothing. The staged spec each supervised module holds is what the
8700    /// configured entry is compared with.
8701    fn plan_rescan(
8702        &self,
8703        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8704    ) -> Result<RescanPlan, String> {
8705        let mut current = BTreeMap::new();
8706        for module in self.supervisor.list() {
8707            let (spec, health) = module.configuration().map_err(|err| {
8708                format!(
8709                    "failed to read configuration for module_id '{}': {err}",
8710                    module.module_id()
8711                )
8712            })?;
8713            let enabled = module
8714                .status()
8715                .map_err(|err| {
8716                    format!(
8717                        "failed to read status for module_id '{}': {err}",
8718                        module.module_id()
8719                    )
8720                })?
8721                .enabled;
8722            current.insert(
8723                module.module_id().to_string(),
8724                (module, spec, health, enabled),
8725            );
8726        }
8727        let configured = configured_modules
8728            .into_iter()
8729            .map(|module| (module.module_id.clone(), module))
8730            .collect::<BTreeMap<_, _>>();
8731
8732        let added = configured
8733            .keys()
8734            .filter(|module_id| !current.contains_key(*module_id))
8735            .cloned()
8736            .collect::<Vec<_>>();
8737        let removed = current
8738            .keys()
8739            .filter(|module_id| !configured.contains_key(*module_id))
8740            .cloned()
8741            .collect::<Vec<_>>();
8742        let mut changed_pending_reload = Vec::new();
8743        let mut configuration_changes = BTreeSet::new();
8744        let mut enabled_changes = BTreeSet::new();
8745        let mut unchanged = 0_u32;
8746
8747        for (module_id, configured_module) in &configured {
8748            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
8749            else {
8750                continue;
8751            };
8752            // Compare the whole launch spec so a future launch field cannot
8753            // accidentally become a live-only policy change. Health is stored
8754            // separately and applies live without replacing the process.
8755            let launch_changed = *current_spec != configured_module.module_spec();
8756            let configuration_changed =
8757                launch_changed || *current_health != configured_module.health;
8758            // Applying a rescan also replaces the recorded config value, even
8759            // when an operator already put the runtime in the requested state.
8760            let enabled_changed = *current_enabled != configured_module.enabled
8761                || self
8762                    .capability_evaluator
8763                    .configured_enabled(module_id)
8764                    .is_some_and(|enabled| enabled != configured_module.enabled);
8765            if configuration_changed {
8766                configuration_changes.insert(module_id.clone());
8767            }
8768            if launch_changed {
8769                changed_pending_reload.push(module_id.clone());
8770            }
8771            if enabled_changed {
8772                enabled_changes.insert(module_id.clone());
8773            }
8774            if !configuration_changed && !enabled_changed {
8775                unchanged = unchanged.saturating_add(1);
8776            }
8777        }
8778        Ok(RescanPlan {
8779            current,
8780            configured,
8781            added,
8782            removed,
8783            changed_pending_reload,
8784            configuration_changes,
8785            enabled_changes,
8786            unchanged,
8787        })
8788    }
8789
8790    /// Reconcile the running module set against the configured one.
8791    ///
8792    /// With `preview` set, the diff is computed and returned WITHOUT applying any
8793    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
8794    /// deliberately shares this function with the executing path rather than
8795    /// computing the same diff somewhere else -- two implementations of one
8796    /// decision agree until they do not, and the whole value of a preview is that
8797    /// it describes the operation that will actually run.
8798    async fn reconcile_supervised_modules(
8799        &self,
8800        supervisor: &Supervisor,
8801        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8802        preview: bool,
8803        requester: Principal,
8804    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
8805        let plan = self.plan_rescan(configured_modules)?;
8806
8807        // Everything above this point is pure computation over two snapshots.
8808        // Everything below MUTATES. The preview returns here so the boundary is a
8809        // single early return rather than a condition repeated at each mutation
8810        // site, where one missed guard would apply part of a change the caller was
8811        // told would not happen.
8812        if preview {
8813            return Ok((plan.preview_result(), Vec::new()));
8814        }
8815        let changed = plan
8816            .configuration_changes
8817            .union(&plan.enabled_changes)
8818            .cloned()
8819            .collect::<Vec<_>>();
8820        let RescanPlan {
8821            current,
8822            configured,
8823            added,
8824            removed,
8825            changed_pending_reload,
8826            configuration_changes,
8827            enabled_changes,
8828            unchanged,
8829        } = plan;
8830        // The staged generation moves before the first change rather than
8831        // after the last, so a rescan that fails partway still invalidates
8832        // every earlier `supervisor.rescan_checked` preview: part of its diff
8833        // may have run.
8834        if !(added.is_empty()
8835            && removed.is_empty()
8836            && configuration_changes.is_empty()
8837            && enabled_changes.is_empty())
8838        {
8839            if let Some(context) = &self.rescan {
8840                context
8841                    .generation
8842                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
8843            }
8844        }
8845
8846        for module_id in &removed {
8847            let module = &current
8848                .get(module_id)
8849                .expect("removed module came from current supervisor state")
8850                .0;
8851            module
8852                .retire_requested(Some(requester.clone()))
8853                .await
8854                .map_err(|err| {
8855                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
8856                })?;
8857            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
8858            //
8859            // `handle_route_open` resolves an absent module in three steps:
8860            // registry, then supervisor status, then tombstone. Retiring first
8861            // opens a window where ALL THREE ARE ABSENT -- the registry entry
8862            // went with the teardown above, the supervisor entry went with
8863            // `retire`, and the tombstone does not exist yet -- so a route.open
8864            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
8865            // it") for a module that was deliberately removed and whose caller
8866            // should get `module_removed` (TERMINAL, carrying a removal age).
8867            //
8868            // Writing the tombstone first closes it: during the window the
8869            // supervisor entry still answers, so the caller gets
8870            // `target_unavailable` -- retryable, and TRUE, because the module
8871            // is mid-teardown. After both statements it is `module_removed`.
8872            // No instant remains where a removed module reads as one that
8873            // never existed.
8874            //
8875            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
8876            // the absence of a test beside a fix invites deletion: these are
8877            // two sync statements with no await between them, so reaching the
8878            // window needs a second worker thread to land exactly between them
8879            // and there is no hook to force it. MEASURED: the 25 daemon_config
8880            // tests pass identically with the old order and the new one, so
8881            // the existing suite cannot see this and a green run is not
8882            // evidence either way. What the suite does hold is the
8883            // post-condition -- a removed module answers `module_removed` --
8884            // which this preserves.
8885            //
8886            // Found by an Athena panel reading the shipped tree against a
8887            // design note (2026-09-19), as the one concrete instance of that
8888            // note's class that survived contact with source. Direction is
8889            // benign: retryable where terminal was intended, never the reverse.
8890            self.supervisor.record_rescan_removal(module_id);
8891            self.supervisor.retire(module_id);
8892            self.route_outages.forget(module_id);
8893        }
8894
8895        for module_id in configured.keys() {
8896            let Some((module, _, _, current_enabled)) = current.get(module_id) else {
8897                continue;
8898            };
8899            let configured_module = configured
8900                .get(module_id)
8901                .expect("configured module id came from configured map");
8902            if configuration_changes.contains(module_id) {
8903                module
8904                    .update_configuration(
8905                        configured_module.module_spec(),
8906                        configured_module.health.clone(),
8907                        configured_module.drain_timeout_ms,
8908                    )
8909                    .await
8910                    .map_err(|err| {
8911                        format!(
8912                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
8913                        )
8914                    })?;
8915            }
8916            // Config-only drift is applied by the evaluator's config update;
8917            // do not turn it into a runtime start or stop.
8918            if *current_enabled != configured_module.enabled {
8919                // A rescan that starts or stops a module applies an operator's
8920                // edit to the config, so the resulting outage was asked for.
8921                self.route_outages.mark_operator_action(module_id);
8922                module
8923                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
8924                    .await
8925                    .map_err(|err| {
8926                        self.route_outages.operator_action_ended_unrefused(module_id);
8927                        format!(
8928                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
8929                            configured_module.enabled
8930                        )
8931                    })?;
8932            }
8933        }
8934
8935        for module_id in &added {
8936            let configured_module = configured
8937                .get(module_id)
8938                .expect("added module id came from configured map");
8939            supervisor
8940                .supervise_configured_requested_async(
8941                    configured_module.module_spec(),
8942                    configured_module.enabled,
8943                    configured_module.health.clone(),
8944                    configured_module.drain_timeout_ms,
8945                    configured_module.restart,
8946                    configured_module.register_timeout,
8947                    Some(requester.clone()),
8948                )
8949                .await
8950                .map_err(|err| {
8951                    format!("failed to add module_id '{module_id}' during rescan: {err}")
8952                })?;
8953        }
8954
8955        Ok((
8956            SupervisorRescanResult {
8957                added,
8958                removed,
8959                changed_pending_reload,
8960                enabled_changes: enabled_changes.iter().cloned().collect(),
8961                unchanged,
8962                preview: false,
8963                // Filled by the caller, which is the only layer that can see the
8964                // previous config to diff against.
8965                restart_required: Vec::new(),
8966                capability_warnings: Vec::new(),
8967                module_actions: Vec::new(),
8968                earlier_pending: Vec::new(),
8969                expect: None,
8970            },
8971            changed,
8972        ))
8973    }
8974
8975    async fn handle_supervisor_set_enabled(
8976        &self,
8977        frame: Frame,
8978        module_id: String,
8979        enabled: bool,
8980        requester: Principal,
8981    ) -> Result<Vec<Frame>, RouterError> {
8982        let operation_lock = self.supervisor.operation_lock();
8983        let _operation_guard = operation_lock.lock().await;
8984        let Some(module) = self.supervisor.get(&module_id) else {
8985            return Ok(vec![control_error_frame(
8986                &frame,
8987                "unknown_module",
8988                format!("module_id '{module_id}' is not supervised"),
8989            )?]);
8990        };
8991
8992        // Enabling counts as well as disabling: a module an operator starts
8993        // is refused until it registers, and that wait was asked for.
8994        self.route_outages.mark_operator_action(&module_id);
8995        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
8996            Ok(applied) => applied,
8997            Err(err) => {
8998                self.route_outages
8999                    .operator_action_ended_unrefused(&module_id);
9000                return Ok(vec![control_error_frame(
9001                    &frame,
9002                    if err.is_config_gate_error() {
9003                        "config_gated"
9004                    } else {
9005                        "target_unavailable"
9006                    },
9007                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
9008                )?]);
9009            }
9010        };
9011        if !applied {
9012            // Already in the requested state: nothing was made unavailable,
9013            // so the mark must not outlive this request.
9014            self.route_outages
9015                .operator_action_ended_unrefused(&module_id);
9016        }
9017
9018        self.capability_evaluator.wake_deadline_loop();
9019        self.refresh_capability_requirements();
9020        let response = ClientControlResponse::SupervisorAck { module_id, applied };
9021        Ok(vec![control_response_body_frame(
9022            &frame,
9023            &response,
9024            "ClientControlResponse::SupervisorAck",
9025        )?])
9026    }
9027
9028    async fn handle_supervisor_health_probe(
9029        &self,
9030        ctx: &RouteCtx,
9031        frame: Frame,
9032        module_id: String,
9033    ) -> Result<Vec<Frame>, RouterError> {
9034        self.refresh_capability_requirements();
9035        let Some(registration) = self
9036            .registry
9037            .get_module(&module_id)
9038            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
9039        else {
9040            return Ok(vec![control_error_frame(
9041                &frame,
9042                "unknown_module",
9043                format!("module_id '{module_id}' is not registered"),
9044            )?]);
9045        };
9046
9047        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
9048        // named for it. Making `module_registration_grants_op` return false
9049        // unconditionally reddens five tests, and every one is named for something
9050        // else -- capability relay, probe/bind demultiplexing, supervision-only
9051        // probing. They exercise a successful advertisement check on the way to their
9052        // own subject.
9053        //
9054        // Real protection, fragile in a specific way: narrowing any of those tests to
9055        // focus on its stated subject would silently remove coverage nobody knows
9056        // they are carrying. Recorded here rather than as a sixth test, because the
9057        // useful fact is WHICH tests hold the guard up -- a new test would add
9058        // coverage without telling the next person what the existing ones quietly do.
9059        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
9060        {
9061            return Ok(vec![control_error_frame(
9062                &frame,
9063                "health_not_advertised",
9064                format!("module_id '{module_id}' did not advertise health.check"),
9065            )?]);
9066        }
9067
9068        self.observe_control_wait(
9069            ctx,
9070            Some(&registration.manifest.module_id),
9071            ops::SUPERVISOR_HEALTH_PROBE,
9072            async {
9073                let deadline = Instant::now() + self.health_probe_timeout;
9074                let pending = match self.forwarding.begin_module_control_rpc_for(
9075                    &module_id,
9076                    MODULE_CONTROL_OP_HEALTH_CHECK,
9077                    deadline,
9078                ) {
9079                    Ok(pending) => pending,
9080                    Err(err) => {
9081                        return Ok(vec![control_error_frame(
9082                            &frame,
9083                            forwarding_error_code(&err),
9084                            err.to_string(),
9085                        )?])
9086                    }
9087                };
9088
9089                let PendingModuleControlRpc {
9090                    endpoint,
9091                    module_sink,
9092                    negotiated_ver,
9093                    corr: probe_corr,
9094                    receiver,
9095                } = pending;
9096                let mut guard =
9097                    ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
9098                let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
9099                    .map_err(|err| {
9100                        RouterError::backend(
9101                            0,
9102                            frame.header.corr,
9103                            format!("failed to encode health.check request: {err}"),
9104                        )
9105                    })?;
9106                let probe_frame = Frame::build_with_version(
9107                    negotiated_ver,
9108                    FrameType::Request,
9109                    control_flags(),
9110                    0,
9111                    0,
9112                    probe_corr,
9113                    probe_body,
9114                )
9115                .map_err(RouterError::FrameBuild)?;
9116
9117                // Queue admission and the reply share one deadline, so a module that
9118                // stops reading cannot park the requesting connection's reader forever.
9119                match timeout_at(deadline, module_sink.send(probe_frame)).await {
9120                    Ok(Ok(())) => {}
9121                    Ok(Err(err)) => {
9122                        return Ok(vec![control_error_frame(
9123                            &frame,
9124                            "target_unavailable",
9125                            err.to_string(),
9126                        )?]);
9127                    }
9128                    Err(_) => {
9129                        return Ok(vec![control_error_frame(
9130                            &frame,
9131                            "module_timeout",
9132                            format!(
9133                                "module_id '{module_id}' did not answer health.check within {:?}",
9134                                self.health_probe_timeout
9135                            ),
9136                        )?]);
9137                    }
9138                }
9139
9140                match timeout_at(deadline, receiver).await {
9141                    Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => {
9142                        Ok(vec![control_error_frame(
9143                            &frame,
9144                            "invalid_control_body",
9145                            "health.check returned an unexpected config response",
9146                        )?])
9147                    }
9148                    Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
9149                        guard.disarm();
9150                        let Some(report) = response.health_report() else {
9151                            return Ok(vec![control_error_frame(
9152                                &frame,
9153                                "invalid_control_body",
9154                                "health.check RPC returned a non-health response",
9155                            )?]);
9156                        };
9157                        // Metrics go out whole here. The supervisor's cached snapshot
9158                        // caps this blob (see truncate_health_metrics), and this path
9159                        // exists precisely to answer without that cap -- so applying it
9160                        // here would leave no way to see what the cached view drops.
9161                        let HealthReport {
9162                            status,
9163                            detail,
9164                            metrics,
9165                        } = report;
9166                        let capability_detail = self
9167                            .capability_evaluator
9168                            .required_problem_detail(&module_id);
9169                        let response = ClientControlResponse::SupervisorHealthProbe {
9170                            module_id,
9171                            status,
9172                            detail: append_capability_problem_detail(detail, capability_detail),
9173                            metrics,
9174                        };
9175                        Ok(vec![control_response_body_frame(
9176                            &frame,
9177                            &response,
9178                            "ClientControlResponse::SupervisorHealthProbe",
9179                        )?])
9180                    }
9181                    Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
9182                        guard.disarm();
9183                        // The module refused the health probe with its own body, so
9184                        // the refusal is relayed without the daemon-origin flag
9185                        // (bit 6): a client must not read it as the daemon's own
9186                        // refusal, which it may treat as never forwarded.
9187                        Ok(vec![control_relayed_error_body_frame(&frame, body)?])
9188                    }
9189                    Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
9190                        guard.disarm();
9191                        Ok(vec![control_error_frame(
9192                            &frame,
9193                            "target_unavailable",
9194                            message,
9195                        )?])
9196                    }
9197                    Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
9198                        guard.disarm();
9199                        Ok(vec![control_error_frame(
9200                            &frame,
9201                            "invalid_control_body",
9202                            message,
9203                        )?])
9204                    }
9205                    Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
9206                        guard.disarm();
9207                        Ok(vec![control_error_frame(
9208                            &frame,
9209                            "invalid_control_body",
9210                            format!("expected module-control op '{expected}', got '{actual}'"),
9211                        )?])
9212                    }
9213                    Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
9214                        guard.disarm();
9215                        Ok(vec![control_error_frame(
9216                            &frame,
9217                            "module_timeout",
9218                            format!(
9219                                "module_id '{module_id}' answered health.check after {:?}",
9220                                self.health_probe_timeout
9221                            ),
9222                        )?])
9223                    }
9224                    Ok(Err(_)) => Ok(vec![control_error_frame(
9225                        &frame,
9226                        "target_unavailable",
9227                        "health.check waiter was canceled before the module responded",
9228                    )?]),
9229                    Err(_) => Ok(vec![control_error_frame(
9230                        &frame,
9231                        "module_timeout",
9232                        format!(
9233                            "module_id '{module_id}' did not answer health.check within {:?}",
9234                            self.health_probe_timeout
9235                        ),
9236                    )?]),
9237                }
9238            },
9239        )
9240        .await
9241    }
9242
9243    fn supervisor_status(
9244        &self,
9245        module_id: &str,
9246        corr: u64,
9247    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
9248        self.supervisor
9249            .get(module_id)
9250            .map(|module| {
9251                let warming = module.is_warming_for_control("status").map_err(|err| {
9252                    RouterError::backend(
9253                        0,
9254                        corr,
9255                        format!(
9256                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
9257                        ),
9258                    )
9259                })?;
9260                module.status_for_control("status").map_err(|err| {
9261                    RouterError::backend(
9262                        0,
9263                        corr,
9264                        format!(
9265                            "failed to read supervisor status for module_id '{module_id}': {err}"
9266                        ),
9267                    )
9268                }).map(|status| (status, warming))
9269            })
9270            .transpose()
9271    }
9272
9273    fn guard_module_control_op(
9274        &self,
9275        frame: &Frame,
9276        module_id: &str,
9277        op: &str,
9278    ) -> Result<Option<Frame>, RouterError> {
9279        if self.module_grants_op(module_id, op, frame.header.corr)? {
9280            return Ok(None);
9281        }
9282
9283        Ok(Some(control_error_frame(
9284            frame,
9285            "op_not_allowed",
9286            format!("module_id '{module_id}' did not grant control op '{op}'"),
9287        )?))
9288    }
9289
9290    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
9291        let Some(registration) = self
9292            .registry
9293            .get_module(module_id)
9294            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
9295        else {
9296            return Ok(false);
9297        };
9298        Ok(module_registration_grants_op(&registration.control_ops, op))
9299    }
9300
9301    fn handle_status_update(
9302        &self,
9303        endpoint: ModuleEndpointId,
9304        frame: Frame,
9305    ) -> Result<Vec<Frame>, RouterError> {
9306        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
9307            Ok(update) => update,
9308            Err(err) => {
9309                // Forward-compat: a newer module may push a channel-0 op this subc
9310                // version doesn't know. The control contract says unknown push ops
9311                // are IGNORED, never answered with an error. Only a malformed body
9312                // for an op we DO know is a real error worth surfacing.
9313                if is_known_module_push_op(&frame.body) {
9314                    return Ok(vec![control_error_frame(
9315                        &frame,
9316                        "invalid_control_body",
9317                        format!("malformed module control push body: {err}"),
9318                    )?]);
9319                }
9320                return Ok(Vec::new());
9321            }
9322        };
9323
9324        match update {
9325            ModuleControlPush::RouteStatus {
9326                route_channel,
9327                route_epoch,
9328                status,
9329            } => {
9330                self.forwarding
9331                    .cache_status(endpoint, route_channel, route_epoch, status)
9332                    .map_err(RouterError::Forwarding)?;
9333            }
9334        }
9335        Ok(Vec::new())
9336    }
9337
9338    fn handle_route_poll(
9339        &self,
9340        ctx: &RouteCtx,
9341        frame: Frame,
9342        route_channel: u16,
9343        route_epoch: u32,
9344        kind: PollKind,
9345    ) -> Result<Vec<Frame>, RouterError> {
9346        let snapshot = self
9347            .forwarding
9348            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
9349            .map_err(RouterError::Forwarding)?;
9350        let response = match (kind, snapshot) {
9351            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
9352                ClientControlResponse::RoutePoll {
9353                    route_channel,
9354                    route_epoch,
9355                    status,
9356                    live: None,
9357                }
9358            }
9359            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9360                route_channel,
9361                route_epoch,
9362                status: None,
9363                live: None,
9364            },
9365            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
9366                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
9367                // is what makes reporting `true` correct rather than a
9368                // confident guess. `process_live` returns None only when the
9369                // module id has no supervisor snapshot at all -- an
9370                // externally-started module the daemon did not spawn -- and
9371                // for those the supervisor has no opinion to offer, ever. It
9372                // is never None for a supervised module in an unknown state:
9373                // a supervised module always has a snapshot, and the answer
9374                // comes from `state == Running && process_alive`.
9375                //
9376                // The route is Bound, so the module completed a HELLO on a
9377                // live connection; "the process this route points at is
9378                // running" is therefore attested by the binding rather than
9379                // assumed. Reporting `false` for an unsupervised module would
9380                // be the actual lie -- it would tell a client its healthy
9381                // route is dead because the daemon does not manage the
9382                // process.
9383                //
9384                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
9385                // module whose liveness is genuinely unknown, e.g. a snapshot
9386                // that has not been populated yet -- THIS DEFAULT BECOMES
9387                // WRONG and must split: unsupervised stays true, unknown
9388                // becomes null so the client can tell the two apart. The
9389                // response field is already `Option<bool>`, so the wire can
9390                // carry that distinction today.
9391                let live = self
9392                    .process_liveness
9393                    .as_ref()
9394                    .and_then(|source| source.process_live(&module_id))
9395                    .unwrap_or(true);
9396                ClientControlResponse::RoutePoll {
9397                    route_channel,
9398                    route_epoch,
9399                    status: None,
9400                    live: Some(live),
9401                }
9402            }
9403            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9404                route_channel,
9405                route_epoch,
9406                status: None,
9407                live: Some(false),
9408            },
9409        };
9410
9411        Ok(vec![control_response_body_frame(
9412            &frame,
9413            &response,
9414            "ClientControlResponse::RoutePoll",
9415        )?])
9416    }
9417
9418    pub(crate) fn observe_module_control_completion(
9419        &self,
9420        completion: ModuleControlRpcCompletion,
9421    ) -> bool {
9422        match completion {
9423            ModuleControlRpcCompletion::Unknown => false,
9424            ModuleControlRpcCompletion::Settled => true,
9425            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
9426                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
9427                info!(
9428                    module_id = %module_id,
9429                    latency_ms,
9430                    "late health.check answer proves the module is alive"
9431                );
9432                match self
9433                    .supervisor
9434                    .record_late_health_answer(&module_id, latency_ms)
9435                {
9436                    Ok(true) => {}
9437                    Ok(false) => debug!(
9438                        module_id = %module_id,
9439                        latency_ms,
9440                        "late health.check answer has no active supervisor snapshot"
9441                    ),
9442                    Err(err) => warn!(
9443                        module_id = %module_id,
9444                        latency_ms,
9445                        error = %err,
9446                        "failed to record late health.check answer"
9447                    ),
9448                }
9449                true
9450            }
9451        }
9452    }
9453
9454    /// Decide whether a failure while settling a relayed `route.bind` belongs to
9455    /// the module connection whose frame is being handled, or to the client that
9456    /// relay was opened for.
9457    ///
9458    /// This runs on the MODULE connection's frame handler, where returning `Err`
9459    /// ends that connection -- and a module connection carries every client's
9460    /// routes to that module, so ending it costs the whole fleet its tools.
9461    /// `ConnectionClosing` carries the id of the connection that is closing, and
9462    /// when that id is a CLIENT's, the condition is entirely about that one
9463    /// client's route.open. A client-scoped condition has no authority over a
9464    /// shared module connection, so it is logged and the single relay is dropped:
9465    /// the client is going away, and `complete_pending_relay` already removed the
9466    /// relay before failing, so there is nothing left to settle. Anything that
9467    /// relay still reserved is released by that client's own connection teardown,
9468    /// which is already under way -- that is what "closing" means.
9469    ///
9470    /// Every other failure is a statement about THIS connection and stays fatal:
9471    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
9472    /// id in `ConnectionClosing` all mean this connection cannot keep serving
9473    /// frames correctly.
9474    fn refuse_to_end_module_connection_for_a_client(
9475        &self,
9476        module_connection_id: ConnectionId,
9477        corr: u64,
9478        err: ForwardingError,
9479    ) -> Result<(), RouterError> {
9480        if let ForwardingError::ConnectionClosing { connection_id } = err {
9481            if connection_id != module_connection_id {
9482                warn!(
9483                    module_connection_id = module_connection_id.get(),
9484                    client_connection_id = connection_id.get(),
9485                    corr,
9486                    "dropping a route.bind response for a closing client; the module connection keeps serving"
9487                );
9488                return Ok(());
9489            }
9490        }
9491        Err(RouterError::Forwarding(err))
9492    }
9493
9494    fn handle_module_relay_response(
9495        &self,
9496        connection_id: ConnectionId,
9497        frame: Frame,
9498    ) -> Result<Vec<Frame>, RouterError> {
9499        let mut secondary_error = None;
9500        let outcome = match frame.header.ty {
9501            FrameType::Response => match module_control_response_probe(&frame.body) {
9502                Ok(probe) if probe.op == "route.bind" => {
9503                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9504                        Ok(ModuleControlResponse::RouteBindAck {}) => {
9505                            RouteBindRelayOutcome::Accepted
9506                        }
9507                        Ok(other) => {
9508                            let message =
9509                                format!("route.bind response carried unexpected body: {other:?}");
9510                            secondary_error = Some(control_error_frame(
9511                                &frame,
9512                                "invalid_control_body",
9513                                message.clone(),
9514                            )?);
9515                            RouteBindRelayOutcome::ModuleGone(message)
9516                        }
9517                        Err(err) => {
9518                            let message = format!("malformed route.bind response body: {err}");
9519                            secondary_error = Some(control_error_frame(
9520                                &frame,
9521                                "invalid_control_body",
9522                                message.clone(),
9523                            )?);
9524                            RouteBindRelayOutcome::ModuleGone(message)
9525                        }
9526                    }
9527                }
9528                Ok(probe) => {
9529                    let outcome = if matches!(probe.op.as_str(), "config.review" | "consent/v1") {
9530                        ModuleControlRpcOutcome::RawResponse(frame.body.clone())
9531                    } else {
9532                        match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9533                            Ok(response) => ModuleControlRpcOutcome::Response(response),
9534                            Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9535                                "malformed {} response body: {err}",
9536                                probe.op
9537                            )),
9538                        }
9539                    };
9540                    let completion = self
9541                        .forwarding
9542                        .complete_module_control_rpc(
9543                            connection_id,
9544                            frame.header.corr,
9545                            Some(&probe.op),
9546                            outcome,
9547                        )
9548                        .map_err(RouterError::Forwarding)?;
9549                    if !self.observe_module_control_completion(completion) {
9550                        debug!(
9551                            connection_id = connection_id.get(),
9552                            corr = frame.header.corr,
9553                            op = %probe.op,
9554                            "dropping late or unknown module-control RPC response"
9555                        );
9556                    }
9557                    return Ok(Vec::new());
9558                }
9559                Err(err) => {
9560                    if let Some(expected_op) = self
9561                        .forwarding
9562                        .pending_module_control_op(connection_id, frame.header.corr)
9563                        .map_err(RouterError::Forwarding)?
9564                    {
9565                        let completion = self
9566                            .forwarding
9567                            .complete_module_control_rpc(
9568                                connection_id,
9569                                frame.header.corr,
9570                                None,
9571                                ModuleControlRpcOutcome::MalformedResponse(format!(
9572                                    "malformed {expected_op} response body: {err}"
9573                                )),
9574                            )
9575                            .map_err(RouterError::Forwarding)?;
9576                        if !self.observe_module_control_completion(completion) {
9577                            debug!(
9578                                connection_id = connection_id.get(),
9579                                corr = frame.header.corr,
9580                                "dropping late malformed module-control RPC response"
9581                            );
9582                        }
9583                        return Ok(Vec::new());
9584                    }
9585                    let message = format!("malformed route.bind response body: {err}");
9586                    secondary_error = Some(control_error_frame(
9587                        &frame,
9588                        "invalid_control_body",
9589                        message.clone(),
9590                    )?);
9591                    RouteBindRelayOutcome::ModuleGone(message)
9592                }
9593            },
9594            FrameType::Error => {
9595                if self
9596                    .forwarding
9597                    .pending_module_control_op(connection_id, frame.header.corr)
9598                    .map_err(RouterError::Forwarding)?
9599                    .is_some()
9600                {
9601                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
9602                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
9603                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9604                            "malformed module-control ERROR body: {err}"
9605                        )),
9606                    };
9607                    let completion = self
9608                        .forwarding
9609                        .complete_module_control_rpc(
9610                            connection_id,
9611                            frame.header.corr,
9612                            None,
9613                            outcome,
9614                        )
9615                        .map_err(RouterError::Forwarding)?;
9616                    if !self.observe_module_control_completion(completion) {
9617                        debug!(
9618                            connection_id = connection_id.get(),
9619                            corr = frame.header.corr,
9620                            "dropping late or unknown module-control RPC error"
9621                        );
9622                    }
9623                    return Ok(Vec::new());
9624                }
9625                match serde_json::from_slice::<ErrorBody>(&frame.body) {
9626                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
9627                    Err(err) => {
9628                        let message = format!("malformed route.bind ERROR body: {err}");
9629                        secondary_error = Some(control_error_frame(
9630                            &frame,
9631                            "invalid_control_body",
9632                            message.clone(),
9633                        )?);
9634                        RouteBindRelayOutcome::ModuleGone(message)
9635                    }
9636                }
9637            }
9638            ty => {
9639                return Ok(vec![control_error_frame(
9640                    &frame,
9641                    "unsupported_control_frame",
9642                    format!("unsupported module channel-0 frame {ty:?}"),
9643                )?])
9644            }
9645        };
9646
9647        let settled =
9648            self.forwarding
9649                .complete_pending_relay(connection_id, frame.header.corr, outcome);
9650        let completion = match settled {
9651            Ok(completion) => completion,
9652            Err(err) => {
9653                self.refuse_to_end_module_connection_for_a_client(
9654                    connection_id,
9655                    frame.header.corr,
9656                    err,
9657                )?;
9658                return Ok(secondary_error.into_iter().collect());
9659            }
9660        };
9661        if let Some(target) = completion.abandoned.as_ref() {
9662            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
9663        }
9664        if !completion.settled {
9665            debug!(
9666                connection_id = connection_id.get(),
9667                corr = frame.header.corr,
9668                frame_type = ?frame.header.ty,
9669                "dropping late or unknown route.bind relay response"
9670            );
9671        }
9672        Ok(secondary_error.into_iter().collect())
9673    }
9674
9675    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
9676        debug!(connection_id = connection_id.get(), "handling GOODBYE");
9677        // GOODBYE ends the connection's logical session even when its socket
9678        // stays open. Use disconnect teardown so verdicts, client notices and
9679        // scope authority are released at the same lifecycle boundary.
9680        self.cleanup_connection_with_end_reason(
9681            connection_id,
9682            RegistrationEndReason::ExplicitGoodbye,
9683        )
9684        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
9685        Ok(Vec::new())
9686    }
9687}
9688
9689impl Default for ControlHandler {
9690    fn default() -> Self {
9691        Self::new(Arc::new(Registry::default()))
9692    }
9693}
9694
9695impl crate::supervise::SwapPromotionObserver for ControlHandler {
9696    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
9697        self.apply_registration_capabilities(registration);
9698    }
9699}
9700
9701fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
9702    CapabilityRequirementStatus {
9703        consumer: status.consumer,
9704        capability: status.capability,
9705        need: match status.need {
9706            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
9707            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
9708        },
9709        verdict: status.verdict.as_str().to_string(),
9710        episode_seq: status.episode_seq,
9711        config_satisfiable: status.config_satisfiable,
9712        runtime_available: status.runtime_available,
9713        detail: status.detail,
9714    }
9715}
9716
9717fn append_capability_problem_detail(
9718    detail: Option<String>,
9719    capability_detail: Option<String>,
9720) -> Option<String> {
9721    match (detail, capability_detail) {
9722        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
9723        (Some(detail), None) => Some(detail),
9724        (None, Some(capability_detail)) => Some(capability_detail),
9725        (None, None) => None,
9726    }
9727}
9728
9729fn subc_ops() -> Vec<String> {
9730    SUBC_CONTROL_OPS
9731        .iter()
9732        .map(|op| (*op).to_string())
9733        .collect()
9734}
9735
9736fn module_subc_ops() -> Vec<String> {
9737    SUBC_CONTROL_OPS
9738        .iter()
9739        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
9740        .map(|op| (*op).to_string())
9741        .collect()
9742}
9743
9744#[cfg(test)]
9745fn module_baseline_control_ops() -> Vec<String> {
9746    MODULE_BASELINE_CONTROL_OPS
9747        .iter()
9748        .map(|op| (*op).to_string())
9749        .collect()
9750}
9751
9752fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
9753    let mut seen = HashSet::new();
9754    let mut effective = Vec::new();
9755    for op in MODULE_BASELINE_CONTROL_OPS {
9756        if seen.insert((*op).to_string()) {
9757            effective.push((*op).to_string());
9758        }
9759    }
9760    for op in declared.unwrap_or_default() {
9761        if seen.insert(op.clone()) {
9762            effective.push(op);
9763        }
9764    }
9765    effective
9766}
9767
9768fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
9769    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
9770}
9771
9772fn target_module_id(target: &RouteTarget) -> &str {
9773    match target {
9774        RouteTarget::ToolProvider { module_id }
9775        | RouteTarget::ManagementSurface { module_id }
9776        | RouteTarget::InternalService { module_id, .. } => module_id,
9777    }
9778}
9779
9780fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
9781    roles.iter().any(|role| match (target, role) {
9782        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
9783        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
9784        (
9785            RouteTarget::InternalService { service_id, .. },
9786            ProviderRole::InternalService {
9787                service_id: provided,
9788                ..
9789            },
9790        ) => service_id == provided,
9791        _ => false,
9792    })
9793}
9794
9795fn is_routable_role(role: &ProviderRole) -> bool {
9796    matches!(
9797        role,
9798        ProviderRole::ToolProvider { .. }
9799            | ProviderRole::ManagementSurface { .. }
9800            | ProviderRole::InternalService { .. }
9801    )
9802}
9803
9804#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9805enum ControlRequestBodyError {
9806    UnknownOp,
9807    InvalidBody,
9808}
9809
9810#[derive(Debug, Deserialize)]
9811struct ControlOpProbe {
9812    op: String,
9813}
9814
9815fn module_control_response_probe(body: &[u8]) -> Result<ControlOpProbe, serde_json::Error> {
9816    serde_json::from_slice(body).or_else(|original| {
9817        #[derive(Deserialize)]
9818        struct ConsentProbe {
9819            protocol: String,
9820        }
9821        match serde_json::from_slice::<ConsentProbe>(body) {
9822            Ok(probe) if probe.protocol == subc_protocol::consent::CONSENT_V1 => {
9823                Ok(ControlOpProbe { op: probe.protocol })
9824            }
9825            _ => Err(original),
9826        }
9827    })
9828}
9829
9830/// Channel-0 push ops this subc version understands. A push whose `op` is not in
9831/// this set is treated as a forward-compat unknown and ignored rather than errored.
9832const MODULE_PUSH_OPS: &[&str] = &["route.status"];
9833
9834fn is_known_module_push_op(body: &[u8]) -> bool {
9835    serde_json::from_slice::<ControlOpProbe>(body)
9836        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
9837        .unwrap_or(false)
9838}
9839
9840fn is_known_module_request_op(body: &[u8]) -> bool {
9841    serde_json::from_slice::<ControlOpProbe>(body)
9842        .map(|probe| is_module_to_subc_op(&probe.op))
9843        .unwrap_or(false)
9844}
9845
9846fn is_module_to_subc_op(op: &str) -> bool {
9847    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
9848}
9849
9850fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
9851    debug!(
9852        op = %op,
9853        connection_id = connection_id.get(),
9854        corr,
9855        "control dispatch"
9856    );
9857}
9858
9859fn log_slow_control_dispatch(
9860    dispatch_started_at: Option<StdInstant>,
9861    op: &'static str,
9862    connection_id: ConnectionId,
9863    corr: u64,
9864    route_open_timings: Option<&RouteOpenTimings>,
9865) {
9866    let Some(dispatch_started_at) = dispatch_started_at else {
9867        return;
9868    };
9869    // Use the same monotonic clock as the phase timers and relay deadline.
9870    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
9871    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
9872        if let Some(timings) = route_open_timings {
9873            warn!(
9874                op = %op,
9875                connection_id = connection_id.get(),
9876                corr,
9877                elapsed_ms = elapsed.as_millis() as u64,
9878                module_id = ?timings.module_id,
9879                principal = %timings.principal,
9880                admission_ms = timings.admission.as_millis() as u64,
9881                project_root_ms = timings.project_root.as_millis() as u64,
9882                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
9883                bind_send_ms = timings.bind_send.as_millis() as u64,
9884                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
9885                "slow control dispatch"
9886            );
9887            return;
9888        }
9889        warn!(
9890            op = %op,
9891            connection_id = connection_id.get(),
9892            corr,
9893            elapsed_ms = elapsed.as_millis() as u64,
9894            "slow control dispatch"
9895        );
9896    }
9897}
9898
9899/// A test stand-in for the OS caller lookup, given the recorded endpoints.
9900#[cfg(test)]
9901type TestCallerLookup =
9902    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
9903
9904/// Every new client operation needs an explicit privilege decision here.
9905fn privileged_operation(request: &impl PrivilegedRequest) -> Option<PrivilegedOperation<'_>> {
9906    request.privileged_operation()
9907}
9908
9909trait PrivilegedRequest {
9910    fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>>;
9911}
9912
9913impl PrivilegedRequest for ClientControlRequest {
9914    fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>> {
9915        let request = self;
9916        match request {
9917        ClientControlRequest::SupervisorRestart { module_id, .. } => {
9918            Some(PrivilegedOperation::Restart { module_id })
9919        }
9920        ClientControlRequest::SupervisorSwap { module_id, .. } => {
9921            Some(PrivilegedOperation::Swap { module_id })
9922        }
9923        ClientControlRequest::SupervisorReload { module_id } => {
9924            Some(PrivilegedOperation::Reload { module_id })
9925        }
9926        ClientControlRequest::SupervisorRescan { preview } => {
9927            if *preview {
9928                None
9929            } else {
9930                Some(PrivilegedOperation::Rescan)
9931            }
9932        }
9933        // Only the well-formed apply asks for approval. A preview changes
9934        // nothing, and any other combination is refused before it touches
9935        // anything, so prompting the operator for it would ask about a request
9936        // that cannot run.
9937        ClientControlRequest::SupervisorRescanChecked {
9938            preview,
9939            candidate,
9940            expect,
9941        } => (!*preview && candidate.is_none() && expect.is_some())
9942            .then_some(PrivilegedOperation::Rescan),
9943        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
9944            Some(PrivilegedOperation::SetEnabled {
9945                module_id,
9946                enabled: *enabled,
9947            })
9948        }
9949        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
9950            Some(PrivilegedOperation::ReleaseReserved { module_id })
9951        }
9952        ClientControlRequest::ServerDescribe {}
9953        // The announcement uses the same client admission as supervisor.restart,
9954        // but cannot stop anything itself and needs no operator confirmation.
9955        | ClientControlRequest::DaemonShutdownIntent { .. }
9956        | ClientControlRequest::CatalogList { .. }
9957        | ClientControlRequest::RouteOpen { .. }
9958        | ClientControlRequest::RoutePoll { .. }
9959        | ClientControlRequest::SupervisorList {}
9960        | ClientControlRequest::SupervisorHealth {}
9961        | ClientControlRequest::SupervisorHealthProbe { .. }
9962        | ClientControlRequest::SupervisorRoutes { .. }
9963        | ClientControlRequest::SupervisorProvenance { .. }
9964        | ClientControlRequest::SupervisorStderrTail { .. }
9965        | ClientControlRequest::SupervisorTerminals { .. }
9966        | ClientControlRequest::SupervisorSpawnSnapshot {}
9967        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
9968    }
9969    }
9970}
9971
9972fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
9973    match request {
9974        ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
9975        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
9976        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
9977        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
9978        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
9979        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
9980        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
9981        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
9982        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
9983        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
9984        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
9985        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
9986        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
9987        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
9988        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
9989        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
9990        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
9991        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
9992        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
9993        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
9994        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
9995    }
9996}
9997
9998/// The caller of one `route.open`, as its `route.open refused` line names it.
9999///
10000/// A connection id alone cannot name a caller after the fact: ids restart at
10001/// every daemon restart, so the same number belongs to different modules
10002/// before and after one. These three fields are what the refused line needs
10003/// to be attributable on its own, and they are rendered with the same helpers
10004/// as the `route.open accepted` line so both directions grep alike.
10005struct RouteOpenCaller {
10006    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
10007    /// when the request presented a consumer identity whose launch nonce did
10008    /// not verify. The claimed module id is never printed as a principal.
10009    principal: String,
10010    /// `none` for an unscoped open; the admitted stamp's label once scope
10011    /// admission issued one; before that, the requested selector's label.
10012    scope: String,
10013    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
10014    /// because it is a string the caller chose.
10015    harness: String,
10016}
10017
10018impl RouteOpenCaller {
10019    fn new(
10020        principal: Option<&Principal>,
10021        scope: Option<&ScopeSelector>,
10022        identity: &BindIdentity,
10023    ) -> Self {
10024        Self {
10025            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
10026            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
10027            harness: bounded_log_text(&identity.harness),
10028        }
10029    }
10030
10031    /// For a refusal that has no parsed request at hand: names whatever the
10032    /// frame still says, and `unknown` where it says nothing parseable.
10033    fn unknown() -> Self {
10034        Self {
10035            principal: "unknown".to_string(),
10036            scope: "unknown".to_string(),
10037            harness: "unknown".to_string(),
10038        }
10039    }
10040}
10041
10042/// Longest client-chosen text (a scope ref, a harness) a route.open log line
10043/// prints before cutting it.
10044const MAX_LOGGED_TEXT_CHARS: usize = 64;
10045
10046/// `value` Debug-quoted, so control characters land escaped and cannot forge
10047/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
10048/// `…` inside the quotes so one request cannot flood the line.
10049fn bounded_log_text(value: &str) -> String {
10050    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
10051    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
10052        bounded.push('…');
10053    }
10054    format!("{bounded:?}")
10055}
10056
10057/// The `scope=` field of the `route.open accepted` line, from the stamp that
10058/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
10059/// so a ref holding control characters cannot forge a log line, and cut to 64
10060/// characters with a trailing `…` to bound the line. The bind proof and the
10061/// scope's attributes never appear.
10062fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
10063    use subc_protocol::scope::ScopeKind;
10064    let kind = match stamp.kind {
10065        ScopeKind::Head => "head",
10066        ScopeKind::Worker => "worker",
10067        ScopeKind::Ephemeral => "ephemeral",
10068        ScopeKind::Federated => "federated",
10069        _ => "unknown",
10070    };
10071    format!(
10072        "{kind}/{}/{}@{}",
10073        principal_label(&stamp.owner),
10074        bounded_log_text(&stamp.scope_ref),
10075        stamp.scope_epoch
10076    )
10077}
10078
10079/// The `scope=` field of a `route.open refused` line for an open refused
10080/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
10081/// The kind is not known yet, so the literal `requested` stands in its place.
10082/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
10083/// owner it has not been checked against the configured modules: a quoted
10084/// owner means the value was never validated. The bind proof never appears.
10085fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
10086    let epoch = selector
10087        .scope_epoch
10088        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
10089    format!(
10090        "requested/{}/{}@{epoch}",
10091        bounded_log_text(&principal_label(&selector.owner)),
10092        bounded_log_text(&selector.scope_ref),
10093    )
10094}
10095
10096fn principal_label(principal: &Principal) -> String {
10097    match principal {
10098        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
10099        Principal::Direct => "direct".to_string(),
10100        other => format!("{other:?}"),
10101    }
10102}
10103
10104fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
10105    match request {
10106        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
10107        ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { .. } => {
10108            MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS
10109        }
10110        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
10111        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
10112        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
10113        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
10114        ModuleControlRequestFromModule::RouteAuthorize { .. } => {
10115            subc_protocol::scope::ROUTE_AUTHORIZE_OP
10116        }
10117        ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
10118        _ => "unsupported",
10119    }
10120}
10121
10122fn parse_client_control_request(
10123    body: &[u8],
10124) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
10125    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
10126        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10127            Ok(probe)
10128                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
10129                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
10130            {
10131                ControlRequestBodyError::InvalidBody
10132            }
10133            Ok(_) => ControlRequestBodyError::UnknownOp,
10134            Err(_) => ControlRequestBodyError::InvalidBody,
10135        };
10136        (err, classification)
10137    })
10138}
10139
10140fn parse_module_control_request_from_module(
10141    body: &[u8],
10142) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
10143    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
10144        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10145            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
10146            Ok(_) => ControlRequestBodyError::UnknownOp,
10147            Err(_) => ControlRequestBodyError::InvalidBody,
10148        };
10149        (err, classification)
10150    })
10151}
10152
10153#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
10154enum ProviderRoleKind {
10155    ToolProvider,
10156    PipelineStage,
10157    ManagementSurface,
10158    InternalService,
10159}
10160
10161fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
10162    match role {
10163        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
10164        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
10165        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
10166        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
10167    }
10168}
10169
10170fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
10171    roles.iter().map(provider_role_kind).collect()
10172}
10173
10174/// Most refused scope records named individually in the log per sync; the
10175/// `refused` count on the accepted line is always complete.
10176const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
10177
10178/// Per-outcome counts of one accepted `scope.sync`, for its log line.
10179#[derive(Debug, Default, PartialEq, Eq)]
10180struct ScopeOutcomeCounts {
10181    created: usize,
10182    replaced: usize,
10183    updated: usize,
10184    unchanged: usize,
10185    refused: usize,
10186}
10187
10188impl ScopeOutcomeCounts {
10189    fn of(results: &[ScopeRecordResult]) -> Self {
10190        let mut counts = Self::default();
10191        for result in results {
10192            let slot = match result.outcome {
10193                ScopeRecordOutcome::Created => &mut counts.created,
10194                ScopeRecordOutcome::Replaced => &mut counts.replaced,
10195                ScopeRecordOutcome::Updated => &mut counts.updated,
10196                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
10197                ScopeRecordOutcome::Refused => &mut counts.refused,
10198            };
10199            *slot += 1;
10200        }
10201        counts
10202    }
10203}
10204
10205#[cfg(test)]
10206mod scope_outcome_count_tests {
10207    use super::*;
10208
10209    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
10210        ScopeRecordResult {
10211            scope_ref: "r".to_string(),
10212            scope_epoch: 1,
10213            outcome,
10214            code: None,
10215            message: None,
10216            version: None,
10217            parent_state: None,
10218        }
10219    }
10220
10221    /// Each outcome lands in its own count, so a refused record can never be
10222    /// hidden inside the total the log already printed.
10223    #[test]
10224    fn every_outcome_is_counted_in_its_own_field() {
10225        let results = [
10226            result(ScopeRecordOutcome::Created),
10227            result(ScopeRecordOutcome::Created),
10228            result(ScopeRecordOutcome::Replaced),
10229            result(ScopeRecordOutcome::Updated),
10230            result(ScopeRecordOutcome::Unchanged),
10231            result(ScopeRecordOutcome::Refused),
10232            result(ScopeRecordOutcome::Refused),
10233            result(ScopeRecordOutcome::Refused),
10234        ];
10235        assert_eq!(
10236            ScopeOutcomeCounts::of(&results),
10237            ScopeOutcomeCounts {
10238                created: 2,
10239                replaced: 1,
10240                updated: 1,
10241                unchanged: 1,
10242                refused: 3,
10243            }
10244        );
10245    }
10246}
10247
10248/// Return whether a catalog change can create a newly violating live route.
10249/// Removing an attested claim is intentionally excluded: it makes fewer routes
10250/// forbidden and therefore must leave the existing route census untouched.
10251fn capability_census_trigger(
10252    old: Option<&CapabilityDeclarations>,
10253    new: Option<&CapabilityDeclarations>,
10254) -> bool {
10255    let old_provides = old
10256        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
10257        .unwrap_or_default();
10258    let old_denies = old
10259        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
10260        .unwrap_or_default();
10261    let new = new.cloned().unwrap_or(CapabilityDeclarations {
10262        provides: Vec::new(),
10263        requires: Vec::new(),
10264        must_never_reach: Vec::new(),
10265    });
10266
10267    new.provides
10268        .iter()
10269        .any(|capability| !old_provides.contains(capability))
10270        || new
10271            .must_never_reach
10272            .iter()
10273            .any(|capability| !old_denies.contains(capability))
10274}
10275
10276fn catalog_update_frozen_field_message(
10277    registered: &ModuleManifest,
10278    provides: &[ProviderRole],
10279) -> Option<String> {
10280    let old_has_provides = !registered.provides.is_empty();
10281    let new_has_provides = !provides.is_empty();
10282    if old_has_provides != new_has_provides {
10283        return Some(format!(
10284            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
10285            registered.module_id
10286        ));
10287    }
10288
10289    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
10290        return Some(format!(
10291            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
10292            registered.module_id
10293        ));
10294    }
10295
10296    let registered_concurrency = manifest_concurrency(registered);
10297    let mut candidate = registered.clone();
10298    candidate.provides = provides.to_vec();
10299    let candidate_concurrency = manifest_concurrency(&candidate);
10300    if candidate_concurrency != registered_concurrency {
10301        return Some(format!(
10302            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
10303            registered.module_id, registered_concurrency, candidate_concurrency
10304        ));
10305    }
10306
10307    // control_ops live beside the manifest in the HELLO body, not inside
10308    // ModuleManifest, so a provides-only catalog.update cannot change them.
10309    None
10310}
10311
10312fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
10313    manifest.provides.iter().any(is_routable_role)
10314}
10315
10316/// Returns the routable-provider concurrency subc should enforce for this manifest.
10317///
10318/// ToolProvider and ManagementSurface store their delivery concurrency directly.
10319/// InternalService has no role-specific concurrency field, so it retains the
10320/// existing ModuleManaged default for backward compatibility.
10321fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
10322    manifest
10323        .provides
10324        .iter()
10325        .find_map(|provider| match provider {
10326            ProviderRole::ToolProvider { concurrency, .. }
10327            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
10328            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
10329        })
10330        .unwrap_or(Concurrency::ModuleManaged)
10331}
10332
10333/// True when the manifest carries a ManagementSurface role whose concurrency
10334/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
10335/// bytes because the typed manifest deliberately erases that distinction: the
10336/// default exists for wire compatibility, and this probe exists so the default
10337/// stays observable. Any parse irregularity returns false -- the caller only
10338/// logs, and a malformed body already failed registration upstream.
10339fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
10340    let has_management_surface = manifest
10341        .provides
10342        .iter()
10343        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
10344    if !has_management_surface {
10345        return false;
10346    }
10347    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
10348        return false;
10349    };
10350    let Some(provides) = raw
10351        .get("manifest")
10352        .and_then(|manifest| manifest.get("provides"))
10353        .and_then(serde_json::Value::as_array)
10354    else {
10355        return false;
10356    };
10357    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
10358    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
10359    // against the management_surface_manifest_without_concurrency golden, not
10360    // recalled (the externally-tagged guess was this function's first bug).
10361    provides.iter().any(|role| {
10362        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
10363            && role.get("concurrency").is_none()
10364    })
10365}
10366
10367fn negotiate_version(peer_version: u8) -> Result<u8, String> {
10368    if peer_version != PROTOCOL_VERSION {
10369        return Err(format!(
10370            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
10371        ));
10372    }
10373    Ok(PROTOCOL_VERSION)
10374}
10375
10376fn pong(frame: &Frame) -> Result<Frame, RouterError> {
10377    Frame::build_with_version(
10378        response_version(frame),
10379        FrameType::Pong,
10380        frame.header.flags,
10381        0,
10382        0,
10383        frame.header.corr,
10384        Vec::new(),
10385    )
10386    .map_err(RouterError::FrameBuild)
10387}
10388
10389pub(crate) fn control_error_frame(
10390    frame: &Frame,
10391    code: &'static str,
10392    message: impl Into<String>,
10393) -> Result<Frame, RouterError> {
10394    control_error_body_frame(
10395        frame,
10396        ErrorBody {
10397            code: code.to_string(),
10398            message: message.into(),
10399            detail: None,
10400        },
10401    )
10402}
10403
10404/// A channel-0 ERROR for a refusal the daemon decided itself: every
10405/// route.open refusal, the drain `module_reloading` and scope-change/end
10406/// bodies, spawn-subscription and supervisor refusals. Carries the
10407/// daemon-origin bit, which is what lets a client treat `module_reloading`
10408/// as proof that its request never reached a module. Response, push,
10409/// goodbye and relay-request frames keep the plain control flags in their
10410/// own builders.
10411pub(crate) fn control_error_body_frame(
10412    frame: &Frame,
10413    error: ErrorBody,
10414) -> Result<Frame, RouterError> {
10415    control_error_frame_with_flags(frame, error, daemon_error_flags())
10416}
10417
10418/// A channel-0 ERROR relaying a module's own rejection body unchanged: a
10419/// refused route.bind answered as a route.open refusal, or a refused
10420/// module-control RPC. Plain control flags — the code and message are
10421/// module-chosen, so the frame must not claim daemon authorship, even when
10422/// the module picks a daemon code like `module_reloading`.
10423fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
10424    control_error_frame_with_flags(frame, error, control_flags())
10425}
10426
10427fn control_error_frame_with_flags(
10428    frame: &Frame,
10429    error: ErrorBody,
10430    flags: Flags,
10431) -> Result<Frame, RouterError> {
10432    let body = serde_json::to_vec(&error).map_err(|err| {
10433        RouterError::backend(
10434            0,
10435            frame.header.corr,
10436            format!("failed to encode control ERROR: {err}"),
10437        )
10438    })?;
10439
10440    Frame::build_with_version(
10441        response_version(frame),
10442        FrameType::Error,
10443        flags,
10444        0,
10445        0,
10446        frame.header.corr,
10447        body,
10448    )
10449    .map_err(RouterError::FrameBuild)
10450}
10451
10452pub(crate) fn control_response_body_frame<T: Serialize>(
10453    frame: &Frame,
10454    reply: &T,
10455    label: &'static str,
10456) -> Result<Frame, RouterError> {
10457    let body = serde_json::to_vec(reply).map_err(|err| {
10458        RouterError::backend(
10459            0,
10460            frame.header.corr,
10461            format!("failed to encode {label}: {err}"),
10462        )
10463    })?;
10464
10465    Frame::build_with_version(
10466        response_version(frame),
10467        FrameType::Response,
10468        control_flags(),
10469        0,
10470        0,
10471        frame.header.corr,
10472        body,
10473    )
10474    .map_err(RouterError::FrameBuild)
10475}
10476
10477/// Map a forwarding failure to the wire code a client sees.
10478///
10479/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
10480/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
10481/// chosen here decides whether a caller retries or gives up.
10482///
10483/// That makes attribution the load-bearing property, not merely having a code. A
10484/// permanent fault published as a retryable one produces a fleet-wide retry storm
10485/// against something that can never recover; a transient fault published as
10486/// permanent gives up on work that would have succeeded. Both look correct in a
10487/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
10488/// the mapping per variant rather than merely asserting that some code exists.
10489///
10490/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
10491/// two codes on the same side of the boundary passes it. Measured rather than
10492/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
10493/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
10494/// a test named for something else that happens to assert the string.
10495///
10496/// That accidental coverage is deliberately left alone rather than promoted to a
10497/// named test, because it guards a property this function does not promise.
10498/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
10499/// specific code within a class, so identity is free to change and only the
10500/// partition is a contract. Splitting it out would assert a guarantee nothing
10501/// depends on — and a suite that promises more than the code does is the harder
10502/// thing to correct later, because the next reader cannot tell which assertions
10503/// are load-bearing.
10504///
10505/// Pin identity here the moment a consumer branches on a specific code.
10506/// The spawn a HELLO would register as and the provider declarations it
10507/// requests, carried from the HELLO's checks to the point the registry
10508/// accepts it.
10509struct HelloAuthority {
10510    record: Arc<crate::spawn_authority::SpawnAuthorityRecord>,
10511    requested: ProviderDeclarations,
10512    /// Whether admitting this HELLO froze the spawn's requested declarations,
10513    /// so a refusal after admission undoes that freeze and no other.
10514    froze: std::sync::atomic::AtomicBool,
10515}
10516
10517fn forwarding_error_code(err: &ForwardingError) -> &'static str {
10518    match err {
10519        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
10520        ForwardingError::NoModuleConnection => "target_unavailable",
10521        ForwardingError::ModuleReloading { .. } => "module_reloading",
10522        ForwardingError::ClientRouteChannelExhausted { .. }
10523        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
10524        ForwardingError::StaleModuleEndpoint
10525        | ForwardingError::UnknownReservation { .. }
10526        | ForwardingError::ConnectionClosing { .. }
10527        | ForwardingError::ClientEgressClosed { .. }
10528        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
10529        // Only a swap candidate's registration can produce this, and it means
10530        // exactly what a second active HELLO for a live id means.
10531        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
10532        ForwardingError::RelayCorrelationExhausted
10533        | ForwardingError::RouteOpenBuild(_)
10534        | ForwardingError::Poisoned => "forwarding_error",
10535    }
10536}
10537
10538fn response_version(frame: &Frame) -> u8 {
10539    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
10540        frame.header.ver
10541    } else {
10542        PROTOCOL_VERSION
10543    }
10544}
10545
10546fn control_flags() -> Flags {
10547    Flags::new(false, Priority::Passive, false)
10548}
10549
10550/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
10551/// channel. The target is the module (a client never saw the route), so this
10552/// takes the module path: delivered late rather than dropped when the module's
10553/// queue is momentarily full, and never closing its connection.
10554fn send_goodbye_target_best_effort(
10555    counters: &DaemonCounters,
10556    target: &GoodbyeTarget,
10557    context: &'static str,
10558) {
10559    let Ok(frame) = Frame::build_with_version(
10560        target.negotiated_ver,
10561        FrameType::Goodbye,
10562        control_flags(),
10563        target.channel,
10564        target.epoch,
10565        0,
10566        Vec::new(),
10567    ) else {
10568        return;
10569    };
10570    crate::forwarding::send_module_route_goodbye(
10571        counters,
10572        &target.sink,
10573        frame,
10574        target.module_id.as_deref(),
10575        context,
10576    );
10577}
10578
10579fn route_close_reason_for_opener(
10580    reason: RouteCloseReason,
10581    accepts_close_reasons: Option<&[String]>,
10582) -> RouteCloseReason {
10583    if reason == RouteCloseReason::AuthorityChanged
10584        && !accepts_close_reasons
10585            .is_some_and(|reasons| reasons.iter().any(|reason| reason == "authority_changed"))
10586    {
10587        // Older readers forbid reopening for unknown reasons. Reload already
10588        // permits reopening and, like an authority change, requires a new bind.
10589        RouteCloseReason::Reload
10590    } else {
10591        reason
10592    }
10593}
10594
10595pub(crate) fn send_route_control_pushes(
10596    forwarding: &ForwardingTable,
10597    routes: Vec<EndpointRoute>,
10598    push: ClientControlPush,
10599) {
10600    let reason = match &push {
10601        ClientControlPush::RouteClosing { reason, .. }
10602        | ClientControlPush::RouteClosed { reason, .. } => *reason,
10603    };
10604    let mut targets: Vec<(GoodbyeTarget, RouteCloseReason, Vec<u16>)> = Vec::new();
10605    for route in routes {
10606        let recipient_reason =
10607            route_close_reason_for_opener(reason, route.accepts_close_reasons.as_deref());
10608        let target = route.goodbye_target;
10609        if let Some((existing, _, channels)) = targets.iter_mut().find(|(existing, reason, _)| {
10610            existing.connection_id == target.connection_id && *reason == recipient_reason
10611        }) {
10612            debug_assert_eq!(
10613                existing.negotiated_ver, target.negotiated_ver,
10614                "one connection cannot negotiate multiple frame versions"
10615            );
10616            if !channels.contains(&target.channel) {
10617                channels.push(target.channel);
10618            }
10619            continue;
10620        }
10621        let channel = target.channel;
10622        targets.push((target, recipient_reason, vec![channel]));
10623    }
10624    for (target, recipient_reason, mut channels) in targets {
10625        channels.sort_unstable();
10626        let mut push = push.clone();
10627        match &mut push {
10628            ClientControlPush::RouteClosing {
10629                channels: covered,
10630                reason,
10631                ..
10632            }
10633            | ClientControlPush::RouteClosed {
10634                channels: covered,
10635                reason,
10636                ..
10637            } => {
10638                *covered = channels;
10639                *reason = recipient_reason;
10640            }
10641        }
10642        let body = match serde_json::to_vec(&push) {
10643            Ok(body) => body,
10644            Err(err) => {
10645                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
10646                continue;
10647            }
10648        };
10649        let frame = match Frame::build_with_version(
10650            target.negotiated_ver,
10651            FrameType::Push,
10652            control_flags(),
10653            0,
10654            0,
10655            0,
10656            body.clone(),
10657        ) {
10658            Ok(frame) => frame,
10659            Err(err) => {
10660                warn!(
10661                    route_channel = target.channel,
10662                    error = %err,
10663                    "failed to build route lifecycle control PUSH frame"
10664                );
10665                continue;
10666            }
10667        };
10668        if let Err(err) = target.sink.try_send(frame) {
10669            if target.close_on_delivery_failure() {
10670                warn!(
10671                    target_connection_id = target.connection_id.get(),
10672                    route_channel = target.channel,
10673                    error = %err,
10674                    "route lifecycle control PUSH was not delivered to client; closing target connection"
10675                );
10676                let _ = forwarding.escalate_client_delivery_failure(
10677                    target.connection_id,
10678                    target.channel,
10679                    target.epoch,
10680                    CloseReason::new(
10681                        "route_lifecycle_push_delivery_failed",
10682                        format!(
10683                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
10684                            target.channel
10685                        ),
10686                    ),
10687                    crate::forwarding::UndeliveredFrame {
10688                        module_id: target.module_id.as_deref(),
10689                        sink: &target.sink,
10690                    },
10691                );
10692            }
10693        }
10694    }
10695}
10696
10697#[cfg(test)]
10698mod tests {
10699    include!("control/agent_commands_tests.rs");
10700    include!("control/resource_revision_tests.rs");
10701    include!("control/route_open_recovery_tests.rs");
10702    #[tokio::test]
10703    async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
10704        let registry = Arc::new(Registry::default());
10705        let handler = ControlHandler::new(registry);
10706        assert_eq!(
10707            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10708            None
10709        );
10710        let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
10711        hello_via_sink(
10712            &handler,
10713            &ctx,
10714            &mut rx,
10715            hello_frame("daemon", PROTOCOL_VERSION, 1),
10716        )
10717        .await;
10718        assert_eq!(
10719            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10720            Some("daemon".into())
10721        );
10722        assert_eq!(
10723            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
10724            None
10725        );
10726        assert_eq!(
10727            handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
10728            None,
10729            "a registered module is not necessarily supervised"
10730        );
10731        assert_eq!(
10732            handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
10733            None,
10734            "aggregate provenance has no single target"
10735        );
10736        assert_eq!(
10737            handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
10738            None
10739        );
10740    }
10741
10742    mod peer_manifest {
10743        include!("peer_manifest_tests.rs");
10744    }
10745    mod catalog_declarations {
10746        include!("control/catalog_tests.rs");
10747    }
10748    mod activity_read {
10749        include!("activity_read_tests.rs");
10750    }
10751
10752    mod lifecycle_activity {
10753        include!("control_lifecycle_tests.rs");
10754    }
10755    #[cfg(unix)]
10756    #[tokio::test]
10757    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
10758        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
10759        let path = dir.join("subc.jsonc");
10760        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
10761            "program":"/bin/sleep","args":["60"],"protocol":"none",
10762            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
10763        }}}).to_string()).unwrap();
10764        let mut configured = crate::daemon_config::load(&path)
10765            .unwrap()
10766            .unwrap()
10767            .modules
10768            .pop()
10769            .unwrap();
10770        let registry = std::sync::Arc::new(crate::Registry::default());
10771        let handle = crate::SupervisorHandle::new();
10772        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
10773            .with_handle(handle.clone());
10774        let module = supervisor
10775            .supervise_configured_with_health(
10776                configured.module_spec(),
10777                true,
10778                configured.health.clone(),
10779                None,
10780                configured.restart,
10781            )
10782            .unwrap();
10783        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
10784        let before = module.status().unwrap().pid;
10785        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
10786        configured.health.cadence = std::time::Duration::from_secs(3600);
10787        let health_only = handler
10788            .reconcile_supervised_modules(
10789                &supervisor,
10790                vec![configured.clone()],
10791                false,
10792                Principal::Direct,
10793            )
10794            .await
10795            .unwrap();
10796        assert!(
10797            health_only.0.changed_pending_reload.is_empty(),
10798            "health policy is already applied live"
10799        );
10800        assert_eq!(module.status().unwrap().pid, before);
10801        assert_eq!(
10802            module.configuration().unwrap().1.http,
10803            configured.health.http
10804        );
10805        configured.args = vec!["61".into()];
10806        let launch = handler
10807            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
10808            .await
10809            .unwrap();
10810        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
10811        assert_eq!(
10812            module.status().unwrap().pid,
10813            before,
10814            "a launch edit is stored until reload"
10815        );
10816        module.drain().await.unwrap();
10817    }
10818    use cortexkit_test_support::ScratchDir;
10819    use std::{
10820        collections::BTreeMap,
10821        fmt,
10822        path::PathBuf,
10823        sync::{Arc, Mutex},
10824        time::Duration,
10825    };
10826
10827    use serde_json::{json, Value};
10828    use subc_protocol::{
10829        manifest::{
10830            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
10831            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
10832        },
10833        session::HealthStatus,
10834        FrameType,
10835    };
10836
10837    use super::*;
10838    use crate::{
10839        forwarding::{DataRoute, DataRouteState},
10840        registry::ChannelState,
10841        router::FrameSink,
10842        stderr_tail::DEFAULT_MAX_LINE_BYTES,
10843        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
10844        RouteCtx, Router,
10845    };
10846    use tokio::{
10847        sync::mpsc,
10848        time::{sleep, Instant},
10849    };
10850    use tracing::{
10851        field::{Field, Visit},
10852        Event, Subscriber,
10853    };
10854    use tracing_subscriber::{layer::Context, prelude::*, Layer};
10855
10856    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
10857    ///
10858    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
10859    /// is only populated for `tests/*.rs` integration test binaries -- this file
10860    /// compiles as part of the library target, which gets neither. The binary
10861    /// lives in the profile folder this test was built into,
10862    /// `<target-dir>/<profile>/fake-aft-stub`, which `cargo_profile_bin` finds
10863    /// from the test's own path.
10864    ///
10865    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
10866    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
10867    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
10868    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
10869    /// `NotFound`, which reads as a broken test rather than an unbuilt
10870    /// dependency -- so state the cause and the remedy instead. Deliberately a
10871    /// panic and not a silent skip: a test that quietly passes when it could not
10872    /// run is worse than one that fails, because it reports health it never
10873    /// verified.
10874    fn fake_aft_stub_path() -> PathBuf {
10875        let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
10876        assert!(
10877            path.exists(),
10878            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
10879             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
10880            path.display()
10881        );
10882        path
10883    }
10884
10885    /// Whether clients retry `code` in place: the predicate itself, never a copy
10886    /// of its set. A copied list breaks silently when a code is added to or
10887    /// removed from the real one, and a stale copy here would let exactly the
10888    /// failure this test exists to catch pass.
10889    fn client_retries(code: &str) -> bool {
10890        subc_protocol::error_codes::is_retryable_route_open(code)
10891    }
10892
10893    /// A code is not a label — clients branch on it, so publishing the wrong KIND
10894    /// of failure is worse than publishing none. A permanent fault dressed as
10895    /// retryable makes every client in the fleet retry forever against something
10896    /// that cannot recover; a transient fault dressed as permanent abandons work
10897    /// that would have succeeded.
10898    ///
10899    /// Asserting "a code exists" cannot catch either, because the string is free
10900    /// to say anything. This enumerates every variant and pins which side of the
10901    /// retry boundary it lands on, so a new variant must be classified here
10902    /// deliberately rather than inheriting whichever arm it was appended to.
10903    #[test]
10904    fn retryability_of_forwarding_codes_matches_the_failure() {
10905        // Transient by nature: the target is booting, reloading, or its endpoint
10906        // was swapped mid-flight. Retrying is how these resolve.
10907        let transient = [
10908            ForwardingError::NoModuleConnection,
10909            ForwardingError::ModuleReloading {
10910                module_id: "m".into(),
10911            },
10912            ForwardingError::StaleModuleEndpoint,
10913            ForwardingError::UnknownReservation {
10914                client_channel: 1,
10915                module_channel: 1,
10916            },
10917            ForwardingError::ConnectionClosing {
10918                connection_id: ConnectionId::new(1),
10919            },
10920            ForwardingError::ClientEgressClosed {
10921                connection_id: ConnectionId::new(1),
10922            },
10923            ForwardingError::ModuleEgressUnavailable {
10924                connection_id: ConnectionId::new(1),
10925            },
10926        ];
10927        for err in transient {
10928            let code = forwarding_error_code(&err);
10929            assert!(
10930                client_retries(code),
10931                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
10932            );
10933        }
10934
10935        // Not fixed by retrying. Channel and correlation exhaustion need the
10936        // caller to close routes, and a poisoned lock is a daemon that cannot
10937        // recover at all — the worst thing to advertise as retryable, since every
10938        // client would storm a daemon that will never answer.
10939        let permanent = [
10940            ForwardingError::ConnectionRoleConflict {
10941                connection_id: ConnectionId::new(1),
10942            },
10943            ForwardingError::ClientRouteChannelExhausted {
10944                connection_id: ConnectionId::new(1),
10945            },
10946            ForwardingError::ModuleRouteChannelExhausted {
10947                endpoint: ModuleEndpointId {
10948                    connection_id: ConnectionId::new(1),
10949                    generation: 1,
10950                },
10951            },
10952            ForwardingError::RelayCorrelationExhausted,
10953            ForwardingError::RouteOpenBuild("x".into()),
10954            ForwardingError::Poisoned,
10955        ];
10956        for err in permanent {
10957            let code = forwarding_error_code(&err);
10958            assert!(
10959                !client_retries(code),
10960                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
10961            );
10962        }
10963    }
10964
10965    /// The principal is the daemon's answer to "who is calling", and modules
10966    /// branch on it: aft gates bash on it, cerebellum gates browser control,
10967    /// plexus gates connector invocation. So a stamp is an authorization input in
10968    /// another process, not a label — and both possible answers SUCCEED, which is
10969    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
10970    /// first-party capability to something that never proved it; a supervised one
10971    /// stamped `Direct` silently strips a module of capability it is entitled to.
10972    ///
10973    /// Neither shows up in a test that only checks the bind succeeded. Before this
10974    /// test the only coverage was accidental —
10975    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
10976    /// stamped principal on its way past, so narrowing that wire-shape test to its
10977    /// stated subject would have deleted the last assertion on this value. It
10978    /// still asserts the stamp, which is now redundancy rather than the only
10979    /// guard: both fail under the same mutation, and this one names the reason.
10980    /// SCOPE: this handler's supervisor has spawned nothing, so
10981    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
10982    /// is unreachable here. Both assertions below are refusals, and a mutant that
10983    /// refuses everything would satisfy them.
10984    ///
10985    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
10986    /// spawns a supervised consumer, reads its live nonce, and asserts the module
10987    /// observed `principal.kind == "reserved"` carrying that module_id — verified
10988    /// at source rather than assumed, since a citation is a claim about another
10989    /// file and ages like one. Recorded because a harness that structurally
10990    /// cannot reach an arm reports "none" for that arm identically to one that
10991    /// covers it and found nothing.
10992    #[tokio::test]
10993    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
10994        let handler = ControlHandler::default();
10995        let frame =
10996            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
10997
10998        // Absent consumer_identity is the ordinary case: a human at a terminal, or
10999        // any process holding the connection file. Nothing was proved, so nothing
11000        // may be granted beyond the unattested floor.
11001        let stamped = handler
11002            .route_open_principal(ConnectionId::new(1), &frame, None)
11003            .unwrap()
11004            .unwrap();
11005        assert_eq!(
11006            stamped,
11007            Principal::Direct,
11008            "a caller that proved nothing must not be stamped as a supervised module"
11009        );
11010
11011        // A claimed module_id with a nonce no supervised child was given is a
11012        // forgery attempt, not a weaker caller: it must be REFUSED rather than
11013        // quietly demoted to Direct, or an impersonation attempt looks identical
11014        // to an ordinary unattested connection.
11015        let forged = handler
11016            .route_open_principal(
11017                ConnectionId::new(1),
11018                &frame,
11019                Some(ConsumerIdentity {
11020                    module_id: "aft".to_string(),
11021                    launch_nonce: "not-a-real-nonce".to_string(),
11022                }),
11023            )
11024            .unwrap();
11025        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
11026        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
11027    }
11028
11029    /// The test above hands `route_open_principal` an identity it built itself,
11030    /// which proves the stamping rule and nothing about where the identity comes
11031    /// from. The real producer is a wire body, and the two are joined by a serde
11032    /// field name that nothing else asserts.
11033    ///
11034    /// That join fails quietly in one specific way: an unrecognised key is simply
11035    /// absent after parsing, so a renamed or misspelled `consumer_identity`
11036    /// yields `None` and every supervised module silently drops to `Direct`.
11037    /// Capability-wise that is the safe direction, but it surfaces far from its
11038    /// cause — as a module mysteriously refused bash — and it would pass every
11039    /// test that builds its own input.
11040    ///
11041    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
11042    /// would break every client the moment the daemon gains a field, trading a
11043    /// quiet demotion for a hard refusal on additive change. Asserting the join
11044    /// instead means a rename breaks a test here rather than the fleet.
11045    #[test]
11046    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
11047        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
11048        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
11049        let ClientControlRequest::RouteOpen {
11050            consumer_identity, ..
11051        } = parsed
11052        else {
11053            panic!("route.open body must parse as RouteOpen");
11054        };
11055        assert_eq!(
11056            consumer_identity,
11057            Some(ConsumerIdentity {
11058                module_id: "aft".to_string(),
11059                launch_nonce: "n".to_string(),
11060            }),
11061            "the wire field name must reach the value route_open_principal reads"
11062        );
11063    }
11064
11065    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
11066        ModuleManifest::builder(module_id, "0.1.0")
11067            .protocol_ver(protocol_ver)
11068            .provides(vec![ProviderRole::ToolProvider {
11069                tools: vec![Tool {
11070                    name: "read".to_string(),
11071                    description: None,
11072                    execution_mode: ExecutionMode::Pure,
11073                    schema: json!({"type": "object"}),
11074                }],
11075                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
11076                concurrency: Concurrency::ModuleManaged,
11077                emits_push: true,
11078                sub_supervises: true,
11079            }])
11080            .build()
11081    }
11082
11083    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
11084        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
11085    }
11086
11087    fn hello_frame_with_control_ops(
11088        module_id: &str,
11089        protocol_ver: u8,
11090        corr: u64,
11091        control_ops: Option<Vec<String>>,
11092    ) -> Frame {
11093        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
11094    }
11095
11096    fn hello_frame_with_nonce(
11097        module_id: &str,
11098        protocol_ver: u8,
11099        corr: u64,
11100        launch_nonce: Option<&str>,
11101    ) -> Frame {
11102        hello_frame_full(
11103            module_id,
11104            protocol_ver,
11105            corr,
11106            None,
11107            launch_nonce.map(ToOwned::to_owned),
11108        )
11109    }
11110
11111    fn hello_frame_full(
11112        module_id: &str,
11113        protocol_ver: u8,
11114        corr: u64,
11115        control_ops: Option<Vec<String>>,
11116        launch_nonce: Option<String>,
11117    ) -> Frame {
11118        let body = serde_json::to_vec(&ModuleHelloBody {
11119            manifest: manifest(module_id, protocol_ver),
11120            protocol_ver,
11121            control_ops,
11122            launch_nonce,
11123        })
11124        .unwrap();
11125        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11126    }
11127
11128    fn hello_frame_with_rich_ui(
11129        module_id: &str,
11130        protocol_ver: u8,
11131        corr: u64,
11132        launch_nonce: Option<&str>,
11133        rich_ui: Value,
11134    ) -> Frame {
11135        let mut body = serde_json::to_value(ModuleHelloBody {
11136            manifest: manifest(module_id, protocol_ver),
11137            protocol_ver,
11138            control_ops: None,
11139            launch_nonce: launch_nonce.map(str::to_string),
11140        })
11141        .unwrap();
11142        body["manifest"]["rich_ui"] = rich_ui;
11143        Frame::build(
11144            FrameType::Hello,
11145            control_flags(),
11146            0,
11147            0,
11148            corr,
11149            serde_json::to_vec(&body).unwrap(),
11150        )
11151        .unwrap()
11152    }
11153
11154    fn non_routable_hello_frame_with_control_ops(
11155        module_id: &str,
11156        corr: u64,
11157        control_ops: Option<Vec<String>>,
11158    ) -> Frame {
11159        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11160        manifest.provides.clear();
11161        let body = serde_json::to_vec(&ModuleHelloBody {
11162            manifest,
11163            protocol_ver: PROTOCOL_VERSION,
11164            control_ops,
11165            launch_nonce: None,
11166        })
11167        .unwrap();
11168        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11169    }
11170
11171    fn capability_grammar_hello_frame(
11172        capabilities: Value,
11173        runtime_computed: Option<Value>,
11174        corr: u64,
11175    ) -> Frame {
11176        let mut body = serde_json::to_value(ModuleHelloBody {
11177            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
11178            protocol_ver: PROTOCOL_VERSION,
11179            control_ops: None,
11180            launch_nonce: None,
11181        })
11182        .expect("HELLO body serializes");
11183        body["manifest"]["capabilities"] = capabilities;
11184        if let Some(runtime_computed) = runtime_computed {
11185            body["runtime_computed"] = runtime_computed;
11186        }
11187        Frame::build(
11188            FrameType::Hello,
11189            control_flags(),
11190            0,
11191            0,
11192            corr,
11193            serde_json::to_vec(&body).expect("HELLO body reserializes"),
11194        )
11195        .expect("HELLO frame builds")
11196    }
11197
11198    fn channel_request(channel: u16, corr: u64) -> Frame {
11199        Frame::build(
11200            FrameType::Request,
11201            Flags::new(true, Priority::Interactive, false),
11202            channel,
11203            0,
11204            corr,
11205            b"opaque".to_vec(),
11206        )
11207        .unwrap()
11208    }
11209
11210    fn route_ctx(
11211        connection_id: ConnectionId,
11212    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11213        let (tx, rx) = mpsc::channel(8);
11214        (
11215            RouteCtx {
11216                connection_id,
11217                egress: FrameSink::new(tx),
11218            },
11219            rx,
11220        )
11221    }
11222
11223    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
11224        serde_json::from_slice(&frame.body).unwrap()
11225    }
11226
11227    /// Register a module over a connection that has a sink and return the
11228    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
11229    /// module's own sink rather than returning it as a reply, so the ack is
11230    /// taken off `rx` here and whatever the test reads next is what followed it.
11231    async fn hello_via_sink(
11232        handler: &ControlHandler,
11233        ctx: &RouteCtx,
11234        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11235        hello: Frame,
11236    ) -> Frame {
11237        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
11238        assert!(
11239            replies.is_empty(),
11240            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
11241        );
11242        let ack = rx
11243            .try_recv()
11244            .expect("HELLO_ACK is queued on the module sink")
11245            .frame;
11246        assert_eq!(ack.header.ty, FrameType::HelloAck);
11247        ack
11248    }
11249
11250    fn parse_error(frame: &Frame) -> Value {
11251        serde_json::from_slice(&frame.body).unwrap()
11252    }
11253
11254    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
11255        serde_json::from_slice(&frame.body).unwrap()
11256    }
11257
11258    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
11259        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
11260            route_channel,
11261            route_epoch: 0,
11262            kind,
11263        })
11264        .unwrap();
11265        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11266    }
11267
11268    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
11269        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
11270            module_id: module_id.to_string(),
11271        })
11272        .unwrap();
11273        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11274    }
11275
11276    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
11277        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
11278    }
11279
11280    // Exercise retired opener fields without putting them back in BindIdentity.
11281    fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
11282        let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
11283        let mut frame = route_open_frame(corr, module_id, root);
11284        let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
11285        body["identity"]["project_root"] = serde_json::json!(path.as_path());
11286        frame.body = serde_json::to_vec(&body).unwrap();
11287        frame
11288    }
11289
11290    fn route_open_frame_with_consumer_capabilities(
11291        corr: u64,
11292        module_id: &str,
11293        _project_root: ScratchDir,
11294        consumer_capabilities: Option<Vec<String>>,
11295    ) -> Frame {
11296        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11297            busy_refusals: None,
11298            target: RouteTarget::ToolProvider {
11299                module_id: module_id.to_string(),
11300            },
11301            identity: BindIdentity::new("unit".to_string(), "session".to_string()),
11302            consumer_identity: None,
11303            consumer_capabilities,
11304            role_versions: None,
11305            admission_facts: None,
11306            scope: None,
11307        })
11308        .unwrap();
11309        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11310    }
11311
11312    fn route_open_frame_with_role_versions(
11313        corr: u64,
11314        module_id: &str,
11315        _project_root: ScratchDir,
11316        role_versions: Option<BTreeMap<String, String>>,
11317    ) -> Frame {
11318        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11319            busy_refusals: None,
11320            target: RouteTarget::ToolProvider {
11321                module_id: module_id.to_string(),
11322            },
11323            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11324            consumer_identity: None,
11325            consumer_capabilities: None,
11326            role_versions,
11327            admission_facts: None,
11328            scope: None,
11329        })
11330        .unwrap();
11331        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11332    }
11333
11334    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
11335        entries
11336            .iter()
11337            .map(|(role, version)| (role.to_string(), version.to_string()))
11338            .collect()
11339    }
11340
11341    fn route_open_frame_with_admission_facts(
11342        corr: u64,
11343        module_id: &str,
11344        _project_root: ScratchDir,
11345        consumer_identity: Option<subc_control::ConsumerIdentity>,
11346        facts: Option<Value>,
11347    ) -> Frame {
11348        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11349            busy_refusals: None,
11350            target: RouteTarget::ToolProvider {
11351                module_id: module_id.to_string(),
11352            },
11353            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11354            consumer_identity,
11355            consumer_capabilities: None,
11356            role_versions: None,
11357            admission_facts: facts,
11358            scope: None,
11359        })
11360        .unwrap();
11361        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11362    }
11363
11364    #[derive(Clone, Default)]
11365    struct EventCapture {
11366        events: Arc<Mutex<Vec<CapturedEvent>>>,
11367    }
11368
11369    #[derive(Clone, Debug)]
11370    struct CapturedEvent {
11371        target: String,
11372        level: tracing::Level,
11373        fields: BTreeMap<String, String>,
11374    }
11375
11376    impl EventCapture {
11377        fn events(&self) -> Vec<CapturedEvent> {
11378            self.events.lock().unwrap().clone()
11379        }
11380    }
11381
11382    impl<S> Layer<S> for EventCapture
11383    where
11384        S: Subscriber,
11385    {
11386        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
11387            let mut visitor = EventFieldVisitor::default();
11388            event.record(&mut visitor);
11389            self.events.lock().unwrap().push(CapturedEvent {
11390                target: event.metadata().target().to_string(),
11391                level: *event.metadata().level(),
11392                fields: visitor.fields,
11393            });
11394        }
11395    }
11396
11397    #[derive(Default)]
11398    struct EventFieldVisitor {
11399        fields: BTreeMap<String, String>,
11400    }
11401
11402    impl Visit for EventFieldVisitor {
11403        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
11404            self.fields
11405                .insert(field.name().to_string(), format!("{value:?}"));
11406        }
11407    }
11408
11409    fn health_response(corr: u64, status: HealthStatus) -> Frame {
11410        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
11411            status,
11412            detail: Some("warming".to_string()),
11413            metrics: Some(json!({"queue_depth": 3})),
11414        })
11415        .unwrap();
11416        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11417    }
11418
11419    fn route_bind_ack(corr: u64) -> Frame {
11420        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
11421        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11422    }
11423
11424    fn unique_project_root(label: &str) -> ScratchDir {
11425        ScratchDir::new(label)
11426    }
11427
11428    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
11429        match parse_route_poll(frame) {
11430            ClientControlResponse::RoutePoll {
11431                status: None,
11432                live: Some(live),
11433                ..
11434            } => assert_eq!(live, expected_live),
11435            other => panic!("unexpected route.poll response: {other:?}"),
11436        }
11437    }
11438
11439    fn bind_liveness_route(
11440        registry: &Registry,
11441        forwarding: &ForwardingTable,
11442        module_id: &str,
11443    ) -> (RouteCtx, u16, u32) {
11444        let module_connection = ConnectionId::new(101);
11445        let client_connection = ConnectionId::new(202);
11446        let registration = registry
11447            .register_with_control_ops(
11448                manifest(module_id, PROTOCOL_VERSION),
11449                PROTOCOL_VERSION,
11450                module_connection,
11451                module_baseline_control_ops(),
11452            )
11453            .unwrap();
11454        let (module_tx, _module_rx) = mpsc::channel(8);
11455        let endpoint = forwarding
11456            .register_module_connection(
11457                module_connection,
11458                module_id.to_string(),
11459                PROTOCOL_VERSION,
11460                manifest_concurrency(&registration.manifest),
11461                FrameSink::new(module_tx),
11462            )
11463            .unwrap();
11464        let (client_ctx, _client_rx) = route_ctx(client_connection);
11465        let pending = forwarding
11466            .begin_route_bind_relay_for_test(
11467                client_connection,
11468                client_ctx.egress.clone(),
11469                1,
11470                module_id,
11471            )
11472            .unwrap();
11473        assert_eq!(pending.endpoint, endpoint);
11474        let route_channel = pending.client_channel;
11475        let route_epoch = pending.client_epoch;
11476        forwarding
11477            .complete_pending_relay(
11478                module_connection,
11479                pending.corr,
11480                RouteBindRelayOutcome::Accepted,
11481            )
11482            .unwrap();
11483        (client_ctx, route_channel, route_epoch)
11484    }
11485
11486    struct FakeProcessLiveness {
11487        live: Option<bool>,
11488    }
11489
11490    impl ModuleProcessLiveness for FakeProcessLiveness {
11491        fn process_live(&self, _module_id: &str) -> Option<bool> {
11492            self.live
11493        }
11494    }
11495
11496    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11497    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
11498    {
11499        let registry = Arc::new(Registry::default());
11500        let supervisor_handle = SupervisorHandle::new();
11501        let supervisor = Supervisor::new_for_test(
11502            Arc::clone(&registry),
11503            RestartPolicy::new(1, Duration::from_millis(10)),
11504        )
11505        .with_handle(supervisor_handle.clone());
11506        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
11507        let module = supervisor
11508            .spawn(ModuleSpec {
11509                script: None,
11510                carry_containment: false,
11511                module_id: "stderr-tail-wire".to_string(),
11512                program: fake_aft_stub_path(),
11513                args: Vec::new(),
11514                env: vec![
11515                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
11516                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
11517                ],
11518                reserved: false,
11519                reserved_prefixes: Vec::new(),
11520                protocol: ModuleProtocol::Subc,
11521                overlap: Default::default(),
11522            })
11523            .unwrap();
11524
11525        let deadline = Instant::now() + Duration::from_secs(5);
11526        loop {
11527            let tail = module.stderr_tail(None, None);
11528            if tail
11529                .entries
11530                .iter()
11531                .any(|entry| matches!(entry, TailEntry::ProcessStart))
11532                && tail.entries.iter().any(|entry| {
11533                    matches!(
11534                        entry,
11535                        TailEntry::Line {
11536                            truncated: true,
11537                            ..
11538                        }
11539                    )
11540                })
11541            {
11542                break;
11543            }
11544            assert!(
11545                Instant::now() < deadline,
11546                "module did not produce a truncated line and restart boundary: {tail:?}"
11547            );
11548            sleep(Duration::from_millis(10)).await;
11549        }
11550
11551        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11552        let request = ClientControlRequest::SupervisorStderrTail {
11553            module_id: "stderr-tail-wire".to_string(),
11554            max_lines: None,
11555            max_bytes: None,
11556        };
11557        let frame = Frame::build(
11558            FrameType::Request,
11559            control_flags(),
11560            0,
11561            0,
11562            1,
11563            serde_json::to_vec(&request).unwrap(),
11564        )
11565        .unwrap();
11566        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11567        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11568        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
11569            serde_json::from_slice(&responses[0].body).unwrap()
11570        else {
11571            panic!("expected supervisor.stderr_tail response");
11572        };
11573
11574        assert!(
11575            tail.entries
11576                .iter()
11577                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
11578            "the control response lost the restart boundary"
11579        );
11580        let Some(StderrTailEntry::Line {
11581            text,
11582            truncated,
11583            at_ms,
11584        }) = tail.entries.iter().find(|entry| {
11585            matches!(
11586                entry,
11587                StderrTailEntry::Line {
11588                    truncated: true,
11589                    ..
11590                }
11591            )
11592        })
11593        else {
11594            panic!("the control response lost the truncated line");
11595        };
11596        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
11597        assert!(*truncated);
11598        assert!(
11599            at_ms.is_some(),
11600            "the control response lost the line's capture time"
11601        );
11602    }
11603
11604    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
11605    /// read done on the worker thread would stall every other task until it
11606    /// finished; the read must run off the worker so this test's own task keeps
11607    /// running while the read is paused.
11608    #[tokio::test(flavor = "current_thread")]
11609    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
11610        let dir = ScratchDir::new("terminals-off-worker");
11611        let journal_path = dir.join("terminals.jsonl");
11612        let registry = Arc::new(Registry::default());
11613        let supervisor_handle = SupervisorHandle::new();
11614        let supervisor =
11615            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
11616                .with_handle(supervisor_handle.clone())
11617                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
11618        let module = supervisor
11619            .spawn(ModuleSpec {
11620                script: None,
11621                carry_containment: false,
11622                module_id: "terminal-off-worker".to_string(),
11623                program: fake_aft_stub_path(),
11624                args: Vec::new(),
11625                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11626                reserved: false,
11627                reserved_prefixes: Vec::new(),
11628                protocol: ModuleProtocol::Subc,
11629                overlap: Default::default(),
11630            })
11631            .unwrap();
11632        let deadline = Instant::now() + Duration::from_secs(5);
11633        while module.terminal_history().entries.len() != 2 {
11634            assert!(Instant::now() < deadline, "module did not record two exits");
11635            sleep(Duration::from_millis(10)).await;
11636        }
11637
11638        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
11639        let handler =
11640            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
11641        let frame = Frame::build(
11642            FrameType::Request,
11643            control_flags(),
11644            0,
11645            0,
11646            1,
11647            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
11648                module_id: "terminal-off-worker".to_string(),
11649            })
11650            .unwrap(),
11651        )
11652        .unwrap();
11653        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11654        let spawned_at = std::time::Instant::now();
11655        let read = tokio::spawn({
11656            let handler = Arc::clone(&handler);
11657            async move { handler.handle_control_frame(&ctx, frame).await }
11658        });
11659        // Waiting for the pause from a blocking thread keeps this task pending,
11660        // so the runtime's single worker is free to run the read task.
11661        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
11662            .await
11663            .unwrap()
11664            .expect("the history read reached its pause");
11665        let elapsed = spawned_at.elapsed();
11666        assert!(
11667            elapsed < Duration::from_secs(2) && !read.is_finished(),
11668            "this task could not run while the history read was paused \
11669             (resumed after {elapsed:?}, read finished: {})",
11670            read.is_finished()
11671        );
11672
11673        drop(release);
11674        let responses = read.await.unwrap().unwrap();
11675        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11676        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
11677            panic!("expected supervisor.terminals response");
11678        };
11679        assert_eq!(terminals.entries.len(), 2);
11680        assert_eq!(terminals.journal_skipped_lines, 0);
11681        assert_eq!(terminals.journal_read_errors, 0);
11682    }
11683
11684    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11685    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
11686        let registry = Arc::new(Registry::default());
11687        let supervisor_handle = SupervisorHandle::new();
11688        let supervisor =
11689            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
11690                .with_handle(supervisor_handle.clone());
11691        let module = supervisor
11692            .spawn(ModuleSpec {
11693                script: None,
11694                carry_containment: false,
11695                module_id: "terminal-golden".to_string(),
11696                program: fake_aft_stub_path(),
11697                args: Vec::new(),
11698                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11699                reserved: false,
11700                reserved_prefixes: Vec::new(),
11701                protocol: ModuleProtocol::Subc,
11702                overlap: Default::default(),
11703            })
11704            .unwrap();
11705
11706        let deadline = Instant::now() + Duration::from_secs(5);
11707        while module.terminal_history().entries.len() != 2 {
11708            assert!(
11709                Instant::now() < deadline,
11710                "module did not retain two terminal exits: {:?}",
11711                module.terminal_history()
11712            );
11713            sleep(Duration::from_millis(10)).await;
11714        }
11715
11716        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11717        let request = ClientControlRequest::SupervisorTerminals {
11718            module_id: "terminal-golden".to_string(),
11719        };
11720        let frame = Frame::build(
11721            FrameType::Request,
11722            control_flags(),
11723            0,
11724            0,
11725            1,
11726            serde_json::to_vec(&request).unwrap(),
11727        )
11728        .unwrap();
11729        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11730        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11731        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11732        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
11733            panic!("expected supervisor.terminals response");
11734        };
11735        assert_eq!(terminals.entries.len(), 2);
11736        assert_eq!(terminals.dropped, 0);
11737
11738        let mut rendered = serde_json::to_value(response).unwrap();
11739        // Wall-clock fields are the observation contract, but not stable fixture
11740        // bytes; normalize only them after the real handler has shaped the response.
11741        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
11742        for (index, entry) in rendered["entries"]
11743            .as_array_mut()
11744            .expect("terminal response entries array")
11745            .iter_mut()
11746            .enumerate()
11747        {
11748            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
11749        }
11750
11751        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11752            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
11753        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
11754        if std::env::var_os("UPDATE_GOLDEN").is_some() {
11755            std::fs::write(&golden_path, &serialized).unwrap();
11756        }
11757        let expected: Value =
11758            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
11759        assert_eq!(rendered, expected);
11760    }
11761
11762    #[test]
11763    fn hello_registers_manifest_and_returns_ack() {
11764        let registry = Arc::new(Registry::default());
11765        let handler = ControlHandler::new(Arc::clone(&registry));
11766        let conn = ConnectionId::new(1);
11767
11768        let responses = handler
11769            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
11770            .unwrap();
11771
11772        assert_eq!(responses.len(), 1);
11773        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11774        assert_eq!(responses[0].header.channel, 0);
11775        assert_eq!(responses[0].header.corr, 7);
11776        let ack = parse_ack(&responses[0]);
11777        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11778        assert!(ack
11779            .subc_capabilities
11780            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
11781        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
11782        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
11783        assert!(ack
11784            .subc_ops
11785            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
11786        assert!(ack
11787            .subc_ops
11788            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
11789        assert!(ack
11790            .subc_ops
11791            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS.to_string()));
11792
11793        let registration = registry.get_module("aft").unwrap().unwrap();
11794        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
11795        assert_eq!(registration.state, ChannelState::Active);
11796        assert_eq!(registration.connection_id, conn);
11797        assert_eq!(registration.control_ops, module_baseline_control_ops());
11798        assert!(registration.manifest.display_name.is_none());
11799    }
11800
11801    #[test]
11802    fn hello_display_name_refuses_invalid_manifest_by_field() {
11803        let registry = Arc::new(Registry::default());
11804        let handler = ControlHandler::new(Arc::clone(&registry));
11805        for (index, invalid) in [
11806            json!(""),
11807            json!("a".repeat(41)),
11808            json!(" Routing"),
11809            json!("a\nb"),
11810            json!("a\u{202e}b"),
11811            json!(7),
11812        ]
11813        .into_iter()
11814        .enumerate()
11815        {
11816            let module_id = format!("bad-display-name-{index}");
11817            let mut body = serde_json::to_value(ModuleHelloBody {
11818                manifest: manifest(&module_id, PROTOCOL_VERSION),
11819                protocol_ver: PROTOCOL_VERSION,
11820                control_ops: None,
11821                launch_nonce: None,
11822            })
11823            .unwrap();
11824            body["manifest"]["display_name"] = invalid;
11825            let hello = Frame::build(
11826                FrameType::Hello,
11827                control_flags(),
11828                0,
11829                0,
11830                7,
11831                serde_json::to_vec(&body).unwrap(),
11832            )
11833            .unwrap();
11834            let response = handler
11835                .handle_control(ConnectionId::new(100 + index as u64), hello)
11836                .unwrap();
11837            assert_eq!(response[0].header.ty, FrameType::Error);
11838            let error = parse_error(&response[0]);
11839            assert_eq!(error["code"], "invalid_manifest");
11840            assert!(error["message"].as_str().unwrap().contains("display_name"));
11841            assert!(registry.get_module(&module_id).unwrap().is_none());
11842        }
11843    }
11844
11845    #[test]
11846    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
11847        let invalid_identifiers = [
11848            ("case_change", "credentials-Provider/v1"),
11849            ("leading_zero", "credentials-provider/v01"),
11850            ("trailing_hyphen", "credentials-provider-/v1"),
11851            ("consecutive_hyphens", "credentials--provider/v1"),
11852            ("uppercase", "Credentials-provider/v1"),
11853            ("missing_v", "credentials-provider/1"),
11854            ("whitespace", "credentials provider/v1"),
11855            ("zero_version", "credentials-provider/v0"),
11856            ("out_of_range_version", "credentials-provider/v4294967296"),
11857            (
11858                "overlength_name",
11859                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
11860            ),
11861        ];
11862        let mut cases = invalid_identifiers
11863            .into_iter()
11864            .map(|(name, identifier)| {
11865                (
11866                    format!("identifier_{name}"),
11867                    "capabilities.provides[0]".to_string(),
11868                    identifier.to_string(),
11869                    json!({ "provides": [identifier] }),
11870                    None,
11871                )
11872            })
11873            .collect::<Vec<_>>();
11874        cases.extend([
11875            (
11876                "unknown_need".to_string(),
11877                "capabilities.requires[0].need".to_string(),
11878                "deferred".to_string(),
11879                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
11880                None,
11881            ),
11882            (
11883                "duplicate_provides".to_string(),
11884                "capabilities.provides[1]".to_string(),
11885                "credentials-provider/v1".to_string(),
11886                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
11887                None,
11888            ),
11889            (
11890                "duplicate_must_never_reach".to_string(),
11891                "capabilities.must_never_reach[1]".to_string(),
11892                "credentials-provider/v1".to_string(),
11893                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
11894                None,
11895            ),
11896            (
11897                "duplicate_requires_same_need".to_string(),
11898                "capabilities.requires[1]".to_string(),
11899                "credentials-provider/v1".to_string(),
11900                json!({ "requires": [
11901                    { "capability": "credentials-provider/v1", "need": "required" },
11902                    { "capability": "credentials-provider/v1", "need": "required" }
11903                ] }),
11904                None,
11905            ),
11906            (
11907                "duplicate_requires_conflicting_need".to_string(),
11908                "capabilities.requires[1]".to_string(),
11909                "credentials-provider/v1".to_string(),
11910                json!({ "requires": [
11911                    { "capability": "credentials-provider/v1", "need": "required" },
11912                    { "capability": "credentials-provider/v1", "need": "optional" }
11913                ] }),
11914                None,
11915            ),
11916            (
11917                "capabilities_root_pointer".to_string(),
11918                "runtime_computed[0]".to_string(),
11919                "/capabilities".to_string(),
11920                json!({}),
11921                Some(json!(["/capabilities"])),
11922            ),
11923            (
11924                "capabilities_descendant_pointer".to_string(),
11925                "runtime_computed[0]".to_string(),
11926                "/capabilities/provides".to_string(),
11927                json!({}),
11928                Some(json!(["/capabilities/provides"])),
11929            ),
11930            (
11931                "malformed_pointer_without_leading_slash".to_string(),
11932                "runtime_computed[0]".to_string(),
11933                "capabilities".to_string(),
11934                json!({}),
11935                Some(json!(["capabilities"])),
11936            ),
11937            (
11938                "malformed_pointer_escape".to_string(),
11939                "runtime_computed[0]".to_string(),
11940                "/roles/~2/tools".to_string(),
11941                json!({}),
11942                Some(json!(["/roles/~2/tools"])),
11943            ),
11944            (
11945                "unknown_capabilities_field".to_string(),
11946                "capabilities.future".to_string(),
11947                "<array>".to_string(),
11948                json!({ "future": [] }),
11949                None,
11950            ),
11951        ]);
11952
11953        for (index, (name, field, value, capabilities, runtime_computed)) in
11954            cases.into_iter().enumerate()
11955        {
11956            let registry = Arc::new(Registry::default());
11957            let handler = ControlHandler::new(Arc::clone(&registry));
11958            let response = handler
11959                .handle_control(
11960                    ConnectionId::new((index + 1) as u64),
11961                    capability_grammar_hello_frame(
11962                        capabilities,
11963                        runtime_computed,
11964                        index as u64 + 1,
11965                    ),
11966                )
11967                .expect("invalid HELLO returns a refusal");
11968
11969            assert_eq!(response.len(), 1, "{name} must emit one refusal");
11970            let error = parse_error(&response[0]);
11971            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
11972            let message = error["message"]
11973                .as_str()
11974                .expect("error message is a string");
11975            assert!(
11976                message.contains(&field),
11977                "{name}: field missing from {message}"
11978            );
11979            assert!(
11980                message.contains(&value),
11981                "{name}: value missing from {message}"
11982            );
11983            assert_eq!(
11984                registry
11985                    .active_registration_count()
11986                    .expect("registry reads"),
11987                0,
11988                "{name}: refused HELLO must not create a catalog entry"
11989            );
11990        }
11991    }
11992
11993    #[test]
11994    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
11995        let registry = Arc::new(Registry::default());
11996        let handler = ControlHandler::new(Arc::clone(&registry));
11997        let capabilities = json!({
11998            "provides": ["credentials-provider/v1"],
11999            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
12000            "must_never_reach": ["federation-transport/v1"]
12001        });
12002        let response = handler
12003            .handle_control(
12004                ConnectionId::new(99),
12005                capability_grammar_hello_frame(
12006                    capabilities.clone(),
12007                    Some(json!(["/roles/0/tools"])),
12008                    99,
12009                ),
12010            )
12011            .expect("valid HELLO registers");
12012        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12013
12014        let request = Frame::build(
12015            FrameType::Request,
12016            control_flags(),
12017            0,
12018            0,
12019            100,
12020            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12021                .expect("catalog request serializes"),
12022        )
12023        .expect("catalog request frame builds");
12024        let response = handler
12025            .handle_catalog_list(request, None)
12026            .expect("catalog list succeeds");
12027        let ClientControlResponse::CatalogList { modules, .. } =
12028            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
12029        else {
12030            panic!("catalog request must return catalog.list");
12031        };
12032        assert_eq!(modules.len(), 1);
12033        assert_eq!(
12034            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
12035            capabilities
12036        );
12037    }
12038
12039    #[test]
12040    fn catalog_list_mirrors_management_operation_description() {
12041        let registry = Arc::new(Registry::default());
12042        let handler = ControlHandler::new(Arc::clone(&registry));
12043        let description = "List managed records and return their identifiers and metadata.";
12044        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
12045        manifest.provides = vec![ProviderRole::ManagementSurface {
12046            operations: vec![ManagementOperation {
12047                name: "records.list".to_string(),
12048                kind: ManagementOperationKind::Query,
12049                description: Some(description.to_string()),
12050            }],
12051            config_schema: json!({"type": "object"}),
12052            observability: vec![ObservabilitySurface {
12053                name: "records.stats".to_string(),
12054                kind: ObservabilityKind::Snapshot,
12055            }],
12056            identity_scope: vec![IdentityScope::Project],
12057            concurrency: Concurrency::ModuleManaged,
12058        }];
12059        registry
12060            .register_with_control_ops(
12061                manifest,
12062                PROTOCOL_VERSION,
12063                ConnectionId::new(99),
12064                Vec::new(),
12065            )
12066            .expect("described management manifest registers");
12067
12068        let request = Frame::build(
12069            FrameType::Request,
12070            control_flags(),
12071            0,
12072            0,
12073            100,
12074            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12075                .expect("catalog request serializes"),
12076        )
12077        .expect("catalog request frame builds");
12078        let response = handler
12079            .handle_catalog_list(request, None)
12080            .expect("catalog list succeeds");
12081        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
12082        assert_eq!(
12083            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
12084            "catalog.list must preserve the declared operation description verbatim"
12085        );
12086    }
12087
12088    #[test]
12089    fn catalog_list_mirrors_registered_operation_delivery() {
12090        let registry = Arc::new(Registry::default());
12091        let handler = ControlHandler::new(Arc::clone(&registry));
12092        let delivery =
12093            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
12094        let mut body = serde_json::to_value(ModuleHelloBody {
12095            manifest: manifest("watch-management", PROTOCOL_VERSION),
12096            protocol_ver: PROTOCOL_VERSION,
12097            control_ops: None,
12098            launch_nonce: None,
12099        })
12100        .unwrap();
12101        body["manifest"]["provides"] = json!([{
12102            "role": "management_surface",
12103            "operations": [
12104                {"name": "status", "kind": "query"},
12105                {"name": "future", "kind": "query"},
12106                {"name": "reset", "kind": "mutate"}
12107            ],
12108            "config_schema": {}, "observability": [], "identity_scope": []
12109        }]);
12110        body["manifest"]["operation_delivery"] = delivery.clone();
12111        let hello = Frame::build(
12112            FrameType::Hello,
12113            control_flags(),
12114            0,
12115            0,
12116            99,
12117            serde_json::to_vec(&body).unwrap(),
12118        )
12119        .unwrap();
12120        let response = handler
12121            .handle_control(ConnectionId::new(99), hello)
12122            .unwrap();
12123        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12124
12125        let (_, registrations) = registry.list_modules().unwrap();
12126        let registered = &registrations[0].manifest;
12127        assert!(registered.is_watch_operation("status"));
12128        assert!(!registered.is_watch_operation("future"));
12129        assert!(!registered.is_watch_operation("reset"));
12130        assert!(!registered.is_watch_operation("missing"));
12131
12132        let request = Frame::build(
12133            FrameType::Request,
12134            control_flags(),
12135            0,
12136            0,
12137            100,
12138            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12139        )
12140        .unwrap();
12141        let response = handler.handle_catalog_list(request, None).unwrap();
12142        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
12143        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
12144        let ClientControlResponse::CatalogList { modules, .. } =
12145            serde_json::from_slice(&response[0].body).unwrap()
12146        else {
12147            panic!("expected catalog.list");
12148        };
12149        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
12150    }
12151
12152    #[test]
12153    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
12154        let registry = Arc::new(Registry::default());
12155        let supervisor = SupervisorHandle::new();
12156        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12157        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
12158        let descriptor = json!({
12159            "schema": "ckios.richui.manifest/1",
12160            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12161            "bytes": 4096,
12162            "chunks": 4
12163        });
12164        let connection = ConnectionId::new(201);
12165        let hello = handler
12166            .handle_control(
12167                connection,
12168                hello_frame_with_rich_ui(
12169                    "vault",
12170                    PROTOCOL_VERSION,
12171                    201,
12172                    Some("the-real-nonce"),
12173                    descriptor.clone(),
12174                ),
12175            )
12176            .unwrap();
12177        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12178
12179        let request =
12180            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
12181        let provides = registry
12182            .get_module_by_connection(connection)
12183            .unwrap()
12184            .unwrap()
12185            .manifest
12186            .provides;
12187        handler
12188            .handle_catalog_update(connection, request, provides, None, Some(false))
12189            .unwrap();
12190
12191        let request = Frame::build(
12192            FrameType::Request,
12193            control_flags(),
12194            0,
12195            0,
12196            203,
12197            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12198        )
12199        .unwrap();
12200        let response = handler.handle_catalog_list(request, None).unwrap();
12201        let ClientControlResponse::CatalogList { modules, .. } =
12202            serde_json::from_slice(&response[0].body).unwrap()
12203        else {
12204            panic!("expected catalog.list");
12205        };
12206        assert_eq!(
12207            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
12208            descriptor
12209        );
12210        assert!(modules[0].nonce_verified);
12211    }
12212
12213    fn hello_with_role_declarations(declarations: Value) -> Frame {
12214        let mut body = serde_json::to_value(ModuleHelloBody {
12215            manifest: manifest("role-test", PROTOCOL_VERSION),
12216            protocol_ver: PROTOCOL_VERSION,
12217            control_ops: None,
12218            launch_nonce: None,
12219        })
12220        .unwrap();
12221        body["manifest"]["role_declarations"] = declarations;
12222        Frame::build(
12223            FrameType::Hello,
12224            control_flags(),
12225            0,
12226            0,
12227            301,
12228            serde_json::to_vec(&body).unwrap(),
12229        )
12230        .unwrap()
12231    }
12232
12233    fn update_role_declarations_frame(declarations: Value) -> Frame {
12234        Frame::build(
12235            FrameType::Request,
12236            control_flags(),
12237            0,
12238            0,
12239            303,
12240            serde_json::to_vec(&json!({
12241                "op": "catalog.update_role_declarations",
12242                "role_declarations": declarations,
12243            }))
12244            .unwrap(),
12245        )
12246        .unwrap()
12247    }
12248
12249    #[tokio::test]
12250    async fn role_declarations_update_replaces_then_clears_catalog() {
12251        let registry = Arc::new(Registry::default());
12252        let handler = ControlHandler::new(Arc::clone(&registry));
12253        let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12254        let initial = json!({"media-provider/v1": {"engines": ["old"]}, "old/v1": {}});
12255        hello_via_sink(
12256            &handler,
12257            &ctx,
12258            &mut rx,
12259            hello_with_role_declarations(initial.clone()),
12260        )
12261        .await;
12262        assert_eq!(
12263            role_declarations_catalog(&handler).0["role_declarations"],
12264            initial
12265        );
12266        let before = registry.get_module("role-test").unwrap().unwrap();
12267        for declarations in [
12268            json!({"media-provider/v1": {"engines": ["new"]}}),
12269            json!({}),
12270        ] {
12271            let response = handler
12272                .handle_control_frame(&ctx, update_role_declarations_frame(declarations.clone()))
12273                .await
12274                .unwrap();
12275            assert_eq!(response[0].header.ty, FrameType::Response);
12276            assert_eq!(
12277                serde_json::from_slice::<Value>(&response[0].body).unwrap(),
12278                json!({"op": "catalog.update_role_declarations"})
12279            );
12280            let (raw, entry) = role_declarations_catalog(&handler);
12281            assert_eq!(
12282                serde_json::to_value(entry.role_declarations).unwrap(),
12283                declarations
12284            );
12285            if declarations.as_object().unwrap().is_empty() {
12286                assert!(raw.get("role_declarations").is_none());
12287            } else {
12288                assert_eq!(raw["role_declarations"], declarations);
12289            }
12290            let after = registry.get_module("role-test").unwrap().unwrap();
12291            let mut expected = before.manifest.clone();
12292            expected.role_declarations = serde_json::from_value(declarations).unwrap();
12293            assert_eq!(after.manifest, expected);
12294            assert_eq!(after.ready, before.ready);
12295        }
12296    }
12297
12298    #[tokio::test]
12299    async fn role_declarations_update_refuses_invalid_maps_without_changing_catalog() {
12300        let registry = Arc::new(Registry::default());
12301        let handler = ControlHandler::new(registry);
12302        let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12303        let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12304        hello_via_sink(
12305            &handler,
12306            &ctx,
12307            &mut rx,
12308            hello_with_role_declarations(initial.clone()),
12309        )
12310        .await;
12311        for (invalid, reason) in [
12312            (json!(null), "role_declarations must be an object"),
12313            (json!([]), "role_declarations must be an object"),
12314            (
12315                json!({"bad key": {}}),
12316                "role_declarations key must use capability-name grammar",
12317            ),
12318            (
12319                json!({"media-provider/v1": []}),
12320                "role_declarations values must be objects",
12321            ),
12322            (
12323                json!({"media-provider/v1": {"padding": "x".repeat(16384)}}),
12324                "role_declarations exceeds 16384 compact JSON bytes",
12325            ),
12326        ] {
12327            let response = handler
12328                .handle_control_frame(&ctx, update_role_declarations_frame(invalid))
12329                .await
12330                .unwrap();
12331            let error = parse_error(&response[0]);
12332            assert_eq!(error["code"], "invalid_role_declarations");
12333            assert_eq!(error["message"], reason);
12334            assert_eq!(
12335                role_declarations_catalog(&handler).0["role_declarations"],
12336                initial
12337            );
12338        }
12339    }
12340
12341    #[tokio::test]
12342    async fn role_declarations_update_is_owned_by_the_registered_connection() {
12343        let registry = Arc::new(Registry::default());
12344        let handler = ControlHandler::new(Arc::clone(&registry));
12345        let (owner, mut owner_rx) = route_ctx(ConnectionId::new(301));
12346        let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12347        hello_via_sink(
12348            &handler,
12349            &owner,
12350            &mut owner_rx,
12351            hello_with_role_declarations(initial.clone()),
12352        )
12353        .await;
12354        let (other, mut other_rx) = route_ctx(ConnectionId::new(302));
12355        let replacement = json!({"media-provider/v1": {"engines": ["other"]}});
12356        let response = handler
12357            .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12358            .await
12359            .unwrap();
12360        assert_eq!(parse_error(&response[0])["code"], "not_registered");
12361        hello_via_sink(
12362            &handler,
12363            &other,
12364            &mut other_rx,
12365            hello_frame("other", PROTOCOL_VERSION, 304),
12366        )
12367        .await;
12368        let response = handler
12369            .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12370            .await
12371            .unwrap();
12372        assert_eq!(response[0].header.ty, FrameType::Response);
12373        assert_eq!(
12374            serde_json::to_value(
12375                registry
12376                    .get_module("role-test")
12377                    .unwrap()
12378                    .unwrap()
12379                    .manifest
12380                    .role_declarations
12381            )
12382            .unwrap(),
12383            initial
12384        );
12385        assert_eq!(
12386            serde_json::to_value(
12387                registry
12388                    .get_module("other")
12389                    .unwrap()
12390                    .unwrap()
12391                    .manifest
12392                    .role_declarations
12393            )
12394            .unwrap(),
12395            replacement
12396        );
12397    }
12398
12399    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
12400        let request = Frame::build(
12401            FrameType::Request,
12402            control_flags(),
12403            0,
12404            0,
12405            302,
12406            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12407        )
12408        .unwrap();
12409        let response = handler.handle_catalog_list(request, None).unwrap();
12410        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
12411        let ClientControlResponse::CatalogList { mut modules, .. } =
12412            serde_json::from_slice(&response[0].body).unwrap()
12413        else {
12414            panic!("expected catalog.list");
12415        };
12416        (raw["modules"][0].clone(), modules.remove(0))
12417    }
12418
12419    #[test]
12420    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
12421        let registry = Arc::new(Registry::default());
12422        let handler = ControlHandler::new(Arc::clone(&registry));
12423        let declarations = json!({
12424            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
12425            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
12426        });
12427        let connection = ConnectionId::new(301);
12428        let response = handler
12429            .handle_control(
12430                connection,
12431                hello_with_role_declarations(declarations.clone()),
12432            )
12433            .unwrap();
12434        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12435        let registered = registry
12436            .get_module_by_connection(connection)
12437            .unwrap()
12438            .unwrap()
12439            .manifest;
12440        // Compared as JSON values, not bytes: object key order is not part of
12441        // the relay's promise, and workspace builds unify serde_json's
12442        // preserve_order feature, which changes the order a literal keeps.
12443        assert_eq!(
12444            serde_json::to_value(&registered.role_declarations).unwrap(),
12445            declarations
12446        );
12447        let (raw, entry) = role_declarations_catalog(&handler);
12448        assert_eq!(raw["role_declarations"], declarations);
12449        assert_eq!(entry.role_declarations, registered.role_declarations);
12450
12451        let request =
12452            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
12453        handler
12454            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
12455            .unwrap();
12456        let (raw, entry) = role_declarations_catalog(&handler);
12457        assert_eq!(raw["role_declarations"], declarations);
12458        assert_eq!(entry.role_declarations, registered.role_declarations);
12459    }
12460
12461    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
12462        let registry = Arc::new(Registry::default());
12463        let handler = ControlHandler::new(Arc::clone(&registry));
12464        let capture = EventCapture::default();
12465        let _subscriber =
12466            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12467        let response = handler
12468            .handle_control(
12469                ConnectionId::new(301),
12470                hello_with_role_declarations(declarations),
12471            )
12472            .unwrap();
12473        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12474        assert!(registry
12475            .get_module_by_connection(ConnectionId::new(301))
12476            .unwrap()
12477            .unwrap()
12478            .manifest
12479            .role_declarations
12480            .is_empty());
12481        let (raw, entry) = role_declarations_catalog(&handler);
12482        assert!(raw.get("role_declarations").is_none());
12483        assert!(entry.role_declarations.is_empty());
12484        let warnings: Vec<_> = capture
12485            .events()
12486            .into_iter()
12487            .filter(|event| {
12488                event.fields.get("message").map(String::as_str)
12489                    == Some("dropping whole role_declarations map from HELLO")
12490            })
12491            .collect();
12492        assert_eq!(warnings.len(), 1);
12493        assert_eq!(warnings[0].level, tracing::Level::WARN);
12494        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
12495        assert_eq!(warnings[0].fields["reason"], reason);
12496    }
12497
12498    #[test]
12499    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
12500        for value in [
12501            json!(null),
12502            json!(false),
12503            json!(0),
12504            json!("opaque"),
12505            json!([]),
12506        ] {
12507            assert_hello_drops_role_declarations_with_warning(
12508                json!({"routing/v1": {}, "broken/v1": value}),
12509                "role_declarations values must be objects",
12510            );
12511        }
12512        assert_hello_drops_role_declarations_with_warning(
12513            json!([]),
12514            "role_declarations must be an object",
12515        );
12516    }
12517
12518    #[test]
12519    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
12520        assert_hello_drops_role_declarations_with_warning(
12521            json!({"routing/v1": {}, "bad key": {}}),
12522            "role_declarations key must use capability-name grammar",
12523        );
12524    }
12525
12526    #[test]
12527    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
12528        assert_hello_drops_role_declarations_with_warning(
12529            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
12530            "role_declarations exceeds 16384 compact JSON bytes",
12531        );
12532    }
12533
12534    fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
12535        let mut body = serde_json::to_value(ModuleHelloBody {
12536            manifest: manifest("flow-ops", PROTOCOL_VERSION),
12537            protocol_ver: PROTOCOL_VERSION,
12538            control_ops: None,
12539            launch_nonce: None,
12540        })
12541        .unwrap();
12542        body["manifest"]["provides"] = json!([{
12543            "role": "management_surface",
12544            "operations": [
12545                {"name": "diagnostics.findings", "kind": "query"},
12546                {"name": "reset", "kind": "mutate"}
12547            ],
12548            "config_schema": {}, "observability": [], "identity_scope": []
12549        }]);
12550        body["manifest"]["capabilities"] = json!({ "provides": capabilities });
12551        if let Some(flow_operations) = flow_operations {
12552            body["manifest"]["flow_operations"] = flow_operations;
12553        }
12554        Frame::build(
12555            FrameType::Hello,
12556            control_flags(),
12557            0,
12558            0,
12559            401,
12560            serde_json::to_vec(&body).unwrap(),
12561        )
12562        .unwrap()
12563    }
12564
12565    #[test]
12566    fn catalog_list_mirrors_flow_operations_unchanged() {
12567        // Order is kept as declared (not sorted) and an empty list stays an
12568        // affirmative "none", distinct from an absent declaration.
12569        for (capability, flow_operations) in [
12570            ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
12571            ("call-modes/v1", json!(["diagnostics.findings"])),
12572            ("flow-scopes/v1", json!([])),
12573        ] {
12574            let registry = Arc::new(Registry::default());
12575            let handler = ControlHandler::new(Arc::clone(&registry));
12576            let connection = ConnectionId::new(401);
12577            let response = handler
12578                .handle_control(
12579                    connection,
12580                    hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
12581                )
12582                .unwrap();
12583            assert_eq!(response[0].header.ty, FrameType::HelloAck);
12584            let registered = registry
12585                .get_module_by_connection(connection)
12586                .unwrap()
12587                .unwrap()
12588                .manifest;
12589            assert_eq!(
12590                serde_json::to_value(&registered.flow_operations).unwrap(),
12591                flow_operations
12592            );
12593            let (raw, entry) = role_declarations_catalog(&handler);
12594            assert_eq!(raw["flow_operations"], flow_operations);
12595            assert_eq!(entry.flow_operations, registered.flow_operations);
12596        }
12597
12598        let registry = Arc::new(Registry::default());
12599        let handler = ControlHandler::new(Arc::clone(&registry));
12600        let response = handler
12601            .handle_control(
12602                ConnectionId::new(401),
12603                hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
12604            )
12605            .unwrap();
12606        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12607        let (raw, entry) = role_declarations_catalog(&handler);
12608        assert!(raw.get("flow_operations").is_none());
12609        assert_eq!(entry.flow_operations, None);
12610    }
12611
12612    #[test]
12613    fn operation_deferral_hello_accepts_valid_and_invalid_maps_and_catalog_relays_only_valid() {
12614        let oversized: serde_json::Map<String, Value> = (0..257)
12615            .map(|n| (format!("tool:t{n}"), json!(true)))
12616            .collect();
12617        for (capabilities, map, valid) in [
12618            (
12619                json!(["scope-federated/v1"]),
12620                json!({"tool:read":true,"management:reset":true}),
12621                true,
12622            ),
12623            (json!(["scope-federated/v1"]), json!({}), true),
12624            (json!([]), json!({"tool:read":true}), false),
12625            (
12626                json!(["scope-federated/v1"]),
12627                json!({"tool:valid":true,"tool:bad":false}),
12628                false,
12629            ),
12630            (
12631                json!(["scope-federated/v1"]),
12632                json!({"tool:valid":true,"management:missing":true}),
12633                false,
12634            ),
12635            (
12636                json!(["scope-federated/v1"]),
12637                Value::Object(oversized),
12638                false,
12639            ),
12640        ] {
12641            let capture = EventCapture::default();
12642            let _subscriber = tracing::subscriber::set_default(
12643                tracing_subscriber::registry().with(capture.clone()),
12644            );
12645            let registry = Arc::new(Registry::default());
12646            let handler = ControlHandler::new(Arc::clone(&registry));
12647            let hello = hello_with_flow_operations(capabilities, None);
12648            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12649            body["manifest"]["operation_deferral"] = map.clone();
12650            let hello = Frame::build(
12651                FrameType::Hello,
12652                control_flags(),
12653                0,
12654                0,
12655                401,
12656                serde_json::to_vec(&body).unwrap(),
12657            )
12658            .unwrap();
12659            let response = handler
12660                .handle_control(ConnectionId::new(401), hello)
12661                .unwrap();
12662            assert_eq!(response[0].header.ty, FrameType::HelloAck);
12663            let (raw, entry) = role_declarations_catalog(&handler);
12664            if valid {
12665                assert_eq!(raw["operation_deferral"], map);
12666                assert_eq!(
12667                    serde_json::to_value(entry.operation_deferral.unwrap()).unwrap(),
12668                    map
12669                );
12670            } else {
12671                assert!(entry.operation_deferral.is_none());
12672                assert!(raw.get("operation_deferral").is_none());
12673                assert_eq!(
12674                    capture
12675                        .events()
12676                        .iter()
12677                        .filter(|event| event.fields.get("message").map(String::as_str)
12678                            == Some("dropping operation_deferral from HELLO"))
12679                        .count(),
12680                    1
12681                );
12682            }
12683        }
12684    }
12685
12686    fn assert_hello_drops_flow_operations_with_warning(
12687        capabilities: Value,
12688        flow_operations: Value,
12689        reason: &str,
12690    ) {
12691        let registry = Arc::new(Registry::default());
12692        let handler = ControlHandler::new(Arc::clone(&registry));
12693        let capture = EventCapture::default();
12694        let _subscriber =
12695            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12696        let response = handler
12697            .handle_control(
12698                ConnectionId::new(401),
12699                hello_with_flow_operations(capabilities, Some(flow_operations)),
12700            )
12701            .unwrap();
12702        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12703        let registered = registry
12704            .get_module_by_connection(ConnectionId::new(401))
12705            .unwrap()
12706            .expect("the module still registers")
12707            .manifest;
12708        assert_eq!(registered.module_id, "flow-ops");
12709        assert_eq!(registered.flow_operations, None);
12710        let (raw, entry) = role_declarations_catalog(&handler);
12711        assert_eq!(raw["module_id"], "flow-ops");
12712        assert!(raw.get("flow_operations").is_none());
12713        assert_eq!(entry.flow_operations, None);
12714        let warnings: Vec<_> = capture
12715            .events()
12716            .into_iter()
12717            .filter(|event| {
12718                event.fields.get("message").map(String::as_str)
12719                    == Some("dropping flow_operations from HELLO")
12720            })
12721            .collect();
12722        assert_eq!(warnings.len(), 1);
12723        assert_eq!(warnings[0].level, tracing::Level::WARN);
12724        assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
12725        assert_eq!(warnings[0].fields["reason"], reason);
12726    }
12727
12728    #[test]
12729    fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
12730        assert_hello_drops_flow_operations_with_warning(
12731            json!(["flow-scopes/v1"]),
12732            json!(["diagnostics.findings", "missing"]),
12733            "flow_operations names 'missing', which is not a declared management operation",
12734        );
12735    }
12736
12737    #[test]
12738    fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
12739        assert_hello_drops_flow_operations_with_warning(
12740            json!([]),
12741            json!(["diagnostics.findings"]),
12742            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12743        );
12744        // A run-only capability does not admit flows either.
12745        assert_hello_drops_flow_operations_with_warning(
12746            json!(["agent-run-scopes/v1"]),
12747            json!([]),
12748            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12749        );
12750    }
12751
12752    #[test]
12753    fn hello_malformed_flow_operations_is_dropped_with_warning() {
12754        assert_hello_drops_flow_operations_with_warning(
12755            json!(["flow-scopes/v1"]),
12756            json!(["reset", "reset"]),
12757            "flow_operations names 'reset' more than once",
12758        );
12759        assert_hello_drops_flow_operations_with_warning(
12760            json!(["flow-scopes/v1"]),
12761            json!(vec!["reset"; 257]),
12762            "flow_operations lists 257 names, more than 256",
12763        );
12764        assert_hello_drops_flow_operations_with_warning(
12765            json!(["flow-scopes/v1"]),
12766            json!("reset"),
12767            "flow_operations must be an array of operation names",
12768        );
12769    }
12770
12771    #[test]
12772    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
12773        let registry = Arc::new(Registry::default());
12774        let handler = ControlHandler::new(registry.clone());
12775        let descriptor = json!({
12776            "schema": "ckios.richui.manifest/1",
12777            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12778            "bytes": 128,
12779            "chunks": 2
12780        });
12781        let malformed = [
12782            ("schema", json!("")),
12783            ("schema", json!("a".repeat(65))),
12784            ("schema", json!("Bad schema")),
12785            (
12786                "manifest_digest",
12787                json!(format!("sha256:{}", "A".repeat(64))),
12788            ),
12789            ("manifest_digest", json!("sha256:abc")),
12790            ("bytes", json!(0)),
12791            ("bytes", json!(16_777_217)),
12792            ("bytes", json!("128")),
12793            ("chunks", json!(0)),
12794            ("chunks", json!(1025)),
12795            ("chunks", json!("2")),
12796        ];
12797        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
12798            let mut invalid = descriptor.clone();
12799            invalid[field] = invalid_value;
12800            let response = handler
12801                .handle_control(
12802                    ConnectionId::new(210 + index as u64),
12803                    hello_frame_with_rich_ui(
12804                        &format!("bad-rich-ui-{index}"),
12805                        PROTOCOL_VERSION,
12806                        210 + index as u64,
12807                        None,
12808                        invalid,
12809                    ),
12810                )
12811                .unwrap();
12812            assert_eq!(response[0].header.ty, FrameType::Error);
12813            let error = parse_error(&response[0]);
12814            assert_eq!(error["code"], "invalid_manifest");
12815            assert!(error["message"]
12816                .as_str()
12817                .unwrap()
12818                .contains(&format!("rich_ui.{field}")));
12819            assert!(registry
12820                .get_module(&format!("bad-rich-ui-{index}"))
12821                .unwrap()
12822                .is_none());
12823        }
12824    }
12825
12826    #[test]
12827    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
12828        let registry = Arc::new(Registry::default());
12829        let handler = ControlHandler::new(Arc::clone(&registry));
12830        let connection = ConnectionId::new(219);
12831        registry
12832            .register_with_control_ops(
12833                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
12834                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
12835                        "bad schema",
12836                        format!("sha256:{}", "a".repeat(64)),
12837                        128,
12838                        2,
12839                    )))
12840                    .build(),
12841                PROTOCOL_VERSION,
12842                connection,
12843                Vec::new(),
12844            )
12845            .unwrap();
12846        let request =
12847            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
12848        let response = handler
12849            .handle_catalog_update(connection, request, Vec::new(), None, None)
12850            .unwrap();
12851        assert_eq!(response[0].header.ty, FrameType::Error);
12852        let error = parse_error(&response[0]);
12853        assert_eq!(error["code"], "invalid_manifest");
12854        assert!(error["message"]
12855            .as_str()
12856            .unwrap()
12857            .contains("rich_ui.schema"));
12858    }
12859
12860    #[test]
12861    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
12862        let registry = Arc::new(Registry::default());
12863        let supervisor = SupervisorHandle::new();
12864        supervisor.open_swap("provider", "candidate-nonce".to_string());
12865        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
12866        let connection = ConnectionId::new(218);
12867        let hello = handler
12868            .handle_control(
12869                connection,
12870                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
12871            )
12872            .unwrap();
12873        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12874        assert!(registry.list_modules().unwrap().1.is_empty());
12875
12876        registry.promote_candidate("provider").unwrap().unwrap();
12877        let request = Frame::build(
12878            FrameType::Request,
12879            control_flags(),
12880            0,
12881            0,
12882            219,
12883            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12884        )
12885        .unwrap();
12886        let response = handler.handle_catalog_list(request, None).unwrap();
12887        let ClientControlResponse::CatalogList { modules, .. } =
12888            serde_json::from_slice(&response[0].body).unwrap()
12889        else {
12890            panic!("expected catalog.list");
12891        };
12892        assert_eq!(modules.len(), 1);
12893        assert!(modules[0].nonce_verified);
12894    }
12895
12896    #[test]
12897    fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
12898        for provenance_json in [
12899            r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
12900            r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
12901        ] {
12902            let registry = Arc::new(Registry::default());
12903            let handler = ControlHandler::new(registry);
12904            let hello = Frame::build(
12905                FrameType::Hello,
12906                control_flags(),
12907                0,
12908                0,
12909                220,
12910                serde_json::to_vec(&ModuleHelloBody {
12911                    manifest: ModuleManifest::builder("built", "0.1.0")
12912                        .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
12913                        .build(),
12914                    protocol_ver: PROTOCOL_VERSION,
12915                    control_ops: None,
12916                    launch_nonce: None,
12917                })
12918                .unwrap(),
12919            )
12920            .unwrap();
12921            assert_eq!(
12922                handler
12923                    .handle_control(ConnectionId::new(220), hello)
12924                    .unwrap()[0]
12925                    .header
12926                    .ty,
12927                FrameType::HelloAck
12928            );
12929            let request = Frame::build(
12930                FrameType::Request,
12931                control_flags(),
12932                0,
12933                0,
12934                221,
12935                serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12936            )
12937            .unwrap();
12938            let response = handler.handle_catalog_list(request, None).unwrap();
12939            let ClientControlResponse::CatalogList { modules, .. } =
12940                serde_json::from_slice(&response[0].body).unwrap()
12941            else {
12942                panic!("expected catalog.list");
12943            };
12944            assert_eq!(modules.len(), 1);
12945            assert_eq!(
12946                serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
12947                format!(
12948                    r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
12949                )
12950                .as_bytes()
12951            );
12952        }
12953    }
12954
12955    #[test]
12956    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
12957        let registry = Arc::new(Registry::default());
12958        let handler = ControlHandler::new(Arc::clone(&registry));
12959        let hello = Frame::build(
12960            FrameType::Hello,
12961            control_flags(),
12962            0,
12963            0,
12964            220,
12965            serde_json::to_vec(&ModuleHelloBody {
12966                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
12967                protocol_ver: PROTOCOL_VERSION,
12968                control_ops: None,
12969                launch_nonce: Some("unverified".to_string()),
12970            })
12971            .unwrap(),
12972        )
12973        .unwrap();
12974        assert_eq!(
12975            handler
12976                .handle_control(ConnectionId::new(220), hello)
12977                .unwrap()[0]
12978                .header
12979                .ty,
12980            FrameType::HelloAck
12981        );
12982        let request = Frame::build(
12983            FrameType::Request,
12984            control_flags(),
12985            0,
12986            0,
12987            221,
12988            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12989        )
12990        .unwrap();
12991        let response = handler.handle_catalog_list(request, None).unwrap();
12992        let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
12993        assert!(wire["modules"][0].get("provenance").is_none());
12994        let ClientControlResponse::CatalogList { modules, .. } =
12995            serde_json::from_slice(&response[0].body).unwrap()
12996        else {
12997            panic!("expected catalog.list");
12998        };
12999        assert_eq!(
13000            serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
13001            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
13002        );
13003        assert!(!modules[0].nonce_verified);
13004        assert!(modules[0].provenance.is_none());
13005    }
13006
13007    // Preserve the legacy-field byte pins while discovery adds optional facts.
13008    fn catalog_legacy_projection(mut entry: CatalogEntry) -> CatalogEntry {
13009        assert_eq!(entry.first_party, Some(false));
13010        entry.incarnation = None;
13011        entry.spawn = None;
13012        entry.first_party = None;
13013        entry.origin = None;
13014        entry.requested_declarations_digest = None;
13015        entry.approved_declarations_digest = None;
13016        entry
13017    }
13018
13019    #[test]
13020    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
13021        let registry = Arc::new(Registry::default());
13022        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
13023            [("vault".to_string(), true), ("squatter".to_string(), true)],
13024            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13025        );
13026        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
13027        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13028            provides: vec!["credentials-provider/v1".to_string()],
13029            requires: Vec::new(),
13030            must_never_reach: Vec::new(),
13031        });
13032        let frame = Frame::build(
13033            FrameType::Hello,
13034            control_flags(),
13035            0,
13036            0,
13037            77,
13038            serde_json::to_vec(&ModuleHelloBody {
13039                manifest: squatter,
13040                protocol_ver: PROTOCOL_VERSION,
13041                control_ops: None,
13042                launch_nonce: None,
13043            })
13044            .expect("HELLO serializes"),
13045        )
13046        .expect("HELLO frame builds");
13047        let response = handler
13048            .handle_control(ConnectionId::new(77), frame)
13049            .expect("reserved claim receives a typed refusal");
13050        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13051        assert_eq!(
13052            registry
13053                .active_registration_count()
13054                .expect("registry reads"),
13055            0,
13056            "a reserved capability refusal must not leave a catalog entry"
13057        );
13058    }
13059
13060    #[test]
13061    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
13062        for settlement in ["timeout", "inconclusive", "drop"] {
13063            let breakers = RouteBindBreakers::default();
13064            let RouteBindAdmission::Admitted {
13065                guard: mut old,
13066                probe: false,
13067            } = breakers.admit("prov")
13068            else {
13069                panic!("ordinary relay admitted")
13070            };
13071            let RouteBindAdmission::Admitted {
13072                guard: mut opener, ..
13073            } = breakers.admit("prov")
13074            else {
13075                panic!("second relay admitted")
13076            };
13077            assert!(
13078                !opener
13079                    .record_timeout(1, Duration::ZERO)
13080                    .unwrap()
13081                    .reopened_after_probe
13082            );
13083            let RouteBindAdmission::Admitted {
13084                guard: mut probe,
13085                probe: true,
13086            } = breakers.admit("prov")
13087            else {
13088                panic!("one cooldown probe admitted")
13089            };
13090            match settlement {
13091                "timeout" => assert!(
13092                    !old.record_timeout(1, Duration::ZERO)
13093                        .unwrap()
13094                        .reopened_after_probe
13095                ),
13096                "inconclusive" => old.record_inconclusive(),
13097                "drop" => drop(old),
13098                _ => unreachable!(),
13099            }
13100            assert!(
13101                matches!(
13102                    breakers.admit("prov"),
13103                    RouteBindAdmission::Refused {
13104                        probe_in_flight: true,
13105                        ..
13106                    }
13107                ),
13108                "{settlement} of a pre-open relay cannot release the real probe"
13109            );
13110            assert!(
13111                probe
13112                    .record_timeout(1, Duration::ZERO)
13113                    .unwrap()
13114                    .reopened_after_probe
13115            );
13116            assert!(matches!(
13117                breakers.admit("prov"),
13118                RouteBindAdmission::Admitted { probe: true, .. }
13119            ));
13120        }
13121        let breakers = RouteBindBreakers::default();
13122        let admit = || match breakers.admit("prov") {
13123            RouteBindAdmission::Admitted { guard, .. } => guard,
13124            _ => panic!("relay admitted"),
13125        };
13126        admit().record_timeout(1, Duration::ZERO);
13127        let mut old_probe = admit();
13128        breakers.reset_for_new_module_connection("prov");
13129        admit().record_timeout(1, Duration::ZERO);
13130        let _new_probe = admit();
13131        old_probe.record_inconclusive();
13132        assert!(matches!(
13133            breakers.admit("prov"),
13134            RouteBindAdmission::Refused {
13135                probe_in_flight: true,
13136                ..
13137            }
13138        ));
13139    }
13140
13141    #[tokio::test]
13142    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
13143        for candidate in [false, true] {
13144            let registry = Arc::new(Registry::default());
13145            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
13146                [("vault".to_string(), true), ("squatter".to_string(), true)],
13147                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13148            );
13149            let conn = ConnectionId::new(77);
13150            let (ctx, mut rx) = route_ctx(conn);
13151            let initial = capability_manifest("squatter", &[], &[]);
13152            if candidate {
13153                registry
13154                    .register_candidate_with_control_ops(
13155                        initial.clone(),
13156                        PROTOCOL_VERSION,
13157                        conn,
13158                        module_baseline_control_ops(),
13159                    )
13160                    .unwrap();
13161                handler
13162                    .forwarding
13163                    .register_candidate_module_connection(
13164                        conn,
13165                        "squatter".to_string(),
13166                        PROTOCOL_VERSION,
13167                        manifest_concurrency(&initial),
13168                        ctx.egress.clone(),
13169                    )
13170                    .unwrap();
13171            } else {
13172                hello_via_sink(
13173                    &handler,
13174                    &ctx,
13175                    &mut rx,
13176                    hello_frame_with_manifest(initial.clone(), 1),
13177                )
13178                .await;
13179            }
13180            let response = handler
13181                .handle_control_frame(
13182                    &ctx,
13183                    catalog_update_with_capabilities_frame(
13184                        2,
13185                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
13186                            .capabilities
13187                            .unwrap(),
13188                    ),
13189                )
13190                .await
13191                .unwrap();
13192            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13193            assert_eq!(
13194                registry
13195                    .get_module_by_connection(conn)
13196                    .unwrap()
13197                    .unwrap()
13198                    .manifest,
13199                initial
13200            );
13201        }
13202    }
13203
13204    #[test]
13205    #[cfg(not(feature = "dev-desktop-approval"))]
13206    fn server_describe_default_build_advertises_no_config_capability() {
13207        let assert_default = |handler: &ControlHandler| {
13208            let request = Frame::build(
13209                FrameType::Request,
13210                control_flags(),
13211                0,
13212                0,
13213                79,
13214                serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13215            )
13216            .unwrap();
13217            let responses = handler.handle_server_describe(request).unwrap();
13218            assert_eq!(responses.len(), 1);
13219            let response: Value = serde_json::from_slice(&responses[0].body).unwrap();
13220            assert_eq!(response["op"], "server.describe");
13221            let capabilities = response["capabilities"]
13222                .as_array()
13223                .expect("capability array");
13224            assert!(
13225                !capabilities.is_empty(),
13226                "exercise an actual describe response, not an empty fixture"
13227            );
13228            assert!(
13229                capabilities
13230                    .iter()
13231                    .all(|capability| !capability.as_str().unwrap().starts_with("config.")),
13232                "a default build must not announce configuration authority: {capabilities:?}"
13233            );
13234        };
13235        assert_default(&ControlHandler::new(Arc::new(Registry::default())));
13236        #[cfg(unix)]
13237        {
13238            // A local store does not prove that project-membership validation,
13239            // active-scope usage tracking and approved-program launch selection
13240            // are connected. Defaults must not advertise that authority.
13241            let (_rig, _owner, _supervisor, handler, _spec) =
13242                crate::config_gate::runtime::RuntimeOwner::unactivated_supervisor_fixture_for_test(
13243                );
13244            assert_default(&handler);
13245        }
13246    }
13247
13248    #[test]
13249    fn server_describe_surfaces_required_capability_verdict_fields() {
13250        let registry = Arc::new(Registry::default());
13251        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
13252            [
13253                ("consumer".to_string(), true),
13254                ("provider".to_string(), false),
13255            ],
13256            BTreeMap::new(),
13257        );
13258        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13259        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13260            provides: Vec::new(),
13261            requires: vec![subc_protocol::manifest::CapabilityRequirement {
13262                capability: "credentials-provider/v1".to_string(),
13263                need: subc_protocol::manifest::CapabilityNeed::Required,
13264            }],
13265            must_never_reach: Vec::new(),
13266        });
13267        let hello = Frame::build(
13268            FrameType::Hello,
13269            control_flags(),
13270            0,
13271            0,
13272            78,
13273            serde_json::to_vec(&ModuleHelloBody {
13274                manifest: consumer,
13275                protocol_ver: PROTOCOL_VERSION,
13276                control_ops: None,
13277                launch_nonce: None,
13278            })
13279            .expect("HELLO serializes"),
13280        )
13281        .expect("HELLO frame builds");
13282        handler
13283            .handle_control(ConnectionId::new(78), hello)
13284            .expect("consumer registers");
13285        let describe = Frame::build(
13286            FrameType::Request,
13287            control_flags(),
13288            0,
13289            0,
13290            79,
13291            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
13292                .expect("request serializes"),
13293        )
13294        .expect("describe frame builds");
13295        let response = handler
13296            .handle_server_describe(describe)
13297            .expect("server.describe succeeds");
13298        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
13299        let requirement = &rendered["capability_requirements"][0];
13300        assert_eq!(requirement["consumer"], "consumer");
13301        assert_eq!(requirement["verdict"], "never_provided");
13302        assert_eq!(requirement["episode_seq"], 1);
13303        assert_eq!(requirement["config_satisfiable"], false);
13304        assert_eq!(requirement["runtime_available"], false);
13305        assert!(requirement["detail"]
13306            .as_str()
13307            .expect("detail string")
13308            .contains("credentials-provider/v1"));
13309    }
13310
13311    /// Register a consumer that requires `project-identity/v1` and the provider
13312    /// of it, optionally mark daemon shutdown as begun, then deregister the
13313    /// provider as its connection closes. Returns the `capability.requirement`
13314    /// events that deregistration logged and the verdict the evaluator then
13315    /// holds for the requirement (the one `route.open` refusals read).
13316    fn deregister_required_provider(
13317        daemon_shutdown_begun: bool,
13318    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
13319        const CAPABILITY: &str = "project-identity/v1";
13320        let registry = Arc::new(Registry::default());
13321        let daemon_shutdown = DaemonShutdownFlag::default();
13322        let handler = ControlHandler::new(Arc::clone(&registry))
13323            .with_capability_config(
13324                [
13325                    ("consumer".to_string(), true),
13326                    ("provider".to_string(), true),
13327                ],
13328                BTreeMap::new(),
13329            )
13330            .with_daemon_shutdown_flag(daemon_shutdown.clone());
13331        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13332        consumer.capabilities = Some(CapabilityDeclarations {
13333            provides: Vec::new(),
13334            requires: vec![subc_protocol::manifest::CapabilityRequirement {
13335                capability: CAPABILITY.to_string(),
13336                need: subc_protocol::manifest::CapabilityNeed::Required,
13337            }],
13338            must_never_reach: Vec::new(),
13339        });
13340        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
13341        let provider_conn = ConnectionId::new(182);
13342        for (conn, manifest, corr) in [
13343            (ConnectionId::new(181), consumer, 181),
13344            (provider_conn, provider, 182),
13345        ] {
13346            let hello = handler
13347                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
13348                .expect("module registers");
13349            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13350        }
13351        assert_eq!(
13352            handler.capability_evaluator.verdict("consumer", CAPABILITY),
13353            Some(CapabilityVerdict::Provided),
13354            "the registered provider must satisfy the requirement first"
13355        );
13356        if daemon_shutdown_begun {
13357            daemon_shutdown.set_for_test();
13358        }
13359
13360        let capture = EventCapture::default();
13361        let _subscriber =
13362            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13363        handler
13364            .cleanup_connection(provider_conn)
13365            .expect("provider deregisters");
13366        let events = capture
13367            .events()
13368            .into_iter()
13369            .filter(|event| {
13370                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
13371            })
13372            .collect();
13373        (
13374            events,
13375            handler.capability_evaluator.verdict("consumer", CAPABILITY),
13376        )
13377    }
13378
13379    #[test]
13380    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
13381        let (events, verdict) = deregister_required_provider(true);
13382        assert_eq!(
13383            verdict,
13384            Some(CapabilityVerdict::NeverProvided),
13385            "shutdown must not change the verdict route.open refusals read"
13386        );
13387        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13388        let event = &events[0];
13389        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
13390        assert_eq!(
13391            event.fields.get("cause").map(String::as_str),
13392            Some("\"daemon_shutdown\""),
13393            "{event:?}"
13394        );
13395        assert_eq!(
13396            event.fields.get("verdict").map(String::as_str),
13397            Some("\"never_provided\""),
13398            "{event:?}"
13399        );
13400    }
13401
13402    #[test]
13403    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
13404        let (events, verdict) = deregister_required_provider(false);
13405        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
13406        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13407        let event = &events[0];
13408        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
13409        assert_eq!(event.fields.get("cause"), None, "{event:?}");
13410        assert_eq!(
13411            event.fields.get("verdict").map(String::as_str),
13412            Some("\"never_provided\""),
13413            "{event:?}"
13414        );
13415    }
13416
13417    #[test]
13418    fn catalog_list_omits_capabilities_for_legacy_manifest() {
13419        let registry = Arc::new(Registry::default());
13420        let handler = ControlHandler::new(Arc::clone(&registry));
13421        let hello = handler
13422            .handle_control(
13423                ConnectionId::new(101),
13424                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
13425            )
13426            .expect("legacy HELLO registers");
13427        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13428
13429        let request = Frame::build(
13430            FrameType::Request,
13431            control_flags(),
13432            0,
13433            0,
13434            102,
13435            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
13436                .expect("catalog request serializes"),
13437        )
13438        .expect("catalog request frame builds");
13439        let response = handler
13440            .handle_catalog_list(request, None)
13441            .expect("catalog list succeeds");
13442        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
13443        assert!(
13444            body["modules"][0].get("capabilities").is_none(),
13445            "legacy manifest must retain an absent capabilities field on catalog.list"
13446        );
13447    }
13448
13449    #[test]
13450    fn hello_ack_omits_storage_when_no_storage_config() {
13451        let registry = Arc::new(Registry::default());
13452        let handler = ControlHandler::new(Arc::clone(&registry));
13453        let responses = handler
13454            .handle_control(
13455                ConnectionId::new(1),
13456                hello_frame("aft", PROTOCOL_VERSION, 7),
13457            )
13458            .unwrap();
13459        let ack = parse_ack(&responses[0]);
13460        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
13461        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
13462    }
13463
13464    #[tokio::test]
13465    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
13466        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
13467        let registry = Arc::new(Registry::default());
13468        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
13469        let responses = handler
13470            .handle_control(
13471                ConnectionId::new(1),
13472                hello_frame("aft", PROTOCOL_VERSION, 7),
13473            )
13474            .unwrap();
13475        let ack = parse_ack(&responses[0]);
13476        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
13477
13478        let described = handler
13479            .handle_control_frame(
13480                &route_ctx(ConnectionId::new(2)).0,
13481                Frame::build(
13482                    FrameType::Request,
13483                    control_flags(),
13484                    0,
13485                    0,
13486                    9,
13487                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13488                )
13489                .unwrap(),
13490            )
13491            .await
13492            .unwrap();
13493        let ClientControlResponse::ServerDescribe { machine_id, .. } =
13494            serde_json::from_slice(&described[0].body).unwrap()
13495        else {
13496            panic!("server.describe answered with another shape");
13497        };
13498        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
13499    }
13500
13501    #[test]
13502    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
13503        // With a central sqlite storage policy, each registering module gets its
13504        // own resolved descriptor in HELLO_ACK, keyed by its module id.
13505        let registry = Arc::new(Registry::default());
13506        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
13507            crate::daemon_config::StorageConfig::Sqlite {
13508                data_home: std::path::PathBuf::from("/data"),
13509            },
13510        ));
13511
13512        let responses = handler
13513            .handle_control(
13514                ConnectionId::new(1),
13515                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
13516            )
13517            .unwrap();
13518        let ack = parse_ack(&responses[0]);
13519        assert_eq!(
13520            ack.storage,
13521            Some(serde_json::json!({
13522                "module_id": "alfonso-routing",
13523                "storage_namespace": "default",
13524                "isolation": { "kind": "module" },
13525                "backend": {
13526                    "backend": "sqlite",
13527                    "path": "/data/cortexkit/alfonso-routing/store.db"
13528                }
13529            })),
13530            "the delivered descriptor is the module's own sqlite store path"
13531        );
13532    }
13533
13534    #[test]
13535    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
13536        let registry = Arc::new(Registry::default());
13537        let handler = ControlHandler::new(Arc::clone(&registry));
13538        let conn = ConnectionId::new(1);
13539        let responses = handler
13540            .handle_control(
13541                conn,
13542                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13543            )
13544            .unwrap();
13545        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
13546        let registration = registry.get_module("aft").unwrap().unwrap();
13547        assert_eq!(registration.control_ops, module_baseline_control_ops());
13548
13549        let frame =
13550            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
13551        assert!(handler
13552            .guard_module_control_op(&frame, "aft", "route.bind")
13553            .unwrap()
13554            .is_none());
13555        let error = handler
13556            .guard_module_control_op(&frame, "aft", "test.synthetic")
13557            .unwrap()
13558            .expect("synthetic ungranted op should be rejected");
13559        assert_eq!(error.header.ty, FrameType::Error);
13560        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
13561    }
13562
13563    #[test]
13564    fn hello_control_ops_some_adds_optional_grants() {
13565        let registry = Arc::new(Registry::default());
13566        let handler = ControlHandler::new(Arc::clone(&registry));
13567        handler
13568            .handle_control(
13569                ConnectionId::new(1),
13570                hello_frame_with_control_ops(
13571                    "aft",
13572                    PROTOCOL_VERSION,
13573                    7,
13574                    Some(vec![
13575                        "future.synthetic".to_string(),
13576                        "route.bind".to_string(),
13577                    ]),
13578                ),
13579            )
13580            .unwrap();
13581        let registration = registry.get_module("aft").unwrap().unwrap();
13582        assert_eq!(
13583            registration.control_ops,
13584            vec![
13585                "route.bind".to_string(),
13586                "route.status".to_string(),
13587                "future.synthetic".to_string(),
13588            ]
13589        );
13590        let frame =
13591            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
13592        assert!(handler
13593            .guard_module_control_op(&frame, "aft", "future.synthetic")
13594            .unwrap()
13595            .is_none());
13596    }
13597
13598    #[tokio::test]
13599    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
13600        let registry = Arc::new(Registry::default());
13601        let forwarding = Arc::new(ForwardingTable::default());
13602        let handler =
13603            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13604        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
13605        hello_via_sink(
13606            &handler,
13607            &module_ctx,
13608            &mut module_rx,
13609            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13610        )
13611        .await;
13612
13613        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
13614        let responses = handler
13615            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
13616            .await
13617            .unwrap();
13618        assert_eq!(responses.len(), 1);
13619        assert_eq!(responses[0].header.ty, FrameType::Error);
13620        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
13621        assert!(module_rx.try_recv().is_err());
13622    }
13623
13624    #[tokio::test]
13625    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
13626        let registry = Arc::new(Registry::default());
13627        let forwarding = Arc::new(ForwardingTable::default());
13628        let handler =
13629            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13630        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
13631        hello_via_sink(
13632            &handler,
13633            &module_ctx,
13634            &mut module_rx,
13635            hello_frame_with_control_ops(
13636                "aft",
13637                PROTOCOL_VERSION,
13638                7,
13639                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13640            ),
13641        )
13642        .await;
13643
13644        let project_root = unique_project_root("demux");
13645        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
13646        let route_handler = handler.clone();
13647        let route_task = tokio::spawn(async move {
13648            route_handler
13649                .handle_control_frame(
13650                    &route_client_ctx,
13651                    route_open_frame(100, "aft", project_root),
13652                )
13653                .await
13654                .unwrap()
13655        });
13656        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13657            .await
13658            .unwrap()
13659            .unwrap();
13660        assert!(matches!(
13661            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
13662            ModuleControlRequest::RouteBind { .. }
13663        ));
13664
13665        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
13666        let health_handler = handler.clone();
13667        let health_task = tokio::spawn(async move {
13668            health_handler
13669                .handle_control_frame(
13670                    &health_client_ctx,
13671                    supervisor_health_probe_frame(101, "aft"),
13672                )
13673                .await
13674                .unwrap()
13675        });
13676        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13677            .await
13678            .unwrap()
13679            .unwrap();
13680        assert_eq!(
13681            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13682            ModuleControlRequest::HealthCheck {}
13683        );
13684
13685        handler
13686            .handle_control_frame(
13687                &module_ctx,
13688                health_response(health_frame.header.corr, HealthStatus::Degraded),
13689            )
13690            .await
13691            .unwrap();
13692        let health_response = health_task.await.unwrap();
13693        assert_eq!(health_response.len(), 1);
13694        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13695            ClientControlResponse::SupervisorHealthProbe {
13696                module_id,
13697                status,
13698                detail,
13699                metrics,
13700            } => {
13701                assert_eq!(module_id, "aft");
13702                assert_eq!(status, HealthStatus::Degraded);
13703                assert_eq!(detail.as_deref(), Some("warming"));
13704                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
13705            }
13706            other => panic!("unexpected health response: {other:?}"),
13707        }
13708
13709        handler
13710            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13711            .await
13712            .unwrap();
13713        let route_response = route_task.await.unwrap();
13714        assert!(route_response.is_empty());
13715        let published = route_client_rx.recv().await.unwrap();
13716        assert!(matches!(
13717            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13718            ClientControlResponse::RouteOpen { .. }
13719        ));
13720    }
13721
13722    /// Start one `route.open` on `client_connection` and return its still-running
13723    /// handler task together with the `route.bind` the module received for it.
13724    /// The handler blocks until the module answers, so it has to run as a task
13725    /// while the test drives the module side.
13726    async fn relay_route_open(
13727        handler: &ControlHandler,
13728        client_connection: ConnectionId,
13729        client_egress: &FrameSink,
13730        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13731        corr: u64,
13732        module_id: &str,
13733        project_root_label: &str,
13734    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
13735        let ctx = RouteCtx {
13736            connection_id: client_connection,
13737            egress: client_egress.clone(),
13738        };
13739        let handler = handler.clone();
13740        let project_root = unique_project_root(project_root_label);
13741        let module_id = module_id.to_string();
13742        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
13743        let task = tokio::spawn(async move {
13744            let _guard = tracing::dispatcher::set_default(&dispatch);
13745            handler
13746                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
13747                .await
13748                .unwrap()
13749        });
13750        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13751            .await
13752            .expect("module receives the relayed route.bind")
13753            .expect("module egress is open");
13754        (task, bind.frame)
13755    }
13756
13757    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
13758        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
13759            ModuleControlRequest::RouteBind {
13760                route_channel,
13761                epoch,
13762                ..
13763            } => (route_channel, epoch),
13764            other => panic!("expected a route.bind request, got {other:?}"),
13765        }
13766    }
13767
13768    fn published_route(frame: &Frame) -> (u16, u32) {
13769        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
13770            ClientControlResponse::RouteOpen {
13771                route_channel,
13772                route_epoch,
13773                ..
13774            } => (route_channel, route_epoch),
13775            other => panic!("expected a route.open response, got {other:?}"),
13776        }
13777    }
13778
13779    /// Reproduction of a production outage. A client had `route.open`s in
13780    /// flight to a module and was already marked closing -- its egress had refused a
13781    /// module frame, so the daemon asked its connection to end -- while its sink
13782    /// was still open. When the module acked those binds, the daemon refused to
13783    /// commit a route for a closing client, and that refusal was returned from
13784    /// the MODULE connection's frame handler, where a router error that has no
13785    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
13786    /// the supervisor correctly did not respawn a clean exit, and every seat lost
13787    /// its tools for hours -- one client's teardown took down a connection
13788    /// carrying ~170 other routes.
13789    ///
13790    /// The window is opened here by calling the production path that opens it
13791    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
13792    /// state that matters is "in `closing_connections`, sink still open, relay
13793    /// still pending", and it lasts only from the close request until the
13794    /// connection loop reacts to it; a socket-level test can flood a client into
13795    /// that escalation but cannot pin the module's ack inside the window. Closing
13796    /// the socket instead takes the other path entirely -- connection teardown
13797    /// removes the pending relay under the same lock, so the ack finds nothing.
13798    #[tokio::test]
13799    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
13800        let registry = Arc::new(Registry::default());
13801        let forwarding = Arc::new(ForwardingTable::default());
13802        let handler =
13803            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13804
13805        let module_connection = ConnectionId::new(30);
13806        let (module_ctx, mut module_rx) = route_ctx(module_connection);
13807        hello_via_sink(
13808            &handler,
13809            &module_ctx,
13810            &mut module_rx,
13811            hello_frame("aft", PROTOCOL_VERSION, 7),
13812        )
13813        .await;
13814
13815        let dying_client = ConnectionId::new(31);
13816        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
13817
13818        // A published route on the dying client. The escalation below only marks
13819        // a connection closing for a route it has already published.
13820        let (first_task, first_bind) = relay_route_open(
13821            &handler,
13822            dying_client,
13823            &dying_ctx.egress,
13824            &mut module_rx,
13825            100,
13826            "aft",
13827            "closing-first",
13828        )
13829        .await;
13830        handler
13831            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
13832            .await
13833            .unwrap();
13834        assert!(first_task.await.unwrap().is_empty());
13835        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
13836
13837        // A second route.open from the same client, relayed and awaiting its ack.
13838        let (second_task, second_bind) = relay_route_open(
13839            &handler,
13840            dying_client,
13841            &dying_ctx.egress,
13842            &mut module_rx,
13843            101,
13844            "aft",
13845            "closing-second",
13846        )
13847        .await;
13848        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
13849
13850        // The window: the client is closing, its sink is still open, and its
13851        // second bind is still pending.
13852        assert!(forwarding
13853            .escalate_client_delivery_failure(
13854                dying_client,
13855                first_channel,
13856                first_epoch,
13857                CloseReason::new(
13858                    "module_to_client_delivery_failed",
13859                    "client egress refused a module frame",
13860                ),
13861                crate::forwarding::UndeliveredFrame {
13862                    module_id: None,
13863                    sink: &dying_ctx.egress,
13864                },
13865            )
13866            .unwrap());
13867        assert!(!dying_ctx.egress.is_closed());
13868
13869        // The frame that used to end the module connection.
13870        let ack = handler
13871            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
13872            .await;
13873        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13874        if module_loop_error.is_some() {
13875            // What the server's connection loop does with a router error that has
13876            // no ERROR-frame translation: end the connection, which releases the
13877            // module's registration and every route on it.
13878            handler.cleanup_connection(module_connection).unwrap();
13879        }
13880        // Read the module's next frame before opening the co-tenant's route, so
13881        // the GOODBYE assertion below is about THIS ack and not about later
13882        // traffic. `None` means the module was told nothing.
13883        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13884            .await
13885            .ok()
13886            .flatten();
13887
13888        // 1. The module connection is still registered.
13889        assert!(
13890            registry
13891                .get_module_by_connection(module_connection)
13892                .unwrap()
13893                .is_some(),
13894            "one client's closing connection ended the shared module connection: \
13895             {module_loop_error:?}"
13896        );
13897        // ...and still serving: another client can open and use a route on it.
13898        let cotenant = ConnectionId::new(32);
13899        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
13900        let (cotenant_task, cotenant_bind) = relay_route_open(
13901            &handler,
13902            cotenant,
13903            &cotenant_ctx.egress,
13904            &mut module_rx,
13905            102,
13906            "aft",
13907            "closing-cotenant",
13908        )
13909        .await;
13910        handler
13911            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
13912            .await
13913            .unwrap();
13914        assert!(cotenant_task.await.unwrap().is_empty());
13915        let (cotenant_channel, cotenant_epoch) =
13916            published_route(&cotenant_rx.recv().await.unwrap());
13917        assert!(matches!(
13918            forwarding
13919                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
13920                .unwrap(),
13921            DataRoute::Client(DataRouteState::Bound(_))
13922        ));
13923
13924        // 2. The module was told to drop the binding it created for the route
13925        //    that will never be published.
13926        let goodbye = post_ack_module_frame
13927            .expect("module receives a GOODBYE for the abandoned route channel");
13928        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13929        assert_eq!(goodbye.header.channel, abandoned_channel);
13930        assert_eq!(goodbye.header.epoch, abandoned_epoch);
13931
13932        // 3. The dying client received nothing: no route was ever published to
13933        //    it. Its route.open is answered as unavailable, which the connection
13934        //    loop would write to a socket that is already going away.
13935        assert!(dying_rx.try_recv().is_err());
13936        let second_response = second_task.await.unwrap();
13937        assert_eq!(second_response.len(), 1);
13938        assert_eq!(
13939            parse_error(&second_response[0])["code"],
13940            "target_unavailable"
13941        );
13942    }
13943
13944    /// One `catalog.list` as a client sees it: the generation and the listed
13945    /// module ids with their readiness.
13946    async fn catalog_snapshot(
13947        handler: &ControlHandler,
13948        client_ctx: &RouteCtx,
13949        corr: u64,
13950    ) -> (u64, Vec<(String, bool)>) {
13951        let body =
13952            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
13953        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
13954        let reply = handler
13955            .handle_control_frame(client_ctx, request)
13956            .await
13957            .unwrap()
13958            .pop()
13959            .unwrap();
13960        let ClientControlResponse::CatalogList {
13961            generation,
13962            modules,
13963            ..
13964        } = serde_json::from_slice(&reply.body).unwrap()
13965        else {
13966            panic!("not a catalog.list reply");
13967        };
13968        (
13969            generation,
13970            modules
13971                .into_iter()
13972                .map(|entry| (entry.module_id, entry.ready))
13973                .collect(),
13974        )
13975    }
13976
13977    /// A handler whose HELLO for module "aft" on connection 40 is parked
13978    /// between reserving its registry entry and installing its forwarding
13979    /// endpoint. Returns the handler, the two-party barrier (wait once to know
13980    /// the HELLO is parked, once more to release it), the HELLO's thread, and
13981    /// the module's context and receiver.
13982    fn hello_held_before_forwarding() -> (
13983        ControlHandler,
13984        Arc<std::sync::Barrier>,
13985        std::thread::JoinHandle<Vec<Frame>>,
13986        RouteCtx,
13987        mpsc::Receiver<crate::router::OutboundFrame>,
13988    ) {
13989        let hold = Arc::new(std::sync::Barrier::new(2));
13990        let handler = ControlHandler::with_forwarding(
13991            Arc::new(Registry::default()),
13992            Arc::new(ForwardingTable::default()),
13993        )
13994        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
13995        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
13996        let hello = {
13997            let handler = handler.clone();
13998            let module_ctx = module_ctx.clone();
13999            std::thread::spawn(move || {
14000                handler
14001                    .handle_hello(
14002                        module_ctx.connection_id,
14003                        Some(module_ctx.egress.clone()),
14004                        hello_frame("aft", PROTOCOL_VERSION, 7),
14005                    )
14006                    .unwrap()
14007            })
14008        };
14009        (handler, hold, hello, module_ctx, module_rx)
14010    }
14011
14012    /// If a lookup can acquire the registry while publication is parked, let
14013    /// the lookup finish before releasing HELLO. Otherwise release HELLO only
14014    /// after the lookup has tried the lock. Both schedules avoid timing guesses
14015    /// and return the actual lookup result, including an unpublished snapshot
14016    /// when the publication lock is missing.
14017    fn lookup_during_hello_publication<T: Send + 'static>(
14018        registry: &Registry,
14019        hold: &std::sync::Barrier,
14020        lookup: impl FnOnce() -> T + Send + 'static,
14021    ) -> T {
14022        let attempted = registry.observe_next_lock_attempt();
14023        let reader = std::thread::spawn(lookup);
14024        if attempted.recv().unwrap() {
14025            let result = reader.join().unwrap();
14026            hold.wait();
14027            result
14028        } else {
14029            hold.wait();
14030            reader.join().unwrap()
14031        }
14032    }
14033
14034    #[test]
14035    fn catalog_read_after_hello_ack_waits_for_publication() {
14036        let registry = Arc::new(Registry::default());
14037        let hold = Arc::new(std::sync::Barrier::new(2));
14038        let mut handler = ControlHandler::with_forwarding(
14039            Arc::clone(&registry),
14040            Arc::new(ForwardingTable::default()),
14041        );
14042        handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
14043        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14044        let hello = {
14045            let handler = handler.clone();
14046            std::thread::spawn(move || {
14047                handler
14048                    .handle_hello(
14049                        module_ctx.connection_id,
14050                        Some(module_ctx.egress),
14051                        hello_frame("aft", PROTOCOL_VERSION, 7),
14052                    )
14053                    .unwrap()
14054            })
14055        };
14056        hold.wait();
14057        assert_eq!(
14058            module_rx.try_recv().unwrap().frame.header.ty,
14059            FrameType::HelloAck
14060        );
14061        let replies = lookup_during_hello_publication(&registry, &hold, move || {
14062            // A different connection can issue catalog.list as soon as the
14063            // module has read its ACK, even while HELLO has not returned.
14064            let request = Frame::build(
14065                FrameType::Request,
14066                control_flags(),
14067                0,
14068                0,
14069                300,
14070                serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
14071            )
14072            .unwrap();
14073            handler.handle_catalog_list(request, None).unwrap()
14074        });
14075        assert!(hello.join().unwrap().is_empty());
14076        let ClientControlResponse::CatalogList {
14077            generation,
14078            modules,
14079            ..
14080        } = serde_json::from_slice(&replies[0].body).unwrap()
14081        else {
14082            panic!("not a catalog.list reply");
14083        };
14084        assert_eq!(
14085            modules
14086                .iter()
14087                .map(|entry| entry.module_id.as_str())
14088                .collect::<Vec<_>>(),
14089            vec!["aft"]
14090        );
14091        assert_eq!(generation, 1);
14092    }
14093
14094    /// While a HELLO has reserved its registry entry but its forwarding
14095    /// endpoint is not installed yet, `route.open` to the module can only be
14096    /// refused. `catalog.list` must not list the module in that window: a
14097    /// client that opens a route as soon as it sees a module listed and ready
14098    /// would be refused for a module the daemon had just advertised. The
14099    /// refusal in the window stays the retryable `target_unavailable`, never
14100    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
14101    /// Once the HELLO completes, the module is listed and a route to it opens.
14102    #[tokio::test]
14103    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
14104        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
14105        let client = ConnectionId::new(41);
14106        let (client_ctx, mut client_rx) = route_ctx(client);
14107
14108        hold.wait();
14109        // Parked: the registry holds "aft", forwarding has no endpoint for it.
14110        assert!(handler.registry.get_module("aft").unwrap().is_some());
14111        assert!(!handler
14112            .forwarding
14113            .has_live_module_connection("aft")
14114            .unwrap());
14115        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
14116        assert!(
14117            listed.is_empty(),
14118            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
14119        );
14120        let refused = handler
14121            .handle_control_frame(
14122                &client_ctx,
14123                route_open_frame(301, "aft", unique_project_root("listed-window")),
14124            )
14125            .await
14126            .unwrap();
14127        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
14128        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
14129        let duplicate = handler
14130            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
14131            .await
14132            .unwrap();
14133        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14134
14135        hold.wait();
14136        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14137        let ack = module_rx.try_recv().unwrap().frame;
14138        assert_eq!(ack.header.ty, FrameType::HelloAck);
14139        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14140        assert_eq!(listed, vec![("aft".to_string(), true)]);
14141
14142        let (open_task, bind) = relay_route_open(
14143            &handler,
14144            client,
14145            &client_ctx.egress,
14146            &mut module_rx,
14147            303,
14148            "aft",
14149            "listed-after",
14150        )
14151        .await;
14152        assert!(handler
14153            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14154            .await
14155            .unwrap()
14156            .is_empty());
14157        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
14158        published_route(&client_rx.try_recv().unwrap().frame);
14159    }
14160
14161    /// A catalog watcher keys on the generation: when it changes, it re-reads
14162    /// the listing. So the generation must advance exactly when the module
14163    /// appears. If it advanced while the module was still hidden, a watcher
14164    /// would read the new generation without the module, and the module would
14165    /// later appear under that same generation with nothing telling the
14166    /// watcher to look again.
14167    #[tokio::test]
14168    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
14169        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
14170        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14171        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14172
14173        hold.wait();
14174        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
14175        assert!(listed.is_empty());
14176        assert_eq!(
14177            during, before,
14178            "the generation moved while the module was not listed"
14179        );
14180
14181        hold.wait();
14182        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14183        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14184        assert_eq!(listed, vec![("aft".to_string(), true)]);
14185        assert_eq!(
14186            after,
14187            before + 1,
14188            "the module appeared without exactly one generation bump"
14189        );
14190    }
14191
14192    /// A HELLO whose forwarding endpoint cannot be installed (here the
14193    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
14194    /// refused and leaves nothing behind: no registry entry, no catalog entry,
14195    /// and the catalog generation where it was.
14196    #[tokio::test]
14197    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
14198        let registry = Arc::new(Registry::default());
14199        let handler = ControlHandler::with_forwarding(
14200            Arc::clone(&registry),
14201            Arc::new(ForwardingTable::default()),
14202        );
14203        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14204        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14205        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14206        drop(module_rx);
14207
14208        let replies = handler
14209            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
14210            .await
14211            .unwrap();
14212        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
14213        assert!(registry.get_module("aft").unwrap().is_none());
14214        assert!(registry
14215            .get_module_by_connection(module_ctx.connection_id)
14216            .unwrap()
14217            .is_none());
14218        assert_eq!(
14219            catalog_snapshot(&handler, &client_ctx, 301).await,
14220            (before, Vec::new())
14221        );
14222    }
14223
14224    /// A handler whose `route.open` parks after queueing the route.bind and
14225    /// before marking it enqueued, with module "aft" registered on connection
14226    /// 40. Returns the two-party barrier that parks and releases it.
14227    async fn route_open_held_after_bind_send() -> (
14228        ControlHandler,
14229        Arc<tokio::sync::Barrier>,
14230        RouteCtx,
14231        mpsc::Receiver<crate::router::OutboundFrame>,
14232    ) {
14233        let hold = Arc::new(tokio::sync::Barrier::new(2));
14234        let handler = ControlHandler::with_forwarding(
14235            Arc::new(Registry::default()),
14236            Arc::new(ForwardingTable::default()),
14237        )
14238        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
14239        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14240        hello_via_sink(
14241            &handler,
14242            &module_ctx,
14243            &mut module_rx,
14244            hello_frame("aft", PROTOCOL_VERSION, 7),
14245        )
14246        .await;
14247        (handler, hold, module_ctx, module_rx)
14248    }
14249
14250    /// A fast module acks the route.bind, and the reader commits the route,
14251    /// before `route.open` has marked the relay enqueued. The daemon used to
14252    /// read the missing relay as an abandoned bind and send the module a
14253    /// GOODBYE for the route it had just committed: the client was told its
14254    /// route was open, the module unbound the channel, and the client's first
14255    /// request was dropped on the module side as unbound.
14256    #[tokio::test]
14257    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
14258        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14259        let client = ConnectionId::new(41);
14260        let (client_ctx, mut client_rx) = route_ctx(client);
14261
14262        let (open_task, bind) = relay_route_open(
14263            &handler,
14264            client,
14265            &client_ctx.egress,
14266            &mut module_rx,
14267            200,
14268            "aft",
14269            "acked-before-mark",
14270        )
14271        .await;
14272        let (module_channel, module_epoch) = route_bind_channel(&bind);
14273        // route.open is now parked between queueing the bind and marking it.
14274        hold.wait().await;
14275
14276        // The module answers inside that window, and the route commits.
14277        assert!(handler
14278            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14279            .await
14280            .unwrap()
14281            .is_empty());
14282        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
14283
14284        hold.wait().await;
14285        assert!(
14286            open_task.await.unwrap().is_empty(),
14287            "route.open answers accepted"
14288        );
14289
14290        // Every frame route.open could have sent is already queued, because it
14291        // has returned; the module must have been sent nothing.
14292        match module_rx.try_recv() {
14293            Err(mpsc::error::TryRecvError::Empty) => {}
14294            Ok(outbound) => panic!(
14295                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
14296                outbound.frame.header.ty,
14297                outbound.frame.header.channel,
14298                outbound.frame.header.epoch
14299            ),
14300            Err(err) => panic!("module egress closed: {err}"),
14301        }
14302
14303        // The client's first request on the route is the next thing the module
14304        // reads, on the binding it just created.
14305        let router = Router::with_control_handler(Arc::new(handler.clone()));
14306        let request = Frame::build(
14307            FrameType::Request,
14308            Flags::new(true, Priority::Interactive, false),
14309            client_channel,
14310            client_epoch,
14311            201,
14312            b"first request".to_vec(),
14313        )
14314        .unwrap();
14315        router
14316            .route_for_connection(&client_ctx, request)
14317            .await
14318            .unwrap();
14319        let delivered = module_rx.try_recv().unwrap().frame;
14320        assert_eq!(delivered.header.ty, FrameType::Request);
14321        assert_eq!(
14322            (delivered.header.channel, delivered.header.epoch),
14323            (module_channel, module_epoch)
14324        );
14325        assert_eq!(delivered.body, b"first request");
14326    }
14327
14328    /// The client goes away while `route.open` is still queueing the bind, so
14329    /// the relay is dropped with no module reply. The teardown cannot send the
14330    /// GOODBYE itself (it could reach the module ahead of the bind), so
14331    /// `route.open` must send it once the bind is queued; otherwise the module
14332    /// keeps a binding the daemon will never commit.
14333    #[tokio::test]
14334    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
14335        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14336        let client = ConnectionId::new(42);
14337        let (client_ctx, _client_rx) = route_ctx(client);
14338
14339        let (open_task, bind) = relay_route_open(
14340            &handler,
14341            client,
14342            &client_ctx.egress,
14343            &mut module_rx,
14344            300,
14345            "aft",
14346            "released-before-mark",
14347        )
14348        .await;
14349        let (module_channel, module_epoch) = route_bind_channel(&bind);
14350        hold.wait().await;
14351
14352        handler.cleanup_connection(client).unwrap();
14353        assert!(
14354            module_rx.try_recv().is_err(),
14355            "client teardown must not GOODBYE a bind route.open is still queueing"
14356        );
14357
14358        hold.wait().await;
14359        let response = open_task.await.unwrap();
14360        assert_eq!(response.len(), 1);
14361        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14362
14363        let goodbye = module_rx
14364            .try_recv()
14365            .expect("module is told to drop the binding the daemon abandoned")
14366            .frame;
14367        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14368        assert_eq!(
14369            (goodbye.header.channel, goodbye.header.epoch),
14370            (module_channel, module_epoch)
14371        );
14372        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14373    }
14374
14375    /// The module acks inside the same window, but the client's egress closed
14376    /// first, so the reader refuses to commit. The ack proves the module holds
14377    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
14378    /// send a second one.
14379    #[tokio::test]
14380    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
14381        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14382        let client = ConnectionId::new(43);
14383        let (client_ctx, client_rx) = route_ctx(client);
14384
14385        let (open_task, bind) = relay_route_open(
14386            &handler,
14387            client,
14388            &client_ctx.egress,
14389            &mut module_rx,
14390            400,
14391            "aft",
14392            "gone-client-before-mark",
14393        )
14394        .await;
14395        let (module_channel, module_epoch) = route_bind_channel(&bind);
14396        hold.wait().await;
14397
14398        drop(client_rx);
14399        assert!(client_ctx.egress.is_closed());
14400        handler
14401            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14402            .await
14403            .unwrap();
14404
14405        hold.wait().await;
14406        let response = open_task.await.unwrap();
14407        assert_eq!(response.len(), 1);
14408        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14409
14410        let goodbye = module_rx
14411            .try_recv()
14412            .expect("module is told to drop the binding it acked")
14413            .frame;
14414        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14415        assert_eq!(
14416            (goodbye.header.channel, goodbye.header.epoch),
14417            (module_channel, module_epoch)
14418        );
14419        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14420        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14421    }
14422
14423    /// The fence at the module-loop boundary, stated as its own contract: which
14424    /// forwarding failures are allowed to end the module connection that is being
14425    /// served. A `ConnectionClosing` naming some client is about that client, and
14426    /// a module connection is shared; the same error naming the module's own
14427    /// connection is about this connection and must stay fatal, as must failures
14428    /// that are about the forwarding table itself.
14429    #[test]
14430    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
14431        let handler = ControlHandler::default();
14432        let module_connection = ConnectionId::new(30);
14433        let client_connection = ConnectionId::new(31);
14434
14435        handler
14436            .refuse_to_end_module_connection_for_a_client(
14437                module_connection,
14438                77,
14439                ForwardingError::ConnectionClosing {
14440                    connection_id: client_connection,
14441                },
14442            )
14443            .expect("a closing client must never end the module connection");
14444
14445        assert!(matches!(
14446            handler.refuse_to_end_module_connection_for_a_client(
14447                module_connection,
14448                78,
14449                ForwardingError::ConnectionClosing {
14450                    connection_id: module_connection,
14451                },
14452            ),
14453            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
14454                connection_id
14455            })) if connection_id == module_connection
14456        ));
14457        assert!(matches!(
14458            handler.refuse_to_end_module_connection_for_a_client(
14459                module_connection,
14460                79,
14461                ForwardingError::Poisoned,
14462            ),
14463            Err(RouterError::Forwarding(ForwardingError::Poisoned))
14464        ));
14465        assert!(matches!(
14466            handler.refuse_to_end_module_connection_for_a_client(
14467                module_connection,
14468                80,
14469                ForwardingError::StaleModuleEndpoint,
14470            ),
14471            Err(RouterError::Forwarding(
14472                ForwardingError::StaleModuleEndpoint
14473            ))
14474        ));
14475    }
14476
14477    /// The spawn-attestation guard is what stops a connected module from claiming
14478    /// another module's identity and being stamped `Reserved` for it. Every other
14479    /// test that supplies a consumer_identity supplies a CORRECT one, because a
14480    /// correct one is what the rest of the flow needs -- so the guard's rejection
14481    /// branch was never the subject of an assertion, only its acceptance branch.
14482    ///
14483    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
14484    /// whole subc-core library suite green; only the forwarding integration tests
14485    /// notice, and they notice for unrelated reasons. This test exists so the
14486    /// refusal itself is asserted where the guard lives: it fails if the identity
14487    /// check stops refusing, which is the direction that matters, since a guard
14488    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
14489    #[tokio::test]
14490    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
14491        let registry = Arc::new(Registry::default());
14492        let forwarding = Arc::new(ForwardingTable::default());
14493        let supervisor = SupervisorHandle::new();
14494        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14495        let handler =
14496            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14497                .with_supervisor(supervisor);
14498
14499        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
14500        hello_via_sink(
14501            &handler,
14502            &target_ctx,
14503            &mut target_rx,
14504            hello_frame("target", PROTOCOL_VERSION, 1),
14505        )
14506        .await;
14507
14508        // A real supervised module id presenting the wrong nonce. This is the
14509        // impersonation case: the attacker knows a privileged module_id, which is
14510        // public, and guesses at the nonce, which is not.
14511        let wrong_nonce = handler
14512            .handle_control_frame(
14513                &route_ctx(ConnectionId::new(91)).0,
14514                route_open_frame_with_admission_facts(
14515                    20,
14516                    "target",
14517                    unique_project_root("admission-facts"),
14518                    Some(subc_control::ConsumerIdentity {
14519                        module_id: "fed".to_string(),
14520                        launch_nonce: "not-the-real-nonce".to_string(),
14521                    }),
14522                    None,
14523                ),
14524            )
14525            .await
14526            .unwrap();
14527        assert_eq!(
14528            parse_error(&wrong_nonce[0])["code"],
14529            "bad_consumer_identity",
14530            "a mismatched launch nonce must be refused, not stamped Reserved"
14531        );
14532
14533        // A module id the supervisor never spawned at all, so no nonce exists to
14534        // compare against. An implementation that treats "no record" as "nothing
14535        // to check" fails open here while passing the case above.
14536        let never_spawned = handler
14537            .handle_control_frame(
14538                &route_ctx(ConnectionId::new(92)).0,
14539                route_open_frame_with_admission_facts(
14540                    21,
14541                    "target",
14542                    unique_project_root("admission-facts"),
14543                    Some(subc_control::ConsumerIdentity {
14544                        module_id: "never-spawned".to_string(),
14545                        launch_nonce: "any-nonce".to_string(),
14546                    }),
14547                    None,
14548                ),
14549            )
14550            .await
14551            .unwrap();
14552        assert_eq!(
14553            parse_error(&never_spawned[0])["code"],
14554            "bad_consumer_identity",
14555            "an unspawned module_id must be refused rather than accepted for lack of a record"
14556        );
14557    }
14558
14559    /// The refusal test above proves the guard says NO. Nothing proved it can say
14560    /// YES, and the difference is not academic: replacing the whole authorization
14561    /// with `false` -- admitting no consumer identity at all, revoking Reserved
14562    /// standing for every supervised module in the fleet -- leaves 110 of the 111
14563    /// library tests GREEN. The one that notices does so by HANGING, because it
14564    /// waits for a bind that can no longer happen.
14565    ///
14566    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
14567    /// or flaky test, invites a RETRY rather than an investigation, and the retry
14568    /// hangs too and gets blamed on the runner. So a total revocation of the
14569    /// daemon's trust grant would have shipped behind a symptom nobody attributes
14570    /// to code.
14571    ///
14572    /// The bias is structural rather than accidental. A REFUSAL looks like a
14573    /// failure someone writes a test for; a GRANT looks like the happy path. Every
14574    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
14575    /// suite for that reason, and this one is the purest case in the daemon.
14576    ///
14577    /// This test asserts the EFFECT rather than the absence of an error: the module
14578    /// receives a RouteBind and it carries `Reserved` naming the attested module.
14579    /// A guard that admitted nobody would produce no bind at all; one that admitted
14580    /// everybody would stamp the wrong principal, which the refusal test catches.
14581    #[tokio::test]
14582    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
14583        let registry = Arc::new(Registry::default());
14584        let forwarding = Arc::new(ForwardingTable::default());
14585        let supervisor = SupervisorHandle::new();
14586        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14587        let handler =
14588            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14589                .with_supervisor(supervisor);
14590
14591        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
14592        hello_via_sink(
14593            &handler,
14594            &target_ctx,
14595            &mut target_rx,
14596            hello_frame("target", PROTOCOL_VERSION, 1),
14597        )
14598        .await;
14599
14600        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
14601        let route_handler = handler.clone();
14602        let route_task = tokio::spawn(async move {
14603            route_handler
14604                .handle_control_frame(
14605                    &client_ctx,
14606                    route_open_frame_with_admission_facts(
14607                        30,
14608                        "target",
14609                        unique_project_root("admission-facts"),
14610                        Some(subc_control::ConsumerIdentity {
14611                            module_id: "fed".to_string(),
14612                            launch_nonce: "fed-nonce".to_string(),
14613                        }),
14614                        None,
14615                    ),
14616                )
14617                .await
14618                .unwrap()
14619        });
14620
14621        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
14622        // the very mutation it exists to catch -- a guard that admits nobody -- no
14623        // bind is ever sent, so it HUNG rather than failing. That reproduces the
14624        // exact defect being fixed: a total revocation detected only as a stalled
14625        // suite, which reads as flakiness and invites a retry. An acceptance test
14626        // that waits for an effect must bound the wait, or a red becomes a hang.
14627        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14628            .await
14629            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
14630            .expect("module control channel closed before route.bind");
14631        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
14632        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14633            panic!("expected route.bind")
14634        };
14635        assert_eq!(
14636            principal,
14637            Some(Principal::Reserved {
14638                module_id: "fed".to_string()
14639            }),
14640            "a correctly attested consumer must be stamped Reserved for its own id"
14641        );
14642
14643        handler
14644            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
14645            .await
14646            .unwrap();
14647        assert!(route_task.await.unwrap().is_empty());
14648        assert!(
14649            matches!(
14650                serde_json::from_slice::<ClientControlResponse>(
14651                    &client_rx.recv().await.unwrap().body
14652                )
14653                .unwrap(),
14654                ClientControlResponse::RouteOpen { .. }
14655            ),
14656            "the route must actually open, not merely avoid an error"
14657        );
14658    }
14659
14660    #[tokio::test(start_paused = true)]
14661    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
14662        let registry = Arc::new(Registry::default());
14663        let forwarding = Arc::new(ForwardingTable::default());
14664        let supervisor = SupervisorHandle::new();
14665        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14666        let handler =
14667            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14668                .with_supervisor(supervisor);
14669
14670        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
14671        hello_via_sink(
14672            &handler,
14673            &target_ctx,
14674            &mut target_rx,
14675            hello_frame("target", PROTOCOL_VERSION, 1),
14676        )
14677        .await;
14678
14679        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
14680        let direct_handler = handler.clone();
14681        let direct_open = tokio::spawn(async move {
14682            direct_handler
14683                .handle_control_frame(
14684                    &direct_ctx,
14685                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
14686                )
14687                .await
14688                .unwrap()
14689        });
14690        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14691            .await
14692            .expect("no direct route.bind within 5s")
14693            .expect("target control channel closed before direct route.bind");
14694        handler
14695            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
14696            .await
14697            .unwrap();
14698        assert!(direct_open.await.unwrap().is_empty());
14699        let _ = direct_rx.recv().await.unwrap();
14700
14701        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
14702        let reserved_handler = handler.clone();
14703        let reserved_open = tokio::spawn(async move {
14704            reserved_handler
14705                .handle_control_frame(
14706                    &reserved_ctx,
14707                    route_open_frame_with_admission_facts(
14708                        3,
14709                        "target",
14710                        unique_project_root("admission-facts"),
14711                        Some(ConsumerIdentity {
14712                            module_id: "fed".to_string(),
14713                            launch_nonce: "fed-nonce".to_string(),
14714                        }),
14715                        None,
14716                    ),
14717                )
14718                .await
14719                .unwrap()
14720        });
14721        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14722            .await
14723            .expect("no reserved route.bind within 5s")
14724            .expect("target control channel closed before reserved route.bind");
14725        handler
14726            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
14727            .await
14728            .unwrap();
14729        assert!(reserved_open.await.unwrap().is_empty());
14730        let _ = reserved_rx.recv().await.unwrap();
14731
14732        forwarding
14733            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
14734            .unwrap();
14735        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
14736        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
14737            module_id: Some("target".to_string()),
14738        })
14739        .unwrap();
14740        let census_frame =
14741            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
14742        let response = handler
14743            .handle_control_frame(&census_ctx, census_frame)
14744            .await
14745            .unwrap()
14746            .pop()
14747            .unwrap();
14748        let actual: Value = serde_json::from_slice(&response.body).unwrap();
14749        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
14750        assert!(matches!(
14751            decoded,
14752            ClientControlResponse::SupervisorRoutes { .. }
14753        ));
14754        let routes = actual["modules"][0]["routes"].as_array().unwrap();
14755        assert_eq!(routes.len(), 2);
14756        assert!(routes.iter().all(|route| route["draining"] == true));
14757        // The census carries WHY: the reason the drain was begun with, in the
14758        // route.closing vocabulary, on every draining route this drain marked.
14759        assert!(
14760            routes.iter().all(|route| route["drain_reason"] == "reload"),
14761            "draining routes must name the drain's reason: {routes:?}"
14762        );
14763        assert!(routes.iter().any(|route| {
14764            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
14765        }));
14766        assert!(routes.iter().any(|route| {
14767            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
14768        }));
14769
14770        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
14771            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
14772        if std::env::var_os("UPDATE_GOLDEN").is_some() {
14773            std::fs::write(
14774                &golden_path,
14775                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
14776            )
14777            .unwrap();
14778        }
14779        let expected: Value =
14780            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
14781        assert_eq!(actual, expected);
14782    }
14783
14784    async fn query_live_roots(
14785        handler: &ControlHandler,
14786        module_ctx: &RouteCtx,
14787    ) -> ModuleControlResponseToModule {
14788        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
14789        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
14790        let response = handler
14791            .handle_control_frame(module_ctx, frame)
14792            .await
14793            .unwrap()
14794            .pop()
14795            .unwrap();
14796        serde_json::from_slice(&response.body).unwrap()
14797    }
14798
14799    #[tokio::test(start_paused = true)]
14800    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
14801        let registry = Arc::new(Registry::default());
14802        let forwarding = Arc::new(ForwardingTable::default());
14803        let handler = ControlHandler::with_forwarding(registry, forwarding);
14804        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
14805        hello_via_sink(
14806            &handler,
14807            &target_ctx,
14808            &mut target_rx,
14809            hello_frame("target", PROTOCOL_VERSION, 1),
14810        )
14811        .await;
14812        let root = unique_project_root("live-roots-known");
14813        let path = ProjectRootId::from_path_allowing_missing(root.path())
14814            .unwrap()
14815            .as_path()
14816            .to_path_buf();
14817        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
14818        let open_handler = handler.clone();
14819        let opened = tokio::spawn(async move {
14820            open_handler
14821                .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
14822                .await
14823                .unwrap()
14824        });
14825        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14826            .await
14827            .unwrap()
14828            .unwrap();
14829        handler
14830            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
14831            .await
14832            .unwrap();
14833        assert!(opened.await.unwrap().is_empty());
14834        let _ = client_rx.recv().await.unwrap();
14835
14836        let root = unique_project_root("live-roots-pending");
14837        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
14838            .unwrap()
14839            .as_path()
14840            .to_path_buf();
14841        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
14842        let open_handler = handler.clone();
14843        let pending = tokio::spawn(async move {
14844            open_handler
14845                .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
14846                .await
14847                .unwrap()
14848        });
14849        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14850            .await
14851            .unwrap()
14852            .unwrap();
14853        let actual = query_live_roots(&handler, &target_ctx).await;
14854        let ModuleControlResponseToModule::LiveRoots {
14855            roots,
14856            unknown_root_bindings,
14857            total_bindings,
14858        } = actual
14859        else {
14860            panic!("expected live roots")
14861        };
14862        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
14863        assert_eq!(unknown_root_bindings, 0);
14864        assert_eq!(
14865            roots.len(),
14866            2,
14867            "root-known arm must retain each canonical root"
14868        );
14869        assert_eq!(
14870            total_bindings,
14871            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
14872        );
14873        let counts = roots
14874            .iter()
14875            .map(|root| (root.project_root.clone(), root.bound, root.pending))
14876            .collect::<Vec<_>>();
14877        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
14878        expected.sort_by(|a, b| a.0.cmp(&b.0));
14879        assert_eq!(
14880            counts, expected,
14881            "roots must sort by path and count pending separately"
14882        );
14883        handler
14884            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
14885            .await
14886            .unwrap();
14887        assert!(pending.await.unwrap().is_empty());
14888    }
14889
14890    #[tokio::test(start_paused = true)]
14891    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
14892        let forwarding = Arc::new(ForwardingTable::default());
14893        let handler =
14894            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
14895        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
14896        hello_via_sink(
14897            &handler,
14898            &target_ctx,
14899            &mut target_rx,
14900            hello_frame("target", PROTOCOL_VERSION, 1),
14901        )
14902        .await;
14903        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
14904        let pending = forwarding
14905            .begin_route_bind_relay_for_test(
14906                client_ctx.connection_id,
14907                client_ctx.egress.clone(),
14908                2,
14909                "target",
14910            )
14911            .unwrap();
14912        forwarding
14913            .complete_pending_relay(
14914                target_ctx.connection_id,
14915                pending.corr,
14916                RouteBindRelayOutcome::Accepted,
14917            )
14918            .unwrap();
14919        let actual = query_live_roots(&handler, &target_ctx).await;
14920        let ModuleControlResponseToModule::LiveRoots {
14921            roots,
14922            unknown_root_bindings,
14923            total_bindings,
14924        } = actual
14925        else {
14926            panic!("expected live roots")
14927        };
14928        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
14929        assert_eq!(
14930            unknown_root_bindings, 1,
14931            "unknown-root arm must not read as no bindings"
14932        );
14933        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
14934        assert_eq!(
14935            total_bindings,
14936            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
14937        );
14938    }
14939
14940    /// A module reads HELLO_ACK as its first frame and exits on anything else,
14941    /// so the ack has to be on its outbound queue before the module is
14942    /// routable. The connection loop writes a handler's replies only after the
14943    /// handler returns; this test stops in exactly that gap, runs a real
14944    /// route.open from another connection, and only then writes whatever the
14945    /// HELLO handler returned, the way the loop would. If the ack were still a
14946    /// reply, the route.bind request would reach the module first.
14947    #[tokio::test(start_paused = true)]
14948    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
14949        let forwarding = Arc::new(ForwardingTable::default());
14950        let handler =
14951            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
14952        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
14953        let replies = handler
14954            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
14955            .await
14956            .unwrap();
14957        let queued_by_hello = module_rx.len();
14958
14959        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
14960        let open_handler = handler.clone();
14961        let open = tokio::spawn(async move {
14962            open_handler
14963                .handle_control_frame(
14964                    &client_ctx,
14965                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
14966                )
14967                .await
14968                .unwrap()
14969        });
14970        // Let the route.open run until its route.bind is on the module's queue.
14971        let mut spins = 0;
14972        while module_rx.len() == queued_by_hello {
14973            spins += 1;
14974            assert!(spins < 10_000, "route.open never queued a route.bind");
14975            tokio::task::yield_now().await;
14976        }
14977
14978        // Now the connection loop's half: write the HELLO handler's replies.
14979        for reply in replies {
14980            module_ctx.egress.send(reply).await.unwrap();
14981        }
14982
14983        let first = module_rx.recv().await.unwrap().frame;
14984        assert_eq!(
14985            first.header.ty,
14986            FrameType::HelloAck,
14987            "the first frame a registering module reads must be its HELLO_ACK"
14988        );
14989        assert_eq!(first.header.corr, 7);
14990        let second = module_rx.recv().await.unwrap().frame;
14991        assert_eq!(second.header.ty, FrameType::Request);
14992        assert!(
14993            matches!(
14994                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
14995                ModuleControlRequest::RouteBind { .. }
14996            ),
14997            "the route.bind follows the ack"
14998        );
14999        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
15000
15001        handler
15002            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
15003            .await
15004            .unwrap();
15005        assert!(open.await.unwrap().is_empty());
15006        let _ = client_rx.recv().await.unwrap();
15007    }
15008
15009    #[tokio::test(start_paused = true)]
15010    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
15011        let handler = ControlHandler::with_forwarding(
15012            Arc::new(Registry::default()),
15013            Arc::new(ForwardingTable::default()),
15014        );
15015        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
15016        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
15017        hello_via_sink(
15018            &handler,
15019            &first_ctx,
15020            &mut first_rx,
15021            hello_frame("first", PROTOCOL_VERSION, 1),
15022        )
15023        .await;
15024        hello_via_sink(
15025            &handler,
15026            &second_ctx,
15027            &mut second_rx,
15028            hello_frame("second", PROTOCOL_VERSION, 2),
15029        )
15030        .await;
15031        let root = unique_project_root("second-only");
15032        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
15033        let cloned = handler.clone();
15034        let open = tokio::spawn(async move {
15035            cloned
15036                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
15037                .await
15038                .unwrap()
15039        });
15040        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
15041            .await
15042            .unwrap()
15043            .unwrap();
15044        let first = query_live_roots(&handler, &first_ctx).await;
15045        let second = query_live_roots(&handler, &second_ctx).await;
15046        assert!(
15047            matches!(
15048                first,
15049                ModuleControlResponseToModule::LiveRoots {
15050                    total_bindings: 0,
15051                    ..
15052                }
15053            ),
15054            "cross-module scope must not expose another module's roots"
15055        );
15056        assert!(
15057            matches!(
15058                second,
15059                ModuleControlResponseToModule::LiveRoots {
15060                    total_bindings: 1,
15061                    ..
15062                }
15063            ),
15064            "second module must see its pending route"
15065        );
15066        handler
15067            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
15068            .await
15069            .unwrap();
15070        assert!(open.await.unwrap().is_empty());
15071    }
15072
15073    #[tokio::test(start_paused = true)]
15074    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
15075        let handler = ControlHandler::with_forwarding(
15076            Arc::new(Registry::default()),
15077            Arc::new(ForwardingTable::default()),
15078        );
15079        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
15080        hello_via_sink(
15081            &handler,
15082            &target_ctx,
15083            &mut target_rx,
15084            hello_frame("target", PROTOCOL_VERSION, 1),
15085        )
15086        .await;
15087        let actual = query_live_roots(&handler, &target_ctx).await;
15088        let ModuleControlResponseToModule::LiveRoots {
15089            roots,
15090            unknown_root_bindings,
15091            total_bindings,
15092        } = actual
15093        else {
15094            panic!("expected live roots")
15095        };
15096        assert!(roots.is_empty());
15097        assert_eq!(unknown_root_bindings, 0);
15098        assert_eq!(total_bindings, 0);
15099        assert_eq!(
15100            total_bindings,
15101            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15102        );
15103    }
15104
15105    /// Read the vendored fed corpus rather than hand-building a package.
15106    ///
15107    /// A hand-built object encodes what the test author believed the carrier
15108    /// emits. These vectors are what it actually emits, and one of them exists
15109    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
15110    /// ignores additive unknown fields at the traversal emit terminus."
15111    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
15112        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
15113            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
15114        let text = std::fs::read_to_string(&path)
15115            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
15116        let vectors: Vec<(String, Value)> = text
15117            .lines()
15118            .filter(|line| !line.trim().is_empty())
15119            .map(|line| {
15120                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
15121                let id = entry["corpus_id"]
15122                    .as_str()
15123                    .expect("every vector carries a corpus_id")
15124                    .to_string();
15125                (id, entry["package"].clone())
15126            })
15127            .collect();
15128        // Pin the count: a corpus that silently shrinks would take its coverage
15129        // with it, and a suite reading N-1 vectors reports the same clean pass
15130        // as one reading N.
15131        assert_eq!(
15132            vectors.len(),
15133            3,
15134            "vendored fed corpus changed size; re-sync from subc-federation"
15135        );
15136
15137        // Pin what makes the corpus DISCRIMINATING, not just present.
15138        //
15139        // The relay test below takes its expected value from the corpus, so the
15140        // corpus supplies the test's power to detect a lossy relay rather than
15141        // its correctness. A relay that dropped unrecognised fields would still
15142        // be caught -- but only by a package carrying fields it does not know.
15143        // Shrink every package to the handful of keys any implementation would
15144        // recognise and the test keeps passing over an input that can no longer
15145        // fail, which is the same clean green as a corpus that shrank away.
15146        //
15147        // So assert the precondition rather than duplicating the packages here:
15148        // at least one vector must carry a field beyond the small common set.
15149        // That is one claim to maintain instead of nine, and it fails loudly if
15150        // a re-sync ever flattens the corpus.
15151        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
15152        let richest = vectors
15153            .iter()
15154            .filter_map(|(_, package)| package.as_object())
15155            .map(|object| {
15156                object
15157                    .keys()
15158                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
15159                    .count()
15160            })
15161            .max()
15162            .unwrap_or(0);
15163        assert!(
15164            richest >= 2,
15165            "vendored corpus no longer carries a package with unmodelled fields, \
15166             so the relay test can no longer distinguish a verbatim relay from a lossy one"
15167        );
15168
15169        vectors
15170    }
15171
15172    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
15173    ///
15174    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
15175    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
15176    /// three-key object, so a relay that quietly dropped fields it did not
15177    /// recognise would satisfy it. These vectors carry nine keys including ones
15178    /// this crate has no type for, so a typed relay fails here and only here.
15179    #[tokio::test]
15180    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
15181        for (corpus_id, package) in fed_admission_facts_vectors() {
15182            let registry = Arc::new(Registry::default());
15183            let forwarding = Arc::new(ForwardingTable::default());
15184            let supervisor = SupervisorHandle::new();
15185            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15186            let handler =
15187                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15188                    .with_supervisor(supervisor)
15189                    .with_admission_facts_config(
15190                        Some("fed".to_string()),
15191                        Some(vec!["target".to_string()]),
15192                    );
15193
15194            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
15195            hello_via_sink(
15196                &handler,
15197                &target_ctx,
15198                &mut target_rx,
15199                hello_frame("target", PROTOCOL_VERSION, 1),
15200            )
15201            .await;
15202
15203            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
15204            let route_handler = handler.clone();
15205            let expected = package.clone();
15206            let route_task = tokio::spawn(async move {
15207                route_handler
15208                    .handle_control_frame(
15209                        &client_ctx,
15210                        route_open_frame_with_admission_facts(
15211                            20,
15212                            "target",
15213                            unique_project_root("admission-facts"),
15214                            Some(subc_control::ConsumerIdentity {
15215                                module_id: "fed".to_string(),
15216                                launch_nonce: "fed-nonce".to_string(),
15217                            }),
15218                            Some(package),
15219                        ),
15220                    )
15221                    .await
15222                    .unwrap()
15223            });
15224
15225            let bind_frame = target_rx.recv().await.unwrap();
15226            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15227            let ModuleControlRequest::RouteBind {
15228                admission_facts, ..
15229            } = bind
15230            else {
15231                panic!("{corpus_id}: expected route.bind")
15232            };
15233            assert_eq!(
15234                admission_facts,
15235                Some(expected),
15236                "{corpus_id}: relay must not add, drop or reshape any field"
15237            );
15238
15239            handler
15240                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15241                .await
15242                .unwrap();
15243            route_task.await.unwrap();
15244        }
15245    }
15246
15247    #[tokio::test]
15248    async fn route_bind_never_relays_the_openers_build_class() {
15249        let registry = Arc::new(Registry::default());
15250        let forwarding = Arc::new(ForwardingTable::default());
15251        let handler =
15252            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15253        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
15254        hello_via_sink(
15255            &handler,
15256            &target_ctx,
15257            &mut target_rx,
15258            hello_frame("target", PROTOCOL_VERSION, 1),
15259        )
15260        .await;
15261
15262        let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
15263        identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
15264        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15265            busy_refusals: None,
15266            target: RouteTarget::ToolProvider {
15267                module_id: "target".to_string(),
15268            },
15269            identity,
15270            consumer_identity: None,
15271            consumer_capabilities: None,
15272            role_versions: None,
15273            admission_facts: None,
15274            scope: None,
15275        })
15276        .unwrap();
15277        let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
15278
15279        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
15280        let route_handler = handler.clone();
15281        let route_task = tokio::spawn(async move {
15282            route_handler
15283                .handle_control_frame(&client_ctx, open)
15284                .await
15285                .unwrap()
15286        });
15287        let bind_frame = target_rx.recv().await.unwrap();
15288        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15289        let ModuleControlRequest::RouteBind { identity, .. } = bind else {
15290            panic!("expected route.bind")
15291        };
15292        assert_eq!(
15293            identity.build_class, None,
15294            "route.bind carried the build class"
15295        );
15296        assert!(
15297            !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
15298            "the route.bind bytes still name build_class"
15299        );
15300        handler
15301            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15302            .await
15303            .unwrap();
15304        route_task.await.unwrap();
15305    }
15306
15307    #[tokio::test]
15308    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
15309        let registry = Arc::new(Registry::default());
15310        let forwarding = Arc::new(ForwardingTable::default());
15311        let supervisor = SupervisorHandle::new();
15312        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15313        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
15314        let handler =
15315            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15316                .with_supervisor(supervisor)
15317                .with_admission_facts_config(
15318                    Some("fed".to_string()),
15319                    Some(vec!["target".to_string()]),
15320                );
15321
15322        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
15323        hello_via_sink(
15324            &handler,
15325            &target_ctx,
15326            &mut target_rx,
15327            hello_frame("target", PROTOCOL_VERSION, 1),
15328        )
15329        .await;
15330        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
15331        hello_via_sink(
15332            &handler,
15333            &other_ctx,
15334            &mut other_rx,
15335            hello_frame("other", PROTOCOL_VERSION, 2),
15336        )
15337        .await;
15338
15339        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
15340        let expected_facts = facts.clone();
15341        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
15342        let route_handler = handler.clone();
15343        let route_task = tokio::spawn(async move {
15344            route_handler
15345                .handle_control_frame(
15346                    &client_ctx,
15347                    route_open_frame_with_admission_facts(
15348                        10,
15349                        "target",
15350                        unique_project_root("admission-facts"),
15351                        Some(subc_control::ConsumerIdentity {
15352                            module_id: "fed".to_string(),
15353                            launch_nonce: "fed-nonce".to_string(),
15354                        }),
15355                        Some(facts.clone()),
15356                    ),
15357                )
15358                .await
15359                .unwrap()
15360        });
15361        let bind_frame = target_rx.recv().await.unwrap();
15362        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15363        let ModuleControlRequest::RouteBind {
15364            admission_facts, ..
15365        } = bind
15366        else {
15367            panic!("expected route.bind")
15368        };
15369        assert_eq!(admission_facts, Some(expected_facts));
15370        handler
15371            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15372            .await
15373            .unwrap();
15374        assert!(route_task.await.unwrap().is_empty());
15375        assert!(matches!(
15376            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
15377                .unwrap(),
15378            ClientControlResponse::RouteOpen { .. }
15379        ));
15380
15381        let direct = handler
15382            .handle_control_frame(
15383                &route_ctx(ConnectionId::new(73)).0,
15384                route_open_frame_with_admission_facts(
15385                    11,
15386                    "target",
15387                    unique_project_root("admission-facts"),
15388                    None,
15389                    Some(json!({"x": 1})),
15390                ),
15391            )
15392            .await
15393            .unwrap();
15394        assert_eq!(
15395            parse_error(&direct[0])["code"],
15396            "admission_facts_not_permitted"
15397        );
15398
15399        let different_reserved = handler
15400            .handle_control_frame(
15401                &route_ctx(ConnectionId::new(77)).0,
15402                route_open_frame_with_admission_facts(
15403                    15,
15404                    "target",
15405                    unique_project_root("admission-facts"),
15406                    Some(subc_control::ConsumerIdentity {
15407                        module_id: "other".to_string(),
15408                        launch_nonce: "other-nonce".to_string(),
15409                    }),
15410                    Some(json!({"x": 1})),
15411                ),
15412            )
15413            .await
15414            .unwrap();
15415        assert_eq!(
15416            parse_error(&different_reserved[0])["code"],
15417            "admission_facts_not_permitted"
15418        );
15419
15420        let other_target = handler
15421            .handle_control_frame(
15422                &route_ctx(ConnectionId::new(74)).0,
15423                route_open_frame_with_admission_facts(
15424                    12,
15425                    "other",
15426                    unique_project_root("admission-facts"),
15427                    Some(subc_control::ConsumerIdentity {
15428                        module_id: "fed".to_string(),
15429                        launch_nonce: "fed-nonce".to_string(),
15430                    }),
15431                    Some(json!({"x": 1})),
15432                ),
15433            )
15434            .await
15435            .unwrap();
15436        assert_eq!(
15437            parse_error(&other_target[0])["code"],
15438            "admission_facts_target_not_allowed"
15439        );
15440
15441        let nonexistent = handler
15442            .handle_control_frame(
15443                &route_ctx(ConnectionId::new(75)).0,
15444                route_open_frame_with_admission_facts(
15445                    13,
15446                    "missing",
15447                    unique_project_root("admission-facts"),
15448                    None,
15449                    Some(json!({"x": 1})),
15450                ),
15451            )
15452            .await
15453            .unwrap();
15454        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
15455
15456        let described = handler
15457            .handle_control_frame(
15458                &route_ctx(ConnectionId::new(76)).0,
15459                Frame::build(
15460                    FrameType::Request,
15461                    control_flags(),
15462                    0,
15463                    0,
15464                    14,
15465                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15466                )
15467                .unwrap(),
15468            )
15469            .await
15470            .unwrap();
15471        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15472            serde_json::from_slice(&described[0].body).unwrap()
15473        else {
15474            panic!("expected server.describe response")
15475        };
15476        assert!(capabilities
15477            .iter()
15478            .any(|cap| cap == "admission_facts_relay_v1"));
15479    }
15480
15481    #[tokio::test]
15482    async fn admission_facts_without_configured_carrier_are_rejected() {
15483        let registry = Arc::new(Registry::default());
15484        let forwarding = Arc::new(ForwardingTable::default());
15485        let handler = ControlHandler::with_forwarding(registry, forwarding);
15486        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
15487        hello_via_sink(
15488            &handler,
15489            &target_ctx,
15490            &mut target_rx,
15491            hello_frame("target", PROTOCOL_VERSION, 1),
15492        )
15493        .await;
15494
15495        let responses = handler
15496            .handle_control_frame(
15497                &route_ctx(ConnectionId::new(79)).0,
15498                route_open_frame_with_admission_facts(
15499                    16,
15500                    "target",
15501                    unique_project_root("admission-facts"),
15502                    None,
15503                    Some(json!({"x": 1})),
15504                ),
15505            )
15506            .await
15507            .unwrap();
15508        assert_eq!(
15509            parse_error(&responses[0])["code"],
15510            "admission_facts_not_permitted"
15511        );
15512    }
15513
15514    #[tokio::test]
15515    async fn route_open_relays_consumer_capabilities_verbatim() {
15516        let registry = Arc::new(Registry::default());
15517        let forwarding = Arc::new(ForwardingTable::default());
15518        let handler =
15519            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15520        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
15521        hello_via_sink(
15522            &handler,
15523            &module_ctx,
15524            &mut module_rx,
15525            hello_frame("aft", PROTOCOL_VERSION, 7),
15526        )
15527        .await;
15528
15529        let expected = vec!["elicitation".to_string(), "roots".to_string()];
15530        let expected_for_request = expected.clone();
15531        let project_root = unique_project_root("consumer-capabilities-present");
15532        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
15533        let route_handler = handler.clone();
15534        let route_task = tokio::spawn(async move {
15535            route_handler
15536                .handle_control_frame(
15537                    &client_ctx,
15538                    route_open_frame_with_consumer_capabilities(
15539                        401,
15540                        "aft",
15541                        project_root,
15542                        Some(expected_for_request),
15543                    ),
15544                )
15545                .await
15546                .unwrap()
15547        });
15548        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15549            .await
15550            .unwrap()
15551            .unwrap();
15552        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15553        let ModuleControlRequest::RouteBind {
15554            consumer_capabilities,
15555            ..
15556        } = bind
15557        else {
15558            panic!("expected route.bind request, got {bind:?}");
15559        };
15560        assert_eq!(consumer_capabilities, Some(expected.clone()));
15561
15562        handler
15563            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15564            .await
15565            .unwrap();
15566        let route_response = route_task.await.unwrap();
15567        assert!(route_response.is_empty());
15568        let published = client_rx.recv().await.unwrap();
15569        assert!(matches!(
15570            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15571            ClientControlResponse::RouteOpen { .. }
15572        ));
15573    }
15574
15575    #[tokio::test]
15576    async fn route_open_without_consumer_capabilities_relays_none() {
15577        let registry = Arc::new(Registry::default());
15578        let forwarding = Arc::new(ForwardingTable::default());
15579        let handler =
15580            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15581        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
15582        hello_via_sink(
15583            &handler,
15584            &module_ctx,
15585            &mut module_rx,
15586            hello_frame("aft", PROTOCOL_VERSION, 7),
15587        )
15588        .await;
15589
15590        let project_root = unique_project_root("consumer-capabilities-absent");
15591        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
15592        let route_handler = handler.clone();
15593        let route_task = tokio::spawn(async move {
15594            route_handler
15595                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
15596                .await
15597                .unwrap()
15598        });
15599        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15600            .await
15601            .unwrap()
15602            .unwrap();
15603        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15604        let ModuleControlRequest::RouteBind {
15605            consumer_capabilities,
15606            ..
15607        } = bind
15608        else {
15609            panic!("expected route.bind request, got {bind:?}");
15610        };
15611        assert_eq!(consumer_capabilities, None);
15612
15613        handler
15614            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15615            .await
15616            .unwrap();
15617        let route_response = route_task.await.unwrap();
15618        assert!(route_response.is_empty());
15619        let published = client_rx.recv().await.unwrap();
15620        assert!(matches!(
15621            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15622            ClientControlResponse::RouteOpen { .. }
15623        ));
15624    }
15625
15626    /// Opens a route to a freshly registered `aft` with `sent` as the
15627    /// route.open's role_versions, acks the bind, and returns the role_versions
15628    /// the module's bind carried.
15629    async fn bind_role_versions_for(
15630        sent: Option<BTreeMap<String, String>>,
15631        connection: u64,
15632    ) -> Option<BTreeMap<String, String>> {
15633        let registry = Arc::new(Registry::default());
15634        let forwarding = Arc::new(ForwardingTable::default());
15635        let handler =
15636            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15637        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
15638        hello_via_sink(
15639            &handler,
15640            &module_ctx,
15641            &mut module_rx,
15642            hello_frame("aft", PROTOCOL_VERSION, 7),
15643        )
15644        .await;
15645        let project_root = unique_project_root("role-versions");
15646        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
15647        let route_handler = handler.clone();
15648        let route_task = tokio::spawn(async move {
15649            route_handler
15650                .handle_control_frame(
15651                    &client_ctx,
15652                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
15653                )
15654                .await
15655                .unwrap()
15656        });
15657        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15658            .await
15659            .expect("a well-formed route.open reaches the module as a bind")
15660            .unwrap();
15661        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15662        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
15663            panic!("expected route.bind request, got {bind:?}");
15664        };
15665        handler
15666            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15667            .await
15668            .unwrap();
15669        assert!(route_task.await.unwrap().is_empty());
15670        let published = client_rx.recv().await.unwrap();
15671        assert!(matches!(
15672            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15673            ClientControlResponse::RouteOpen { .. }
15674        ));
15675        role_versions
15676    }
15677
15678    #[tokio::test]
15679    async fn route_open_relays_role_versions_verbatim() {
15680        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
15681        assert_eq!(
15682            bind_role_versions_for(Some(sent.clone()), 141).await,
15683            Some(sent)
15684        );
15685    }
15686
15687    /// An empty map declares nothing, so the provider sees no field rather
15688    /// than an empty object it would have to treat as a second "none".
15689    #[tokio::test]
15690    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
15691        assert_eq!(bind_role_versions_for(None, 143).await, None);
15692        assert_eq!(
15693            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
15694            None
15695        );
15696    }
15697
15698    #[tokio::test]
15699    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
15700        let registry = Arc::new(Registry::default());
15701        let forwarding = Arc::new(ForwardingTable::default());
15702        let handler =
15703            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15704        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
15705        hello_via_sink(
15706            &handler,
15707            &module_ctx,
15708            &mut module_rx,
15709            hello_frame("aft", PROTOCOL_VERSION, 7),
15710        )
15711        .await;
15712
15713        let nine: BTreeMap<String, String> = (0..9)
15714            .map(|index| (format!("role-{index}"), "v1".to_string()))
15715            .collect();
15716        for (label, malformed) in [
15717            (
15718                "invalid role name",
15719                role_versions(&[("Tool_Provider", "v1")]),
15720            ),
15721            ("invalid version", role_versions(&[("tool-provider", "v0")])),
15722            ("nine entries", nine),
15723        ] {
15724            // A refused open answers at once; one that reached the module
15725            // would wait for its bind ack and trip this timeout.
15726            let responses = tokio::time::timeout(
15727                Duration::from_secs(1),
15728                handler.handle_control_frame(
15729                    &route_ctx(ConnectionId::new(148)).0,
15730                    route_open_frame_with_role_versions(
15731                        404,
15732                        "aft",
15733                        unique_project_root("role-versions-malformed"),
15734                        Some(malformed),
15735                    ),
15736                ),
15737            )
15738            .await
15739            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
15740            .unwrap();
15741            assert_eq!(responses.len(), 1, "{label}");
15742            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
15743            let error = parse_error(&responses[0]);
15744            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
15745            assert_eq!(
15746                error["detail"]["field"], "role_versions",
15747                "{label}: {error}"
15748            );
15749            assert!(
15750                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
15751                "{label}: a malformed declaration is terminal"
15752            );
15753            assert!(
15754                module_rx.try_recv().is_err(),
15755                "{label}: the module must never see a bind"
15756            );
15757        }
15758    }
15759
15760    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
15761    /// consumer can tell this daemon forwards the field from one that would
15762    /// drop it.
15763    #[tokio::test]
15764    async fn route_role_versions_capability_is_advertised() {
15765        let handler = ControlHandler::new(Arc::new(Registry::default()));
15766        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
15767        let ack = hello_via_sink(
15768            &handler,
15769            &ctx,
15770            &mut rx,
15771            hello_frame("m", PROTOCOL_VERSION, 1),
15772        )
15773        .await;
15774        let ack = parse_ack(&ack);
15775        assert!(
15776            ack.subc_capabilities
15777                .iter()
15778                .any(|c| c == "route-role-versions/v1"),
15779            "{:?}",
15780            ack.subc_capabilities
15781        );
15782
15783        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15784        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15785        let reply = handler
15786            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
15787            .await
15788            .unwrap()
15789            .pop()
15790            .unwrap();
15791        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15792            serde_json::from_slice(&reply.body).unwrap()
15793        else {
15794            panic!("not a server.describe reply");
15795        };
15796        assert!(
15797            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
15798            "{capabilities:?}"
15799        );
15800    }
15801
15802    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
15803    /// catalog entry without `events` (the module declares none) from a daemon
15804    /// that never reports declared events.
15805    #[tokio::test]
15806    async fn catalog_events_capability_is_advertised() {
15807        let handler = ControlHandler::new(Arc::new(Registry::default()));
15808        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15809        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15810        let reply = handler
15811            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
15812            .await
15813            .unwrap()
15814            .pop()
15815            .unwrap();
15816        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15817            serde_json::from_slice(&reply.body).unwrap()
15818        else {
15819            panic!("not a server.describe reply");
15820        };
15821        assert!(
15822            capabilities
15823                .iter()
15824                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
15825            "{capabilities:?}"
15826        );
15827    }
15828
15829    #[tokio::test]
15830    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
15831        let handler = ControlHandler::new(Arc::new(Registry::default()));
15832        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
15833        let ack = hello_via_sink(
15834            &handler,
15835            &module,
15836            &mut module_rx,
15837            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
15838        )
15839        .await;
15840        let ack = parse_ack(&ack);
15841        assert!(
15842            ack.subc_capabilities
15843                .iter()
15844                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15845            "{:?}",
15846            ack.subc_capabilities
15847        );
15848        assert!(ack
15849            .subc_capabilities
15850            .iter()
15851            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15852
15853        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
15854        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15855        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15856        let reply = handler
15857            .handle_control_frame(&client, frame)
15858            .await
15859            .unwrap()
15860            .pop()
15861            .unwrap();
15862        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15863            serde_json::from_slice(&reply.body).unwrap()
15864        else {
15865            panic!("not a server.describe reply");
15866        };
15867        assert!(
15868            capabilities
15869                .iter()
15870                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15871            "{capabilities:?}"
15872        );
15873        assert!(capabilities
15874            .iter()
15875            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15876    }
15877
15878    #[tokio::test]
15879    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
15880        let handler = ControlHandler::new(Arc::new(Registry::default()));
15881        let (module, mut rx) = route_ctx(ConnectionId::new(156));
15882        let ack = hello_via_sink(
15883            &handler,
15884            &module,
15885            &mut rx,
15886            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
15887        )
15888        .await;
15889        assert!(parse_ack(&ack)
15890            .subc_capabilities
15891            .iter()
15892            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
15893        assert!(parse_ack(&ack)
15894            .subc_capabilities
15895            .iter()
15896            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
15897        let (client, _rx) = route_ctx(ConnectionId::new(157));
15898        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15899        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15900        let reply = handler
15901            .handle_control_frame(&client, frame)
15902            .await
15903            .unwrap()
15904            .pop()
15905            .unwrap();
15906        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15907            serde_json::from_slice(&reply.body).unwrap()
15908        else {
15909            panic!("server.describe reply")
15910        };
15911        assert!(capabilities
15912            .iter()
15913            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
15914        assert!(capabilities
15915            .iter()
15916            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
15917    }
15918
15919    #[tokio::test]
15920    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
15921        let handler = ControlHandler::new(Arc::new(Registry::default()));
15922        let (module, mut rx) = route_ctx(ConnectionId::new(158));
15923        let ack = hello_via_sink(
15924            &handler,
15925            &module,
15926            &mut rx,
15927            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
15928        )
15929        .await;
15930        assert!(parse_ack(&ack)
15931            .subc_capabilities
15932            .iter()
15933            .any(|cap| cap == "scope-workdir/v1"));
15934        let (client, _rx) = route_ctx(ConnectionId::new(159));
15935        let frame = Frame::build(
15936            FrameType::Request,
15937            control_flags(),
15938            0,
15939            0,
15940            5,
15941            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15942        )
15943        .unwrap();
15944        let reply = handler
15945            .handle_control_frame(&client, frame)
15946            .await
15947            .unwrap()
15948            .remove(0);
15949        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15950            serde_json::from_slice(&reply.body).unwrap()
15951        else {
15952            panic!("server.describe reply")
15953        };
15954        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
15955    }
15956
15957    #[tokio::test]
15958    async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
15959        let handler = ControlHandler::new(Arc::new(Registry::default()));
15960        let (module, mut rx) = route_ctx(ConnectionId::new(160));
15961        let ack = hello_via_sink(
15962            &handler,
15963            &module,
15964            &mut rx,
15965            hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
15966        )
15967        .await;
15968        assert!(parse_ack(&ack)
15969            .subc_capabilities
15970            .iter()
15971            .any(|cap| cap == "worker-runs/v1"));
15972        let (client, _rx) = route_ctx(ConnectionId::new(161));
15973        let frame = Frame::build(
15974            FrameType::Request,
15975            control_flags(),
15976            0,
15977            0,
15978            5,
15979            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15980        )
15981        .unwrap();
15982        let reply = handler
15983            .handle_control_frame(&client, frame)
15984            .await
15985            .unwrap()
15986            .remove(0);
15987        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15988            serde_json::from_slice(&reply.body).unwrap()
15989        else {
15990            panic!("server.describe reply")
15991        };
15992        assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
15993    }
15994
15995    #[tokio::test]
15996    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
15997        let handler = ControlHandler::new(Arc::new(Registry::default()));
15998        let (module, mut rx) = route_ctx(ConnectionId::new(154));
15999        let ack = hello_via_sink(
16000            &handler,
16001            &module,
16002            &mut rx,
16003            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
16004        )
16005        .await;
16006        assert!(parse_ack(&ack)
16007            .subc_capabilities
16008            .iter()
16009            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16010        let (client, _rx) = route_ctx(ConnectionId::new(155));
16011        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16012        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16013        let reply = handler
16014            .handle_control_frame(&client, frame)
16015            .await
16016            .unwrap()
16017            .pop()
16018            .unwrap();
16019        let ClientControlResponse::ServerDescribe { capabilities, .. } =
16020            serde_json::from_slice(&reply.body).unwrap()
16021        else {
16022            panic!("not a server.describe reply")
16023        };
16024        assert!(capabilities
16025            .iter()
16026            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16027    }
16028
16029    #[tokio::test]
16030    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
16031        let registry = Arc::new(Registry::default());
16032        let forwarding = Arc::new(ForwardingTable::default());
16033        let handler =
16034            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16035                .with_health_probe_timeout(Duration::from_secs(5));
16036        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
16037        hello_via_sink(
16038            &handler,
16039            &module_ctx,
16040            &mut module_rx,
16041            non_routable_hello_frame_with_control_ops(
16042                "mcp",
16043                300,
16044                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
16045            ),
16046        )
16047        .await;
16048        assert!(registry
16049            .get_module("mcp")
16050            .unwrap()
16051            .unwrap()
16052            .manifest
16053            .provides
16054            .is_empty());
16055
16056        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
16057        let route_response = handler
16058            .handle_control_frame(
16059                &route_client_ctx,
16060                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
16061            )
16062            .await
16063            .unwrap();
16064        assert_eq!(route_response[0].header.ty, FrameType::Error);
16065        assert_eq!(
16066            parse_error(&route_response[0])["code"],
16067            "target_unavailable"
16068        );
16069        assert!(parse_error(&route_response[0])["message"]
16070            .as_str()
16071            .unwrap()
16072            .contains("does not provide the requested target"));
16073        assert!(module_rx.try_recv().is_err());
16074
16075        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
16076        let health_handler = handler.clone();
16077        let health_task = tokio::spawn(async move {
16078            health_handler
16079                .handle_control_frame(
16080                    &health_client_ctx,
16081                    supervisor_health_probe_frame(302, "mcp"),
16082                )
16083                .await
16084                .unwrap()
16085        });
16086        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
16087            .await
16088            .unwrap()
16089            .unwrap();
16090        assert_eq!(
16091            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
16092            ModuleControlRequest::HealthCheck {}
16093        );
16094        handler
16095            .handle_control_frame(
16096                &module_ctx,
16097                health_response(health_frame.header.corr, HealthStatus::Ok),
16098            )
16099            .await
16100            .unwrap();
16101        let health_response = health_task.await.unwrap();
16102        assert_eq!(health_response[0].header.ty, FrameType::Response);
16103        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
16104            ClientControlResponse::SupervisorHealthProbe {
16105                module_id, status, ..
16106            } => {
16107                assert_eq!(module_id, "mcp");
16108                assert_eq!(status, HealthStatus::Ok);
16109            }
16110            other => panic!("unexpected health response: {other:?}"),
16111        }
16112
16113        // Exercise the forwarding cleanup path directly while leaving the registry
16114        // advertisement in place. If cleanup leaves a stale control sink behind,
16115        // the next probe will enqueue onto it and wait for the long probe timeout
16116        // instead of returning an immediate no-connection error.
16117        forwarding
16118            .cleanup_connection(module_ctx.connection_id)
16119            .unwrap();
16120        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
16121        let cleanup_response = tokio::time::timeout(
16122            Duration::from_millis(200),
16123            handler.handle_control_frame(
16124                &cleanup_probe_ctx,
16125                supervisor_health_probe_frame(303, "mcp"),
16126            ),
16127        )
16128        .await
16129        .expect("probe should fail immediately when the control lane is gone")
16130        .unwrap();
16131        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
16132        assert_eq!(
16133            parse_error(&cleanup_response[0])["code"],
16134            "target_unavailable"
16135        );
16136        assert!(parse_error(&cleanup_response[0])["message"]
16137            .as_str()
16138            .unwrap()
16139            .contains("no module connection"));
16140
16141        handler
16142            .cleanup_connection(module_ctx.connection_id)
16143            .unwrap();
16144    }
16145
16146    #[tokio::test]
16147    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
16148        let registry = Arc::new(Registry::default());
16149        let supervisor_handle = SupervisorHandle::new();
16150        let supervisor =
16151            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
16152                .with_handle(supervisor_handle.clone())
16153                .with_connection_file_path(
16154                    std::env::temp_dir()
16155                        .join(format!("subc-route-open-warming-{}", std::process::id())),
16156                );
16157        let module = supervisor
16158            .supervise_configured(
16159                ModuleSpec {
16160                    script: None,
16161                    carry_containment: false,
16162                    module_id: "warming".to_string(),
16163                    program: fake_aft_stub_path(),
16164                    args: Vec::new(),
16165                    env: Vec::new(),
16166                    reserved: false,
16167                    reserved_prefixes: Vec::new(),
16168                    protocol: ModuleProtocol::Subc,
16169                    overlap: Default::default(),
16170                },
16171                true,
16172            )
16173            .unwrap();
16174        assert_eq!(module.state().unwrap(), ModuleState::Running);
16175
16176        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
16177        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
16178        let response = handler
16179            .handle_control_frame(
16180                &ctx,
16181                route_open_frame(304, "warming", unique_project_root("warming")),
16182            )
16183            .await
16184            .unwrap();
16185        module.stop().await.unwrap();
16186
16187        assert_eq!(response[0].header.ty, FrameType::Error);
16188        let error = parse_error(&response[0]);
16189        assert_eq!(error["code"], "module_warming");
16190        assert!(error["message"]
16191            .as_str()
16192            .unwrap()
16193            .contains("state=running, enabled=true, live=false"));
16194    }
16195
16196    #[test]
16197    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
16198        let handler = ControlHandler::new(Arc::new(Registry::default()));
16199        let capture = EventCapture::default();
16200        let _subscriber =
16201            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16202        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
16203        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
16204        let pending = (0..limit).collect::<Vec<_>>();
16205        let response = handler
16206            .route_open_capacity_refusal(
16207                &ctx,
16208                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
16209                "busy",
16210                pending.len(),
16211                limit,
16212            )
16213            .unwrap();
16214        assert_eq!(parse_error(&response)["code"], "target_unavailable");
16215        let event = capture
16216            .events()
16217            .into_iter()
16218            .find(|event| {
16219                event.target == "control"
16220                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
16221            })
16222            .expect("connection admission refusal event");
16223        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16224        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16225        // Refused before `handle_route_open` parses the request, so the caller
16226        // fields are read from the frame itself.
16227        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
16228        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
16229    }
16230
16231    #[test]
16232    fn route_open_target_cap_logs_admission_reason_and_capacity() {
16233        let handler = ControlHandler::new(Arc::new(Registry::default()));
16234        let capture = EventCapture::default();
16235        let _subscriber =
16236            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16237        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
16238        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
16239        let guards = (0..limit)
16240            .map(|_| {
16241                handler
16242                    .route_bind_concurrency
16243                    .try_admit("busy", limit)
16244                    .unwrap()
16245            })
16246            .collect::<Vec<_>>();
16247        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
16248            Err(in_flight) => in_flight,
16249            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
16250        };
16251        let response = handler
16252            .route_open_target_capacity_refusal(
16253                &ctx,
16254                &RouteOpenCaller::unknown(),
16255                &route_open_frame(397, "busy", unique_project_root("target-cap")),
16256                "busy",
16257                in_flight,
16258            )
16259            .unwrap();
16260        assert_eq!(parse_error(&response)["code"], "target_unavailable");
16261        let event = capture
16262            .events()
16263            .into_iter()
16264            .find(|event| {
16265                event.target == "control"
16266                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
16267            })
16268            .expect("target admission refusal event");
16269        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16270        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16271        drop(guards);
16272    }
16273
16274    #[cfg(unix)]
16275    #[tokio::test(start_paused = true)]
16276    async fn supervisor_list_shows_a_failed_modules_next_retry() {
16277        let registry = Arc::new(Registry::default());
16278        let handle = SupervisorHandle::new();
16279        let supervisor =
16280            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
16281                .with_handle(handle.clone());
16282        let module = supervisor
16283            .spawn(crate::ModuleSpec {
16284                script: None,
16285                carry_containment: false,
16286                module_id: "retry-list".into(),
16287                program: PathBuf::from("/bin/sh"),
16288                args: vec!["-c".into(), "exit 17".into()],
16289                env: vec![],
16290                reserved: false,
16291                reserved_prefixes: vec![],
16292                protocol: ModuleProtocol::None,
16293                overlap: Default::default(),
16294            })
16295            .unwrap();
16296        let deadline = std::time::Instant::now() + Duration::from_secs(10);
16297        while module.state().unwrap() != crate::ModuleState::Failed {
16298            assert!(
16299                std::time::Instant::now() < deadline,
16300                "waiting for budget failure"
16301            );
16302            tokio::task::yield_now().await;
16303        }
16304        let handler = ControlHandler::new(registry).with_supervisor(handle);
16305        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16306        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16307        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16308        assert_eq!(body["modules"][0]["state"], "failed");
16309        let retry = body["modules"][0]["next_retry_at_ms"]
16310            .as_u64()
16311            .expect("scheduled retry on the wire");
16312        let now = std::time::SystemTime::now()
16313            .duration_since(std::time::UNIX_EPOCH)
16314            .unwrap()
16315            .as_millis() as u64;
16316        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
16317        module.set_enabled(false).await.unwrap();
16318        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16319        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16320        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16321        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
16322    }
16323
16324    #[tokio::test]
16325    async fn slow_step_before_pid_read_does_not_spend_its_budget() {
16326        let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
16327        let supervisor =
16328            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
16329        let module = supervisor
16330            .spawn(crate::ModuleSpec {
16331                script: None,
16332                carry_containment: false,
16333                module_id: "pid-read-budget".into(),
16334                program: std::env::current_exe().unwrap(),
16335                args: vec![
16336                    "--exact".into(),
16337                    "control::tests::pid_resource_read_fixture".into(),
16338                ],
16339                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16340                    .into_iter()
16341                    .map(|key| (key.into(), dir.display().to_string()))
16342                    .chain(std::iter::once((
16343                        "SUBC_SUBTREE_PID_FIXTURE".into(),
16344                        "1".into(),
16345                    )))
16346                    .collect(),
16347                reserved: false,
16348                reserved_prefixes: Vec::new(),
16349                protocol: ModuleProtocol::None,
16350                overlap: Default::default(),
16351            })
16352            .unwrap();
16353        let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16354        while !matches!(
16355            module.child_resource_usage(),
16356            subc_control::ChildResourceUsage::Measured(_)
16357        ) {
16358            assert!(
16359                tokio::time::Instant::now() < ready_deadline,
16360                "fixture process was not measurable"
16361            );
16362            tokio::time::sleep(Duration::from_millis(10)).await;
16363        }
16364        tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
16365        let reading = {
16366            let _guard = crate::subtree_resources::fault_read_guard().await;
16367            crate::subtree_resources::read_process(module.clone()).await
16368        };
16369        stop_resource_fixture(&module, &dir).await;
16370        assert!(
16371            matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
16372            "{reading:?}"
16373        );
16374    }
16375
16376    #[test]
16377    fn pid_resource_read_fixture() {
16378        if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
16379            let dir = std::path::PathBuf::from(std::env::var_os("XDG_DATA_HOME").unwrap());
16380            let deadline = std::time::Instant::now() + Duration::from_secs(60);
16381            while !dir.join("stop-module").exists() && std::time::Instant::now() < deadline {
16382                std::thread::sleep(Duration::from_millis(5));
16383            }
16384            std::fs::write(dir.join("module-stopped"), "").unwrap();
16385        }
16386    }
16387
16388    async fn stop_resource_fixture(
16389        module: &crate::supervise::SupervisedModule,
16390        dir: &std::path::Path,
16391    ) {
16392        let stopping = module.stop();
16393        tokio::pin!(stopping);
16394        tokio::select! {
16395            result = &mut stopping => result.unwrap(),
16396            () = async {
16397                while module.state().unwrap() != crate::supervise::ModuleState::Draining {
16398                    tokio::time::sleep(Duration::from_millis(5)).await;
16399                }
16400                std::fs::write(dir.join("stop-module"), "").unwrap();
16401            } => stopping.await.unwrap(),
16402        }
16403        #[cfg(windows)]
16404        assert!(
16405            dir.join("module-stopped").exists(),
16406            "the resource fixture must exit cooperatively, not wait for the drain kill"
16407        );
16408        assert_eq!(
16409            module.state().unwrap(),
16410            crate::supervise::ModuleState::Stopped
16411        );
16412    }
16413
16414    #[tokio::test]
16415    async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
16416        let registry = Arc::new(Registry::default());
16417        let handle = SupervisorHandle::new();
16418        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16419            .with_handle(handle.clone());
16420        supervisor
16421            .supervise_configured(
16422                crate::ModuleSpec {
16423                    script: None,
16424                    module_id: "disabled-resource-reading".into(),
16425                    program: std::env::current_exe().unwrap(),
16426                    args: Vec::new(),
16427                    env: Vec::new(),
16428                    reserved: false,
16429                    reserved_prefixes: Vec::new(),
16430                    protocol: ModuleProtocol::None,
16431                    overlap: Default::default(),
16432                    carry_containment: false,
16433                },
16434                false,
16435            )
16436            .unwrap();
16437        let handler = ControlHandler::new(registry).with_supervisor(handle);
16438        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16439        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16440        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16441        assert!(body["modules"][0].get("tree").is_none());
16442        assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
16443    }
16444
16445    #[tokio::test]
16446    async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
16447        let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
16448        let registry = Arc::new(Registry::default());
16449        let handle = SupervisorHandle::new();
16450        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16451            .with_handle(handle.clone());
16452        let module = supervisor
16453            .spawn(crate::ModuleSpec {
16454                script: None,
16455                carry_containment: false,
16456                module_id: "list-process-priority".into(),
16457                program: std::env::current_exe().unwrap(),
16458                args: vec![
16459                    "--exact".into(),
16460                    "control::tests::pid_resource_read_fixture".into(),
16461                ],
16462                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16463                    .into_iter()
16464                    .map(|key| (key.into(), dir.display().to_string()))
16465                    .chain(std::iter::once((
16466                        "SUBC_SUBTREE_PID_FIXTURE".into(),
16467                        "1".into(),
16468                    )))
16469                    .collect(),
16470                reserved: false,
16471                reserved_prefixes: Vec::new(),
16472                protocol: ModuleProtocol::None,
16473                overlap: Default::default(),
16474            })
16475            .unwrap();
16476        let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16477        loop {
16478            let image = module.running_image_agreement().await;
16479            if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
16480                break;
16481            }
16482            assert!(
16483                tokio::time::Instant::now() < ready_deadline,
16484                "the fixture process never became observable: {image:?}"
16485            );
16486            tokio::time::sleep(Duration::from_millis(10)).await;
16487        }
16488        let guard = crate::subtree_resources::fault_read_guard().await;
16489        let (target, release, _) = crate::subtree_resources::held();
16490        module.set_subtree_target_for_test(target);
16491        let handler = ControlHandler::new(registry).with_supervisor(handle);
16492        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16493        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16494        drop(release);
16495        drop(guard);
16496        stop_resource_fixture(&module, &dir).await;
16497        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16498        assert_eq!(
16499            body["modules"][0]["resources"]["status"], "measured",
16500            "the optional tree timeout must not discard the process reading: {body}"
16501        );
16502        assert!(body["modules"][0].get("tree").is_none());
16503        assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
16504        assert_eq!(
16505            body["modules"][0]["pending_reload"]["image"]["status"], "match",
16506            "the tree timeout must not hide a completed image observation"
16507        );
16508    }
16509
16510    #[cfg(any(target_os = "linux", target_os = "macos"))]
16511    #[tokio::test]
16512    async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
16513        let mut handler = ControlHandler::new(Arc::new(Registry::default()))
16514            .with_daemon_provenance(
16515                std::process::id(),
16516                0,
16517                Some(std::env::current_exe().unwrap()),
16518                None,
16519                None,
16520            );
16521        let (probe, release, began) = ExecutableIdentityProbe::held();
16522        handler.daemon_provenance.probe = probe;
16523        let observe = |handler: ControlHandler, corr| {
16524            tokio::spawn(async move {
16525                let (ctx, _) = route_ctx(ConnectionId::new(corr));
16526                let frame =
16527                    Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
16528                        .unwrap();
16529                handler
16530                    .handle_supervisor_provenance(&ctx, frame, None)
16531                    .await
16532                    .unwrap()
16533            })
16534        };
16535        let explicit = observe(handler.clone(), 114);
16536        began.await.unwrap();
16537        let concurrent = observe(handler.clone(), 115);
16538        // Keep explicit provenance waiting on its first image observation beyond
16539        // the list deadline; release it only after the concurrent list replies.
16540        tokio::time::sleep(Duration::from_millis(300)).await;
16541        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16542        let list = tokio::time::timeout(
16543            Duration::from_secs(1),
16544            handler.handle_supervisor_list(frame),
16545        )
16546        .await;
16547        let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
16548        drop(release);
16549        let replies = explicit.await.unwrap();
16550        let concurrent_replies = concurrent.await.unwrap();
16551        assert!(
16552            explicit_was_pending,
16553            "explicit provenance must await the full observation"
16554        );
16555        let list = list
16556            .expect("a concurrent explicit probe must not stall list")
16557            .unwrap();
16558        let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
16559        assert_eq!(list_body["op"], "supervisor.list");
16560        for replies in [replies, concurrent_replies] {
16561            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16562            assert_eq!(
16563                body["daemon"]["daemon_observed"]["running_image"]["status"],
16564                "match"
16565            );
16566        }
16567    }
16568
16569    #[tokio::test]
16570    async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
16571        let registry = Arc::new(Registry::default());
16572        let handle = SupervisorHandle::new();
16573        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16574            .with_handle(handle.clone());
16575        let module = supervisor
16576            .supervise_configured(
16577                crate::ModuleSpec {
16578                    script: None,
16579                    carry_containment: false,
16580                    module_id: "memory-read".into(),
16581                    program: PathBuf::from("disabled-module"),
16582                    args: Vec::new(),
16583                    env: Vec::new(),
16584                    reserved: false,
16585                    reserved_prefixes: Vec::new(),
16586                    protocol: ModuleProtocol::Subc,
16587                    overlap: Default::default(),
16588                },
16589                true,
16590            )
16591            .unwrap();
16592        let _guard = crate::subtree_resources::fault_read_guard().await;
16593        let (target, release, began) = crate::subtree_resources::held();
16594        // Optional list work may be skipped when its deadline is already spent.
16595        // Establish the stalled fixture before asking the list to observe it.
16596        let stalled_target = target.clone();
16597        let stalled = tokio::task::spawn_blocking(move || {
16598            stalled_target.tree(&subc_os::ProcessTable::default())
16599        });
16600        began.await.unwrap();
16601        module.set_subtree_target_for_test(target);
16602        let handler = ControlHandler::new(registry).with_supervisor(handle);
16603        let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
16604        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16605        let replies = handler
16606            .handle_client_control_request(
16607                &ctx,
16608                frame,
16609                ClientControlRequest::SupervisorList {},
16610                &mut None,
16611            )
16612            .await
16613            .unwrap();
16614        assert!(
16615            replies.is_empty(),
16616            "list reply must be detached from the connection reader"
16617        );
16618        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16619        let replies = handler
16620            .handle_client_control_request(
16621                &ctx,
16622                frame,
16623                ClientControlRequest::SupervisorSpawnSnapshot {},
16624                &mut None,
16625            )
16626            .await
16627            .unwrap();
16628        assert_eq!(
16629            replies.len(),
16630            1,
16631            "another request must complete while the subtree read stalls"
16632        );
16633        let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16634            .await
16635            .unwrap()
16636            .unwrap();
16637        release.send(()).unwrap();
16638        drop(release);
16639        stalled.await.unwrap();
16640        let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
16641        assert!(
16642            body["modules"][0].get("tree").is_none(),
16643            "an unknown tree reading must be omitted after the bounded read expires"
16644        );
16645        // Lists export completed work rather than measuring synchronously.
16646        // Complete a refresh after the fixture is released before asserting
16647        // that a later list contains its measured tree.
16648        handler
16649            .resource_tracker
16650            .read(handler.supervisor.list(), false, "released_tree_fixture")
16651            .await;
16652        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16653        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16654        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16655        assert_eq!(
16656            body["modules"][0]["tree"]["source"], "process_tree",
16657            "the next request must export the completed refresh"
16658        );
16659        assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
16660    }
16661
16662    #[tokio::test]
16663    async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
16664        use subc_control::{
16665            AgentCommandResources, ChildMemoryKind, ChildResourceReading,
16666            ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
16667            TreeReading, TreeSource,
16668        };
16669        let registry = Arc::new(Registry::default());
16670        let handle = SupervisorHandle::new();
16671        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16672            .with_handle(handle.clone());
16673        supervisor
16674            .supervise_configured(
16675                crate::ModuleSpec {
16676                    script: None,
16677                    carry_containment: false,
16678                    module_id: "budget-worker".into(),
16679                    program: PathBuf::from("disabled-module"),
16680                    args: Vec::new(),
16681                    env: Vec::new(),
16682                    reserved: false,
16683                    reserved_prefixes: Vec::new(),
16684                    protocol: ModuleProtocol::Subc,
16685                    overlap: Default::default(),
16686                },
16687                true,
16688            )
16689            .unwrap();
16690        let handler = ControlHandler::new(registry).with_supervisor(handle);
16691        let now = u64::try_from(
16692            std::time::SystemTime::now()
16693                .duration_since(std::time::UNIX_EPOCH)
16694                .unwrap()
16695                .as_millis(),
16696        )
16697        .unwrap();
16698        let measured = now - 1_000;
16699        let hour = CpuLastHour::new(100, 60_000);
16700        let daemon = DaemonResources::new(7, 1, "test")
16701            .with_measured_at_ms(Some(measured))
16702            .with_resources(Some(ChildResourceUsage::Measured(
16703                ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
16704            )))
16705            .with_cpu_last_hour(Some(hour.clone()));
16706        let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
16707            .with_measured_at_ms(Some(measured))
16708            .with_cpu_last_hour(Some(hour.clone()));
16709        let tree = TreeReading::new(
16710            150,
16711            50,
16712            ChildMemoryKind::ResidentSet,
16713            1,
16714            TreeSource::ProcessTree,
16715        )
16716        .with_measured_at_ms(Some(measured));
16717        handler.resource_tracker.seed_committed_for_test(
16718            &handler.supervisor.list(),
16719            crate::resource_totals::Readings {
16720                modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
16721                daemon: Some(daemon),
16722                agent_commands: Some(vec![agents]),
16723                ..Default::default()
16724            },
16725            now,
16726        );
16727        list_diagnostics::scope(async {
16728            tokio::time::sleep(Duration::from_millis(110)).await;
16729            assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
16730            let request =
16731                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16732            let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
16733            let ClientControlResponse::SupervisorList {
16734                modules,
16735                daemon,
16736                agent_commands,
16737                ..
16738            } = serde_json::from_slice(&replies[0].body).unwrap()
16739            else {
16740                panic!("expected a list response")
16741            };
16742            assert!(matches!(
16743                modules[0].resources,
16744                Some(ChildResourceUsage::Unavailable {
16745                    reason: ChildResourceUnavailableReason::Unreadable
16746                })
16747            ));
16748            assert_eq!(
16749                (
16750                    modules[0].tree.as_ref().unwrap().cpu_ms,
16751                    modules[0].tree.as_ref().unwrap().measured_at_ms
16752                ),
16753                (150, Some(measured))
16754            );
16755            let daemon = daemon.unwrap();
16756            let agents = agent_commands.unwrap();
16757            assert_eq!(daemon.measured_at_ms, Some(measured));
16758            assert_eq!(
16759                (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
16760                (1, 50, Some(measured))
16761            );
16762            for hour in [
16763                modules[0].cpu_last_hour.as_ref(),
16764                daemon.cpu_last_hour.as_ref(),
16765                agents[0].cpu_last_hour.as_ref(),
16766            ] {
16767                let hour = hour.unwrap();
16768                assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
16769            }
16770        })
16771        .await;
16772    }
16773
16774    #[tokio::test]
16775    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
16776        let registry = Arc::new(Registry::default());
16777        let handle = SupervisorHandle::new();
16778        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16779            .with_handle(handle.clone());
16780        supervisor
16781            .supervise_configured(
16782                crate::ModuleSpec {
16783                    script: None,
16784                    carry_containment: false,
16785                    module_id: "drift".into(),
16786                    program: PathBuf::from("disabled-module"),
16787                    args: Vec::new(),
16788                    env: Vec::new(),
16789                    reserved: false,
16790                    reserved_prefixes: Vec::new(),
16791                    protocol: ModuleProtocol::Subc,
16792                    overlap: Default::default(),
16793                },
16794                false,
16795            )
16796            .unwrap();
16797        let handler = ControlHandler::new(registry).with_supervisor(handle);
16798        for configured in [None, Some(false), Some(true)] {
16799            handler.capability_evaluator.configure(
16800                configured.map(|enabled| ("drift".to_string(), enabled)),
16801                BTreeMap::new(),
16802            );
16803            let frame =
16804                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16805            let replies = handler
16806                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
16807                .await
16808                .unwrap();
16809            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16810            assert!(body.get("configured_enabled").is_none());
16811            assert_eq!(body["op"], "supervisor.ack");
16812            let frame =
16813                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16814            let replies = handler.handle_supervisor_list(frame).await.unwrap();
16815            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16816            assert_eq!(
16817                body["modules"][0].get("configured_enabled"),
16818                configured.map(Value::Bool).as_ref()
16819            );
16820            assert!(body["modules"][0].get("tree").is_none());
16821            assert!(body["modules"][0].get("subtree").is_none());
16822        }
16823    }
16824
16825    #[tokio::test(flavor = "current_thread")]
16826    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
16827        let handler = ControlHandler::new(Arc::new(Registry::default()));
16828        let capture = EventCapture::default();
16829        let _subscriber =
16830            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16831        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
16832        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16833
16834        let _ = handler
16835            .handle_client_control_request(
16836                &ctx,
16837                frame,
16838                ClientControlRequest::SupervisorSetEnabled {
16839                    module_id: "broca".to_string(),
16840                    enabled: false,
16841                },
16842                &mut None,
16843            )
16844            .await
16845            .unwrap();
16846
16847        let events = capture
16848            .events()
16849            .into_iter()
16850            .filter(|event| {
16851                event.target == "control"
16852                    && event.fields.get("message").map(String::as_str)
16853                        == Some("supervisor request received")
16854            })
16855            .collect::<Vec<_>>();
16856        assert_eq!(events.len(), 1, "one supervisor request log line");
16857        let event = &events[0];
16858        assert_eq!(event.level, tracing::Level::INFO);
16859        assert_eq!(
16860            event.fields.get("op"),
16861            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
16862        );
16863        assert_eq!(
16864            event.fields.get("module_id"),
16865            Some(&"\"broca\"".to_string())
16866        );
16867        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
16868        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
16869        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
16870    }
16871
16872    #[tokio::test(flavor = "current_thread")]
16873    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
16874        let registry = Arc::new(Registry::default());
16875        let module_connection = ConnectionId::new(303);
16876        registry
16877            .register_with_control_ops(
16878                manifest("aft", PROTOCOL_VERSION),
16879                PROTOCOL_VERSION,
16880                module_connection,
16881                module_baseline_control_ops(),
16882            )
16883            .unwrap();
16884        let handler = ControlHandler::new(registry);
16885        let capture = EventCapture::default();
16886        let _subscriber =
16887            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16888        let (ctx, _rx) = route_ctx(module_connection);
16889        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16890
16891        let _ = handler
16892            .handle_client_control_request(
16893                &ctx,
16894                frame,
16895                ClientControlRequest::SupervisorSetEnabled {
16896                    module_id: "broca".to_string(),
16897                    enabled: false,
16898                },
16899                &mut None,
16900            )
16901            .await
16902            .unwrap();
16903
16904        let callers = capture
16905            .events()
16906            .into_iter()
16907            .filter(|event| {
16908                event.fields.get("message").map(String::as_str)
16909                    == Some("supervisor request received")
16910            })
16911            .map(|event| event.fields.get("caller").cloned())
16912            .collect::<Vec<_>>();
16913        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
16914    }
16915
16916    #[tokio::test(flavor = "current_thread")]
16917    async fn supervisor_list_does_not_log_request_received() {
16918        let handler = ControlHandler::new(Arc::new(Registry::default()));
16919        let capture = EventCapture::default();
16920        let _subscriber =
16921            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16922        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
16923        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16924
16925        handler
16926            .handle_client_control_request(
16927                &ctx,
16928                frame,
16929                ClientControlRequest::SupervisorList {},
16930                &mut None,
16931            )
16932            .await
16933            .unwrap();
16934
16935        assert!(capture.events().into_iter().all(|event| {
16936            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
16937        }));
16938    }
16939
16940    #[tokio::test(flavor = "current_thread")]
16941    async fn supervisor_rescan_preview_does_not_log_request_received() {
16942        let handler = ControlHandler::new(Arc::new(Registry::default()));
16943        let capture = EventCapture::default();
16944        let _subscriber =
16945            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16946        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
16947        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16948
16949        handler
16950            .handle_client_control_request(
16951                &ctx,
16952                frame,
16953                ClientControlRequest::SupervisorRescan { preview: true },
16954                &mut None,
16955            )
16956            .await
16957            .unwrap();
16958
16959        assert!(capture.events().into_iter().all(|event| {
16960            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
16961        }));
16962    }
16963
16964    struct ApproveOperator;
16965    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
16966        fn prompt(
16967            &self,
16968            _: &str,
16969            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
16970        ) -> crate::operator_confirm::ProviderResult {
16971            crate::operator_confirm::ProviderResult::Approved
16972        }
16973    }
16974
16975    #[tokio::test]
16976    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
16977        assert_eq!(
16978            ControlHandler::default().privileged_operator_prompt,
16979            PrivilegedOperatorPrompt::Log
16980        );
16981        for (initial, changed, running) in [
16982            ("log", "enforce", PrivilegedOperatorPrompt::Log),
16983            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
16984        ] {
16985            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
16986            let path = dir.join("subc.jsonc");
16987            std::fs::write(
16988                &path,
16989                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
16990            )
16991            .unwrap();
16992            let config = crate::daemon_config::load(&path).unwrap().unwrap();
16993            let registry = Arc::new(Registry::default());
16994            let handle = SupervisorHandle::new();
16995            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
16996                .with_handle(handle.clone());
16997            let handler = ControlHandler::new(registry)
16998                .with_supervisor(handle)
16999                .with_privileged_operator_prompt(config.privileged_operator_prompt)
17000                .with_supervisor_rescan(supervisor, &path, config.port);
17001            handler.forwarding.operator_confirms().configure(
17002                Arc::new(ApproveOperator),
17003                crate::operator_confirm::OperatorLimits::default(),
17004            );
17005            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
17006
17007            for (disk_mode, needs_restart) in [
17008                (Some(initial), false),
17009                (Some(changed), true),
17010                (Some(changed), true),
17011                (None, running == PrivilegedOperatorPrompt::Enforce),
17012                (Some(initial), false),
17013            ] {
17014                let mut doc = json!({"version":1});
17015                if let Some(mode) = disk_mode {
17016                    doc["privileged_operator_prompt"] = json!(mode);
17017                }
17018                std::fs::write(&path, doc.to_string()).unwrap();
17019                for preview in [true, false] {
17020                    let request = ClientControlRequest::SupervisorRescan { preview };
17021                    let frame = Frame::build(
17022                        FrameType::Request,
17023                        control_flags(),
17024                        0,
17025                        0,
17026                        1,
17027                        serde_json::to_vec(&request).unwrap(),
17028                    )
17029                    .unwrap();
17030                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17031                    assert_eq!(frames.len(), 1);
17032                    let ClientControlResponse::SupervisorRescan { result } =
17033                        serde_json::from_slice(&frames[0].body).unwrap()
17034                    else {
17035                        panic!("expected a rescan response");
17036                    };
17037                    let expected = if needs_restart {
17038                        vec!["privileged_operator_prompt"]
17039                    } else {
17040                        Vec::new()
17041                    };
17042                    assert_eq!(result.restart_required, expected);
17043                    assert_eq!(handler.privileged_operator_prompt, running);
17044                    assert_eq!(
17045                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
17046                        running
17047                    );
17048                }
17049            }
17050        }
17051    }
17052
17053    #[tokio::test]
17054    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
17055        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
17056        let path = dir.join("subc.jsonc");
17057        let registry = Arc::new(Registry::default());
17058        let handle = SupervisorHandle::new();
17059        let supervisor =
17060            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
17061        let handler = ControlHandler::new(registry)
17062            .with_supervisor(handle)
17063            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
17064            .with_supervisor_rescan(supervisor, &path, None);
17065        handler.forwarding.operator_confirms().configure(
17066            Arc::new(ApproveOperator),
17067            crate::operator_confirm::OperatorLimits::default(),
17068        );
17069        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
17070        for preview in [true, false] {
17071            std::fs::write(
17072                &path,
17073                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
17074            )
17075            .unwrap();
17076            let request = ClientControlRequest::SupervisorRescan { preview };
17077            let frame = Frame::build(
17078                FrameType::Request,
17079                control_flags(),
17080                0,
17081                0,
17082                1,
17083                serde_json::to_vec(&request).unwrap(),
17084            )
17085            .unwrap();
17086            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17087            assert_eq!(frames.len(), 1);
17088            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
17089            assert_eq!(
17090                handler.privileged_operator_prompt,
17091                PrivilegedOperatorPrompt::Enforce
17092            );
17093        }
17094    }
17095
17096    /// One wire code has several senders, so the refusal line names the check
17097    /// that refused. This drives the shared refusal path for ordinary refusals
17098    /// with an unregistered
17099    /// target and requires the branch label on the event.
17100    #[tokio::test]
17101    async fn route_open_refusal_names_the_check_that_refused() {
17102        let handler = ControlHandler::new(Arc::new(Registry::default()));
17103        let capture = EventCapture::default();
17104        let _subscriber =
17105            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17106        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17107        let response = handler
17108            .handle_control_frame(
17109                &ctx,
17110                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
17111            )
17112            .await
17113            .unwrap();
17114
17115        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17116        let event = capture
17117            .events()
17118            .into_iter()
17119            .find(|event| {
17120                event.target == "control"
17121                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17122            })
17123            .expect("route.open refusal event");
17124        assert_eq!(
17125            event.fields.get("reason"),
17126            Some(&"\"not_registered\"".to_string())
17127        );
17128    }
17129
17130    #[tokio::test]
17131    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
17132        let registry = Arc::new(Registry::default());
17133        let supervisor_handle = SupervisorHandle::new();
17134        let supervisor =
17135            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
17136                .with_handle(supervisor_handle.clone())
17137                .with_connection_file_path(std::env::temp_dir().join(format!(
17138                    "subc-route-open-refusal-info-{}",
17139                    std::process::id()
17140                )));
17141        let module = supervisor
17142            .supervise_configured(
17143                ModuleSpec {
17144                    script: None,
17145                    carry_containment: false,
17146                    module_id: "warming".to_string(),
17147                    program: fake_aft_stub_path(),
17148                    args: Vec::new(),
17149                    env: Vec::new(),
17150                    reserved: false,
17151                    reserved_prefixes: Vec::new(),
17152                    protocol: ModuleProtocol::Subc,
17153                    overlap: Default::default(),
17154                },
17155                true,
17156            )
17157            .unwrap();
17158        assert_eq!(module.state().unwrap(), ModuleState::Running);
17159
17160        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17161        assert!(handler
17162            .counters()
17163            .snapshot()
17164            .get("route_open_refused_by_code")
17165            .is_none());
17166        let capture = EventCapture::default();
17167        let _subscriber =
17168            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17169        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
17170        let response = handler
17171            .handle_control_frame(
17172                &ctx,
17173                route_open_frame(394, "warming", unique_project_root("refusal-info")),
17174            )
17175            .await
17176            .unwrap();
17177        module.stop().await.unwrap();
17178
17179        assert_eq!(parse_error(&response[0])["code"], "module_warming");
17180        let event = capture
17181            .events()
17182            .into_iter()
17183            .find(|event| {
17184                event.target == "control"
17185                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
17186            })
17187            .expect("route.open refusal event");
17188        assert_eq!(
17189            event.fields.get("module_id"),
17190            Some(&"\"warming\"".to_string())
17191        );
17192        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
17193        assert_eq!(
17194            event.fields.get("reason"),
17195            Some(&"\"supervised_not_registered\"".to_string())
17196        );
17197        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
17198        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
17199        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
17200        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
17201        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
17202        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
17203        assert_eq!(
17204            handler.counters().snapshot()["route_open_refused_by_code"],
17205            json!({ "module_warming": 1 })
17206        );
17207    }
17208
17209    const OUTAGE_START: &str = "route.open refusing module: not serving";
17210    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
17211
17212    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
17213        capture
17214            .events()
17215            .into_iter()
17216            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
17217            .collect()
17218    }
17219
17220    fn supervise_stub(
17221        registry: &Arc<Registry>,
17222        module_id: &str,
17223        enabled: bool,
17224    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
17225        let supervisor_handle = SupervisorHandle::new();
17226        let supervisor =
17227            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
17228                .with_handle(supervisor_handle.clone())
17229                .with_connection_file_path(std::env::temp_dir().join(format!(
17230                    "subc-route-outage-{module_id}-{}",
17231                    std::process::id()
17232                )));
17233        let module = supervisor
17234            .supervise_configured(
17235                ModuleSpec {
17236                    script: None,
17237                    carry_containment: false,
17238                    module_id: module_id.to_string(),
17239                    program: fake_aft_stub_path(),
17240                    args: Vec::new(),
17241                    env: Vec::new(),
17242                    reserved: false,
17243                    reserved_prefixes: Vec::new(),
17244                    protocol: ModuleProtocol::Subc,
17245                    overlap: Default::default(),
17246                },
17247                enabled,
17248            )
17249            .unwrap();
17250        (supervisor_handle, module)
17251    }
17252
17253    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
17254        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
17255            module_id: module_id.to_string(),
17256            drain_timeout_ms: Some(50),
17257        })
17258        .unwrap();
17259        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
17260    }
17261
17262    /// Two handlers built over one forwarding table must share one outage
17263    /// tracker; separate trackers would each log their own opening line for
17264    /// the same outage.
17265    #[test]
17266    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
17267        let registry = Arc::new(Registry::default());
17268        let forwarding = Arc::new(ForwardingTable::default());
17269        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17270        let second = ControlHandler::with_forwarding(registry, forwarding);
17271        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
17272    }
17273
17274    /// A client can name any module id it likes. Refusing an unknown one,
17275    /// however often, must not create outage state or outage lines, or the
17276    /// tracker would be a memory sink any client could fill.
17277    #[tokio::test(flavor = "current_thread")]
17278    async fn route_open_unknown_module_refusals_add_no_outage_state() {
17279        let handler = ControlHandler::new(Arc::new(Registry::default()));
17280        let capture = EventCapture::default();
17281        let _subscriber =
17282            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17283        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
17284        for corr in 0..8 {
17285            let response = handler
17286                .handle_control_frame(
17287                    &ctx,
17288                    route_open_frame(
17289                        380 + corr,
17290                        &format!("nobody-{corr}"),
17291                        unique_project_root("outage-unknown"),
17292                    ),
17293                )
17294                .await
17295                .unwrap();
17296            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17297        }
17298
17299        assert_eq!(handler.route_outages.tracked_module_count(), 0);
17300        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
17301        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
17302    }
17303
17304    /// Drives the refusal path end to end: a supervised module that served
17305    /// before and stopped being registered with no instruction to stop is a
17306    /// WARN, and the same module refused after an operator `supervisor.restart`
17307    /// is an INFO.
17308    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17309    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
17310        let registry = Arc::new(Registry::default());
17311        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
17312        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17313        let capture = EventCapture::default();
17314        let _subscriber =
17315            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17316        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
17317        // The stub never registers, so pretend it served once: otherwise every
17318        // refusal would fall in its startup window.
17319        handler.route_outages.record_accepted("outage-restart");
17320
17321        let response = handler
17322            .handle_control_frame(
17323                &ctx,
17324                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
17325            )
17326            .await
17327            .unwrap();
17328        assert_eq!(response[0].header.ty, FrameType::Error);
17329        let starts = outage_lines(&capture, OUTAGE_START);
17330        assert_eq!(starts.len(), 1, "{starts:?}");
17331        assert_eq!(starts[0].level, tracing::Level::WARN);
17332        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
17333        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
17334        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
17335        handler.route_outages.record_accepted("outage-restart");
17336        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
17337
17338        let restart = handler
17339            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
17340            .await
17341            .unwrap();
17342        assert_eq!(
17343            restart[0].header.ty,
17344            FrameType::Response,
17345            "{:?}",
17346            parse_error(&restart[0])
17347        );
17348        handler
17349            .handle_control_frame(
17350                &ctx,
17351                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
17352            )
17353            .await
17354            .unwrap();
17355        module.stop().await.unwrap();
17356
17357        let starts = outage_lines(&capture, OUTAGE_START);
17358        assert_eq!(starts.len(), 2, "{starts:?}");
17359        assert_eq!(starts[1].level, tracing::Level::INFO);
17360        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
17361    }
17362
17363    /// A restart refused before it touched the module (here: the module is
17364    /// disabled) must clear its operator mark, so the next real outage is
17365    /// still reported as a warning.
17366    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17367    async fn failed_operator_restart_leaves_no_operator_mark() {
17368        let registry = Arc::new(Registry::default());
17369        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
17370        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17371        let capture = EventCapture::default();
17372        let _subscriber =
17373            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17374        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
17375        handler.route_outages.record_accepted("outage-disabled");
17376
17377        let restart = handler
17378            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
17379            .await
17380            .unwrap();
17381        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
17382        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
17383
17384        handler
17385            .handle_control_frame(
17386                &ctx,
17387                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
17388            )
17389            .await
17390            .unwrap();
17391        let starts = outage_lines(&capture, OUTAGE_START);
17392        assert_eq!(starts.len(), 1, "{starts:?}");
17393        assert_eq!(starts[0].level, tracing::Level::WARN);
17394    }
17395
17396    #[tokio::test(flavor = "current_thread")]
17397    async fn route_open_unknown_module_escapes_target_module_id() {
17398        let handler = ControlHandler::new(Arc::new(Registry::default()));
17399        let capture = EventCapture::default();
17400        let _subscriber =
17401            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17402        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
17403        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17404        let response = handler
17405            .handle_control_frame(
17406                &ctx,
17407                route_open_frame(
17408                    395,
17409                    hostile_module_id,
17410                    unique_project_root("hostile-target-module-id"),
17411                ),
17412            )
17413            .await
17414            .unwrap();
17415
17416        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17417        let event = capture
17418            .events()
17419            .into_iter()
17420            .find(|event| {
17421                event.target == "control"
17422                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17423            })
17424            .expect("route.open unknown-module refusal event");
17425        let logged = event.fields.get("module_id").expect("module_id field");
17426        assert!(!logged.bytes().any(|byte| byte < 0x20));
17427        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17428    }
17429
17430    /// A route.open refusal is the daemon's own answer, so its ERROR carries
17431    /// the daemon-origin bit on the frame the client receives.
17432    #[tokio::test(flavor = "current_thread")]
17433    async fn route_open_refusal_carries_daemon_origin() {
17434        let handler = ControlHandler::new(Arc::new(Registry::default()));
17435        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17436        let response = handler
17437            .handle_control_frame(
17438                &ctx,
17439                route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
17440            )
17441            .await
17442            .unwrap();
17443
17444        assert_eq!(response.len(), 1);
17445        assert_eq!(response[0].header.ty, FrameType::Error);
17446        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17447        assert!(
17448            response[0].header.flags.is_daemon_origin(),
17449            "route.open refusal must carry daemon origin: {:?}",
17450            response[0].header.flags
17451        );
17452    }
17453
17454    /// The drain `module_reloading` refusal is daemon-synthesized for an
17455    /// endpoint that is reloading, so it carries the daemon-origin bit on the
17456    /// frame the client receives.
17457    #[tokio::test(flavor = "current_thread")]
17458    async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
17459        let registry = Arc::new(Registry::default());
17460        let forwarding = Arc::new(ForwardingTable::default());
17461        let handler =
17462            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17463        let module_connection = ConnectionId::new(95);
17464        let (module_ctx, mut module_rx) = route_ctx(module_connection);
17465        hello_via_sink(
17466            &handler,
17467            &module_ctx,
17468            &mut module_rx,
17469            hello_frame("aft", PROTOCOL_VERSION, 395),
17470        )
17471        .await;
17472        forwarding
17473            .begin_module_drain("aft", RouteCloseReason::Restart)
17474            .unwrap()
17475            .expect("the module endpoint drains");
17476
17477        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
17478        let response = handler
17479            .handle_control_frame(
17480                &client_ctx,
17481                route_open_frame(396, "aft", unique_project_root("drain-origin")),
17482            )
17483            .await
17484            .unwrap();
17485
17486        assert_eq!(response.len(), 1);
17487        assert_eq!(response[0].header.ty, FrameType::Error);
17488        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17489        assert!(
17490            response[0].header.flags.is_daemon_origin(),
17491            "drain module_reloading must carry daemon origin: {:?}",
17492            response[0].header.flags
17493        );
17494    }
17495
17496    /// A module that refuses a bind with a daemon code still cannot mint a
17497    /// daemon refusal: the relayed `module_reloading` body reaches the client
17498    /// with the daemon-origin bit clear.
17499    #[tokio::test(flavor = "current_thread")]
17500    async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
17501        let registry = Arc::new(Registry::default());
17502        let forwarding = Arc::new(ForwardingTable::default());
17503        let handler =
17504            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17505        let module_connection = ConnectionId::new(95);
17506        let (module_ctx, mut module_rx) = route_ctx(module_connection);
17507        hello_via_sink(
17508            &handler,
17509            &module_ctx,
17510            &mut module_rx,
17511            hello_frame("aft", PROTOCOL_VERSION, 395),
17512        )
17513        .await;
17514
17515        let client_connection = ConnectionId::new(96);
17516        let (client_ctx, _client_rx) = route_ctx(client_connection);
17517        let (route_task, bind) = relay_route_open(
17518            &handler,
17519            client_connection,
17520            &client_ctx.egress,
17521            &mut module_rx,
17522            396,
17523            "aft",
17524            "spoofed-reloading",
17525        )
17526        .await;
17527        // The module sets the origin bit on its own rejection too: neither
17528        // the bit nor the daemon code may survive the relay as daemon-owned.
17529        let rejection = Frame::build(
17530            FrameType::Error,
17531            crate::router::daemon_error_flags(),
17532            0,
17533            0,
17534            bind.header.corr,
17535            serde_json::to_vec(&ErrorBody::new(
17536                "module_reloading",
17537                "module refused route.bind",
17538            ))
17539            .unwrap(),
17540        )
17541        .unwrap();
17542        handler
17543            .handle_control_frame(&module_ctx, rejection)
17544            .await
17545            .unwrap();
17546
17547        let response = route_task.await.unwrap();
17548        assert_eq!(response.len(), 1);
17549        assert_eq!(response[0].header.ty, FrameType::Error);
17550        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17551        assert!(
17552            !response[0].header.flags.is_daemon_origin(),
17553            "a module's own rejection must not carry daemon origin: {:?}",
17554            response[0].header.flags
17555        );
17556    }
17557
17558    #[tokio::test(flavor = "current_thread")]
17559    async fn route_open_module_rejection_uses_daemon_counter_key() {
17560        let registry = Arc::new(Registry::default());
17561        let forwarding = Arc::new(ForwardingTable::default());
17562        let handler =
17563            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17564        let module_connection = ConnectionId::new(95);
17565        let (module_ctx, mut module_rx) = route_ctx(module_connection);
17566        hello_via_sink(
17567            &handler,
17568            &module_ctx,
17569            &mut module_rx,
17570            hello_frame("aft", PROTOCOL_VERSION, 395),
17571        )
17572        .await;
17573
17574        let client_connection = ConnectionId::new(96);
17575        let (client_ctx, _client_rx) = route_ctx(client_connection);
17576        let capture = EventCapture::default();
17577        // tracing-core decides once per callsite whether its events are enabled,
17578        // and caches the answer. If a parallel test with no subscriber reaches
17579        // this callsite first, the cached answer is "disabled" and this test's
17580        // capture sees nothing. Keeping a second dispatch alive makes tracing-core
17581        // ask every live subscriber instead, while only this test's default
17582        // receives the events.
17583        static INTEREST_ANCHOR: std::sync::OnceLock<tracing::Dispatch> = std::sync::OnceLock::new();
17584        INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
17585        let _subscriber =
17586            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17587        let (route_task, bind) = relay_route_open(
17588            &handler,
17589            client_connection,
17590            &client_ctx.egress,
17591            &mut module_rx,
17592            396,
17593            "aft",
17594            "hostile-module-code",
17595        )
17596        .await;
17597        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
17598        let rejection = Frame::build(
17599            FrameType::Error,
17600            control_flags(),
17601            0,
17602            0,
17603            bind.header.corr,
17604            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
17605        )
17606        .unwrap();
17607        handler
17608            .handle_control_frame(&module_ctx, rejection)
17609            .await
17610            .unwrap();
17611
17612        let response = route_task.await.unwrap();
17613        assert_eq!(parse_error(&response[0])["code"], hostile_code);
17614        let counters = handler.counters().snapshot();
17615        assert_eq!(
17616            counters["route_open_refused_by_code"],
17617            json!({ "module_rejected": 1 })
17618        );
17619        assert!(counters["route_open_refused_by_code"]
17620            .get(hostile_code)
17621            .is_none());
17622
17623        let event = capture
17624            .events()
17625            .into_iter()
17626            .find(|event| {
17627                event.target == "control"
17628                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
17629            })
17630            .expect("route.open module-rejection refusal event");
17631        let logged = event.fields.get("module_code").expect("module_code field");
17632        assert!(!logged.bytes().any(|byte| byte < 0x20));
17633        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17634    }
17635
17636    #[tokio::test]
17637    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
17638        let registry = Arc::new(Registry::default());
17639        let supervisor_handle = SupervisorHandle::new();
17640        let missing_program = std::env::temp_dir().join(format!(
17641            "subc-route-open-missing-program-{}",
17642            std::process::id()
17643        ));
17644        let supervisor =
17645            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
17646                .with_handle(supervisor_handle.clone());
17647        let module = supervisor
17648            .supervise_configured(
17649                ModuleSpec {
17650                    script: None,
17651                    carry_containment: false,
17652                    module_id: "failed".to_string(),
17653                    program: missing_program,
17654                    args: Vec::new(),
17655                    env: Vec::new(),
17656                    reserved: false,
17657                    reserved_prefixes: Vec::new(),
17658                    protocol: ModuleProtocol::Subc,
17659                    overlap: Default::default(),
17660                },
17661                true,
17662            )
17663            .unwrap();
17664        assert_eq!(module.state().unwrap(), ModuleState::Failed);
17665
17666        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17667        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
17668        let response = handler
17669            .handle_control_frame(
17670                &ctx,
17671                route_open_frame(305, "failed", unique_project_root("failed")),
17672            )
17673            .await
17674            .unwrap();
17675
17676        assert_eq!(response[0].header.ty, FrameType::Error);
17677        let error = parse_error(&response[0]);
17678        assert_eq!(error["code"], "target_unavailable");
17679        assert!(error["message"]
17680            .as_str()
17681            .unwrap()
17682            .contains("state=failed, enabled=true, live=false"));
17683    }
17684
17685    #[tokio::test]
17686    async fn route_open_role_mismatch_remains_target_unavailable() {
17687        let registry = Arc::new(Registry::default());
17688        let handler = ControlHandler::new(Arc::clone(&registry));
17689        handler
17690            .handle_control(
17691                ConnectionId::new(41),
17692                non_routable_hello_frame_with_control_ops("health-only", 306, None),
17693            )
17694            .unwrap();
17695
17696        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
17697        let response = handler
17698            .handle_control_frame(
17699                &ctx,
17700                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
17701            )
17702            .await
17703            .unwrap();
17704
17705        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17706        assert!(parse_error(&response[0])["message"]
17707            .as_str()
17708            .unwrap()
17709            .contains("does not provide the requested target"));
17710    }
17711
17712    #[tokio::test]
17713    async fn route_open_inactive_registration_remains_target_unavailable() {
17714        let registry = Arc::new(Registry::default());
17715        let handler = ControlHandler::new(Arc::clone(&registry));
17716        handler
17717            .handle_control(
17718                ConnectionId::new(43),
17719                hello_frame("inactive", PROTOCOL_VERSION, 308),
17720            )
17721            .unwrap();
17722        assert!(registry
17723            .set_module_state_for_test("inactive", ChannelState::Closed)
17724            .unwrap());
17725
17726        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
17727        let response = handler
17728            .handle_control_frame(
17729                &ctx,
17730                route_open_frame(309, "inactive", unique_project_root("inactive")),
17731            )
17732            .await
17733            .unwrap();
17734
17735        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17736        assert!(parse_error(&response[0])["message"]
17737            .as_str()
17738            .unwrap()
17739            .contains("is not active"));
17740    }
17741
17742    #[tokio::test]
17743    async fn late_health_reply_is_recorded_through_the_module_response_path() {
17744        let registry = Arc::new(Registry::default());
17745        let forwarding = Arc::new(ForwardingTable::default());
17746        let supervisor_handle = SupervisorHandle::new();
17747        let supervisor =
17748            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
17749                .with_forwarding(Arc::clone(&forwarding))
17750                .with_handle(supervisor_handle.clone());
17751        let module = supervisor
17752            .supervise_configured(
17753                crate::ModuleSpec {
17754                    script: None,
17755                    carry_containment: false,
17756                    module_id: "late-health-response".to_string(),
17757                    program: PathBuf::from("disabled-module"),
17758                    args: Vec::new(),
17759                    env: Vec::new(),
17760                    reserved: false,
17761                    reserved_prefixes: Vec::new(),
17762                    protocol: ModuleProtocol::Subc,
17763                    overlap: Default::default(),
17764                },
17765                false,
17766            )
17767            .unwrap();
17768        let handler =
17769            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17770                .with_supervisor(supervisor_handle);
17771        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
17772        handler
17773            .handle_control_frame(
17774                &module_ctx,
17775                hello_frame_with_control_ops(
17776                    "late-health-response",
17777                    PROTOCOL_VERSION,
17778                    7,
17779                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17780                ),
17781            )
17782            .await
17783            .unwrap();
17784        let probe_started_at = Instant::now() - Duration::from_millis(80);
17785        let pending = forwarding
17786            .begin_health_probe_rpc_for(
17787                "late-health-response",
17788                MODULE_CONTROL_OP_HEALTH_CHECK,
17789                probe_started_at,
17790                Instant::now() - Duration::from_millis(1),
17791            )
17792            .unwrap();
17793        assert!(forwarding
17794            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
17795            .unwrap());
17796
17797        let responses = handler
17798            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
17799            .await
17800            .unwrap();
17801
17802        assert!(responses.is_empty());
17803        let health = module.status().unwrap().health;
17804        assert_eq!(health.late_answer_count, 1);
17805        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
17806    }
17807
17808    #[tokio::test]
17809    async fn health_probe_timeout_and_module_death_are_typed() {
17810        let registry = Arc::new(Registry::default());
17811        let forwarding = Arc::new(ForwardingTable::default());
17812        let handler =
17813            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17814                .with_health_probe_timeout(Duration::from_millis(50));
17815        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
17816        hello_via_sink(
17817            &handler,
17818            &module_ctx,
17819            &mut module_rx,
17820            hello_frame_with_control_ops(
17821                "aft",
17822                PROTOCOL_VERSION,
17823                7,
17824                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17825            ),
17826        )
17827        .await;
17828
17829        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
17830        let responses = handler
17831            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
17832            .await
17833            .unwrap();
17834        assert_eq!(responses[0].header.ty, FrameType::Error);
17835        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
17836        let _ = module_rx.try_recv();
17837
17838        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
17839        let health_handler = handler.clone();
17840        let death_task = tokio::spawn(async move {
17841            health_handler
17842                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
17843                .await
17844                .unwrap()
17845        });
17846        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
17847            .await
17848            .unwrap()
17849            .unwrap();
17850        handler
17851            .cleanup_connection(module_ctx.connection_id)
17852            .unwrap();
17853        let responses = death_task.await.unwrap();
17854        assert_eq!(responses[0].header.ty, FrameType::Error);
17855        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
17856    }
17857
17858    #[test]
17859    fn hello_requires_exact_protocol_version() {
17860        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
17861            let registry = Arc::new(Registry::default());
17862            let handler = ControlHandler::new(Arc::clone(&registry));
17863            let responses = handler
17864                .handle_control(
17865                    ConnectionId::new(connection),
17866                    hello_frame("aft", offered, 9),
17867                )
17868                .unwrap();
17869
17870            assert_eq!(responses.len(), 1);
17871            assert_eq!(responses[0].header.ty, FrameType::Error);
17872            let error = parse_error(&responses[0]);
17873            assert_eq!(error["code"], "version_unsupported");
17874            assert!(registry.get_module("aft").unwrap().is_none());
17875            assert_eq!(registry.active_registration_count().unwrap(), 0);
17876        }
17877    }
17878
17879    #[test]
17880    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
17881        let registry = Arc::new(Registry::default());
17882        let forwarding = Arc::new(ForwardingTable::default());
17883        let handler =
17884            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17885        let module_connection = ConnectionId::new(301);
17886        let registration = registry
17887            .register_with_control_ops(
17888                manifest("aft-push", PROTOCOL_VERSION),
17889                PROTOCOL_VERSION,
17890                module_connection,
17891                module_baseline_control_ops(),
17892            )
17893            .unwrap();
17894        let (module_tx, _module_rx) = mpsc::channel(8);
17895        let endpoint = forwarding
17896            .register_module_connection(
17897                module_connection,
17898                "aft-push".to_string(),
17899                PROTOCOL_VERSION,
17900                manifest_concurrency(&registration.manifest),
17901                FrameSink::new(module_tx),
17902            )
17903            .unwrap();
17904
17905        // A push op this version does not know is ignored (forward-compat), not errored.
17906        let unknown = Frame::build(
17907            FrameType::Push,
17908            control_flags(),
17909            0,
17910            0,
17911            5,
17912            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
17913        )
17914        .unwrap();
17915        let out = handler.handle_status_update(endpoint, unknown).unwrap();
17916        assert!(
17917            out.is_empty(),
17918            "unknown push op must be ignored, got {out:?}"
17919        );
17920
17921        // A malformed body for a KNOWN op is a real error worth surfacing.
17922        let malformed = Frame::build(
17923            FrameType::Push,
17924            control_flags(),
17925            0,
17926            0,
17927            6,
17928            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
17929        )
17930        .unwrap();
17931        let out = handler.handle_status_update(endpoint, malformed).unwrap();
17932        assert_eq!(out.len(), 1);
17933        assert_eq!(out[0].header.ty, FrameType::Error);
17934        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
17935    }
17936
17937    #[test]
17938    fn hello_rejected_when_connection_already_owns_client_routes() {
17939        let registry = Arc::new(Registry::default());
17940        let forwarding = Arc::new(ForwardingTable::default());
17941        let handler =
17942            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17943        // Commits a client route on connection 202 (bound to a module on conn 101).
17944        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
17945        let client_connection = ConnectionId::new(202);
17946
17947        // That same connection now tries to register as a module: rejected, so one
17948        // connection never holds both client-route and module-endpoint state.
17949        let responses = handler
17950            .handle_control(
17951                client_connection,
17952                hello_frame("aft-second", PROTOCOL_VERSION, 9),
17953            )
17954            .unwrap();
17955        assert_eq!(responses[0].header.ty, FrameType::Error);
17956        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
17957        assert!(registry.get_module("aft-second").unwrap().is_none());
17958    }
17959
17960    #[tokio::test]
17961    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
17962        let registry = Arc::new(Registry::default());
17963        let forwarding = Arc::new(ForwardingTable::default());
17964        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
17965        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
17966        hello_via_sink(
17967            &handler,
17968            &module_ctx,
17969            &mut module_rx,
17970            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
17971        )
17972        .await;
17973        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
17974        let pending = forwarding
17975            .begin_route_bind_relay_for_test(
17976                client_ctx.connection_id,
17977                client_ctx.egress.clone(),
17978                2,
17979                "alpha",
17980            )
17981            .unwrap();
17982        forwarding
17983            .complete_pending_relay(
17984                module_ctx.connection_id,
17985                pending.corr,
17986                RouteBindRelayOutcome::Accepted,
17987            )
17988            .unwrap();
17989        client_rx.try_recv().unwrap();
17990        for module_id in ["beta", "alpha"] {
17991            let replies = handler
17992                .handle_control_frame(
17993                    &module_ctx,
17994                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
17995                )
17996                .await
17997                .unwrap();
17998            assert_eq!(replies.len(), 1, "second HELLO must be refused");
17999            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
18000        }
18001        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
18002        assert!(registry.get_module("beta").unwrap().is_none());
18003        assert!(matches!(
18004            forwarding
18005                .lookup_data_route(
18006                    client_ctx.connection_id,
18007                    pending.client_channel,
18008                    pending.client_epoch,
18009                )
18010                .unwrap(),
18011            DataRoute::Client(DataRouteState::Bound(_))
18012        ));
18013        assert!(handler
18014            .hello_launch_nonces
18015            .lock()
18016            .unwrap()
18017            .presented(module_ctx.connection_id, Some("alpha-nonce")));
18018        assert!(module_rx.try_recv().is_err());
18019    }
18020
18021    #[test]
18022    fn reserved_module_hello_requires_matching_launch_nonce() {
18023        let registry = Arc::new(Registry::default());
18024        let supervisor = SupervisorHandle::new();
18025        // The supervisor recorded the nonce it injected when it spawned the reserved
18026        // module; the HELLO verifier checks against the same shared handle.
18027        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
18028        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
18029
18030        // A HELLO with NO nonce is rejected.
18031        let no_nonce = handler
18032            .handle_control(
18033                ConnectionId::new(1),
18034                hello_frame("vault", PROTOCOL_VERSION, 1),
18035            )
18036            .unwrap();
18037        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
18038        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
18039        assert!(registry.get_module("vault").unwrap().is_none());
18040
18041        // A HELLO with the WRONG nonce is rejected.
18042        let wrong = handler
18043            .handle_control(
18044                ConnectionId::new(2),
18045                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
18046            )
18047            .unwrap();
18048        assert_eq!(wrong[0].header.ty, FrameType::Error);
18049        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
18050        assert!(registry.get_module("vault").unwrap().is_none());
18051
18052        // A HELLO with the CORRECT nonce registers.
18053        let ok = handler
18054            .handle_control(
18055                ConnectionId::new(3),
18056                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
18057            )
18058            .unwrap();
18059        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
18060        assert!(registry.get_module("vault").unwrap().is_some());
18061    }
18062
18063    #[test]
18064    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
18065        let registry = Arc::new(Registry::default());
18066        let supervisor = SupervisorHandle::new();
18067        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18068        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18069        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
18070
18071        let squat = handler
18072            .handle_control(
18073                ConnectionId::new(1),
18074                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
18075            )
18076            .unwrap();
18077        assert_eq!(squat[0].header.ty, FrameType::Error);
18078        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
18079        assert!(parse_error(&squat[0])["message"]
18080            .as_str()
18081            .unwrap()
18082            .contains("fed:"));
18083
18084        let accepted_peer = handler
18085            .handle_control(
18086                ConnectionId::new(2),
18087                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
18088            )
18089            .unwrap();
18090        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
18091
18092        let accepted_short = handler
18093            .handle_control(
18094                ConnectionId::new(3),
18095                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
18096            )
18097            .unwrap();
18098        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
18099
18100        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
18101            let response = handler
18102                .handle_control(
18103                    ConnectionId::new(conn),
18104                    hello_frame(module_id, PROTOCOL_VERSION, conn),
18105                )
18106                .unwrap();
18107            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
18108        }
18109    }
18110
18111    #[test]
18112    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
18113        let registry = Arc::new(Registry::default());
18114        let supervisor = SupervisorHandle::new();
18115        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18116        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18117        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
18118        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
18119
18120        let owner_nonce = handler
18121            .handle_control(
18122                ConnectionId::new(1),
18123                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
18124            )
18125            .unwrap();
18126        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
18127        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
18128        assert!(registry.get_module("fed:special").unwrap().is_none());
18129
18130        let exact_nonce = handler
18131            .handle_control(
18132                ConnectionId::new(2),
18133                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
18134            )
18135            .unwrap();
18136        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
18137        assert!(registry.get_module("fed:special").unwrap().is_some());
18138    }
18139
18140    #[test]
18141    fn non_reserved_module_ignores_launch_nonce() {
18142        let registry = Arc::new(Registry::default());
18143        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
18144        // registration succeeds whether a spawned process echoes a nonce or not.
18145        let handler = ControlHandler::new(Arc::clone(&registry));
18146        let no_nonce = handler
18147            .handle_control(
18148                ConnectionId::new(1),
18149                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
18150            )
18151            .unwrap();
18152        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
18153        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
18154
18155        let echoed_nonce = handler
18156            .handle_control(
18157                ConnectionId::new(2),
18158                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
18159            )
18160            .unwrap();
18161        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
18162        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
18163    }
18164
18165    #[test]
18166    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
18167        let handler = ControlHandler::default();
18168        let conn = ConnectionId::new(1);
18169        let malformed = Frame::build(
18170            FrameType::Hello,
18171            control_flags(),
18172            0,
18173            0,
18174            3,
18175            b"{not json".to_vec(),
18176        )
18177        .unwrap();
18178
18179        let error = handler.handle_control(conn, malformed).unwrap();
18180        assert_eq!(error[0].header.ty, FrameType::Error);
18181        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
18182
18183        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18184        let pong = handler.handle_control(conn, ping).unwrap();
18185        assert_eq!(pong[0].header.ty, FrameType::Pong);
18186        assert_eq!(pong[0].header.corr, 4);
18187    }
18188
18189    #[test]
18190    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
18191        let registry = Arc::new(Registry::default());
18192        let handler = ControlHandler::new(Arc::clone(&registry));
18193
18194        handler
18195            .handle_control(
18196                ConnectionId::new(1),
18197                hello_frame("aft", PROTOCOL_VERSION, 1),
18198            )
18199            .unwrap();
18200        let duplicate = handler
18201            .handle_control(
18202                ConnectionId::new(2),
18203                hello_frame("aft", PROTOCOL_VERSION, 2),
18204            )
18205            .unwrap();
18206
18207        assert_eq!(duplicate[0].header.ty, FrameType::Error);
18208        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
18209        let registration = registry.get_module("aft").unwrap().unwrap();
18210        assert_eq!(registration.connection_id, ConnectionId::new(1));
18211    }
18212
18213    #[test]
18214    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
18215        let registry = Arc::new(Registry::default());
18216        let forwarding = Arc::new(ForwardingTable::default());
18217        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
18218        let handler =
18219            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
18220                .with_process_liveness(process_liveness);
18221        let (ctx, route_channel, route_epoch) =
18222            bind_liveness_route(&registry, &forwarding, "aft-dead");
18223        let responses = handler
18224            .handle_route_poll(
18225                &ctx,
18226                route_poll_frame(41, PollKind::Liveness, route_channel),
18227                route_channel,
18228                route_epoch,
18229                PollKind::Liveness,
18230            )
18231            .unwrap();
18232
18233        assert_eq!(responses.len(), 1);
18234        assert_eq!(responses[0].header.ty, FrameType::Response);
18235        assert_route_poll_liveness(&responses[0], false);
18236    }
18237
18238    #[test]
18239    fn liveness_poll_without_process_source_uses_bound_route() {
18240        let registry = Arc::new(Registry::default());
18241        let forwarding = Arc::new(ForwardingTable::default());
18242        let handler =
18243            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
18244        let (ctx, route_channel, route_epoch) =
18245            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
18246        let responses = handler
18247            .handle_route_poll(
18248                &ctx,
18249                route_poll_frame(42, PollKind::Liveness, route_channel),
18250                route_channel,
18251                route_epoch,
18252                PollKind::Liveness,
18253            )
18254            .unwrap();
18255
18256        assert_route_poll_liveness(&responses[0], true);
18257    }
18258
18259    #[test]
18260    fn liveness_poll_untracked_process_source_uses_bound_route() {
18261        let registry = Arc::new(Registry::default());
18262        let forwarding = Arc::new(ForwardingTable::default());
18263        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
18264        let handler =
18265            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
18266                .with_process_liveness(process_liveness);
18267        let (ctx, route_channel, route_epoch) =
18268            bind_liveness_route(&registry, &forwarding, "aft-untracked");
18269        let responses = handler
18270            .handle_route_poll(
18271                &ctx,
18272                route_poll_frame(43, PollKind::Liveness, route_channel),
18273                route_channel,
18274                route_epoch,
18275                PollKind::Liveness,
18276            )
18277            .unwrap();
18278
18279        assert_route_poll_liveness(&responses[0], true);
18280    }
18281
18282    #[tokio::test]
18283    async fn unknown_op_returns_unknown_control_op() {
18284        let handler = ControlHandler::default();
18285        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
18286        let request = Frame::build(
18287            FrameType::Request,
18288            control_flags(),
18289            0,
18290            0,
18291            55,
18292            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
18293        )
18294        .unwrap();
18295
18296        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18297
18298        assert_eq!(response.len(), 1);
18299        assert_eq!(response[0].header.ty, FrameType::Error);
18300        assert_eq!(response[0].header.corr, 55);
18301        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
18302    }
18303
18304    #[tokio::test]
18305    async fn supervisor_provenance_rejects_unknown_exact_module() {
18306        let handler = ControlHandler::default();
18307        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
18308        let request = Frame::build(
18309            FrameType::Request,
18310            control_flags(),
18311            0,
18312            0,
18313            57,
18314            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
18315        )
18316        .unwrap();
18317
18318        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18319
18320        assert_eq!(response.len(), 1);
18321        assert_eq!(response[0].header.ty, FrameType::Error);
18322        assert_eq!(response[0].header.corr, 57);
18323        let error = parse_error(&response[0]);
18324        assert_eq!(error["code"], "unknown_module");
18325        assert_eq!(error["message"], "module_id 'missing' is not supervised");
18326    }
18327
18328    #[test]
18329    fn provenance_probe_override_keeps_handler_tests_deterministic() {
18330        let expected = subc_control::RunningImageAgreement::Unavailable {
18331            reason: subc_control::RunningImageUnavailableReason::HashFailed,
18332        };
18333        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
18334        assert_eq!(handler.provenance_probe_override, Some(expected));
18335    }
18336
18337    #[test]
18338    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
18339        let verdict = reload_verdict(
18340            std::path::Path::new("/bin/new"),
18341            Some(std::path::Path::new("/bin/old")),
18342            subc_control::RunningImageAgreement::Unavailable {
18343                reason: subc_control::RunningImageUnavailableReason::HashFailed,
18344            },
18345        );
18346        assert!(matches!(
18347            verdict.path,
18348            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
18349                if configured == std::path::Path::new("/bin/new")
18350                    && spawned_from == std::path::Path::new("/bin/old")
18351        ));
18352    }
18353
18354    #[test]
18355    fn launch_spec_without_running_digest_is_unavailable_not_match() {
18356        use subc_control::{
18357            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
18358        };
18359        let configured = crate::supervise::ModuleSpec {
18360            script: None,
18361            carry_containment: false,
18362            module_id: "legacy".into(),
18363            program: "/module".into(),
18364            args: Vec::new(),
18365            env: Vec::new(),
18366            reserved: false,
18367            reserved_prefixes: Vec::new(),
18368            protocol: subc_control::ModuleProtocol::Subc,
18369            overlap: Default::default(),
18370        };
18371        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
18372        assert_eq!(
18373            launch_spec_agreement(&configured, false, Some(&digest)),
18374            Agreement::Unavailable {
18375                reason: Reason::NotRunning
18376            }
18377        );
18378        assert_eq!(
18379            launch_spec_agreement(&configured, true, None),
18380            Agreement::Unavailable {
18381                reason: Reason::SpawnedSpecUnavailable
18382            }
18383        );
18384        assert_eq!(
18385            launch_spec_agreement(&configured, true, Some(&digest)),
18386            Agreement::Match
18387        );
18388    }
18389
18390    #[cfg(windows)]
18391    #[test]
18392    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
18393        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
18394        let configured = dir.join("module.exe");
18395        std::fs::write(&configured, b"executable path fixture").unwrap();
18396        let spawned = std::fs::canonicalize(&configured).unwrap();
18397        assert_ne!(
18398            configured, spawned,
18399            "fixture must exercise different path spellings"
18400        );
18401        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
18402        let lower_case = configured.to_string_lossy().to_lowercase();
18403        for spelling in [
18404            &configured,
18405            Path::new(&forward_slashes),
18406            Path::new(&lower_case),
18407        ] {
18408            let verdict = reload_verdict(
18409                spelling,
18410                Some(&spawned),
18411                subc_control::RunningImageAgreement::Unavailable {
18412                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18413                },
18414            );
18415            assert_eq!(
18416                verdict.path,
18417                ReloadPathAgreement::Match,
18418                "configured {spelling:?}, spawned {spawned:?}"
18419            );
18420        }
18421    }
18422
18423    #[cfg(windows)]
18424    #[test]
18425    fn windows_reload_path_keeps_different_existing_program_pending() {
18426        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
18427        let old = dir.join("old.exe");
18428        let new = dir.join("new.exe");
18429        std::fs::write(&old, b"same executable content").unwrap();
18430        std::fs::copy(&old, &new).unwrap();
18431        let spawned = std::fs::canonicalize(&old).unwrap();
18432        let image = subc_control::RunningImageAgreement::Unavailable {
18433            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18434        };
18435        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
18436        assert!(
18437            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
18438            if configured == new && spawned_from == spawned)
18439        );
18440        let missing = dir.join("missing.exe");
18441        assert!(matches!(
18442            reload_verdict(&missing, Some(&spawned), image).path,
18443            ReloadPathAgreement::Mismatch { .. }
18444        ));
18445    }
18446
18447    #[test]
18448    fn reload_verdict_detects_replaced_image_at_same_path() {
18449        let image = subc_control::RunningImageAgreement::Mismatch {
18450            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
18451                digest: "old".into(),
18452            },
18453            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
18454                digest: "new".into(),
18455            },
18456        };
18457        let verdict = reload_verdict(
18458            std::path::Path::new("/bin/same"),
18459            Some(std::path::Path::new("/bin/same")),
18460            image.clone(),
18461        );
18462        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18463        assert_eq!(verdict.image, image);
18464    }
18465
18466    #[test]
18467    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
18468        let image = subc_control::RunningImageAgreement::Unavailable {
18469            reason: subc_control::RunningImageUnavailableReason::NotRunning,
18470        };
18471        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
18472        assert_eq!(
18473            verdict.path,
18474            subc_control::ReloadPathAgreement::Unavailable {
18475                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
18476            }
18477        );
18478        assert_eq!(verdict.image, image);
18479
18480        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
18481            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18482        };
18483        let verdict = reload_verdict(
18484            std::path::Path::new("/bin/same"),
18485            Some(std::path::Path::new("/bin/same")),
18486            unconfirmed.clone(),
18487        );
18488        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18489        assert_eq!(verdict.image, unconfirmed);
18490    }
18491
18492    #[test]
18493    fn reload_verdict_preserves_each_image_unavailability_reason() {
18494        use subc_control::RunningImageUnavailableReason as Reason;
18495
18496        for reason in [
18497            Reason::NotRunning,
18498            Reason::UnsupportedPlatform,
18499            Reason::RunningExecutableUnreadable,
18500            Reason::SpawnedPathUnreadable,
18501            Reason::HashFailed,
18502            Reason::ProcessIdentityUnconfirmed,
18503            Reason::Unknown("future_probe_reason".to_string()),
18504        ] {
18505            let image = subc_control::RunningImageAgreement::Unavailable {
18506                reason: reason.clone(),
18507            };
18508            let verdict = reload_verdict(
18509                std::path::Path::new("/bin/same"),
18510                Some(std::path::Path::new("/bin/same")),
18511                image.clone(),
18512            );
18513            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18514            assert_eq!(verdict.image, image, "{reason:?}");
18515        }
18516    }
18517
18518    #[tokio::test]
18519    async fn malformed_control_bodies_return_invalid_control_body() {
18520        let handler = ControlHandler::default();
18521        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
18522
18523        for (corr, body) in [
18524            (56, br#"{"route_channel":1}"#.as_slice()),
18525            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
18526            (
18527                58,
18528                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
18529            ),
18530        ] {
18531            let request = Frame::build(
18532                FrameType::Request,
18533                control_flags(),
18534                0,
18535                0,
18536                corr,
18537                body.to_vec(),
18538            )
18539            .unwrap();
18540            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18541
18542            assert_eq!(response.len(), 1);
18543            assert_eq!(response[0].header.ty, FrameType::Error);
18544            assert_eq!(response[0].header.corr, corr);
18545            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
18546        }
18547    }
18548
18549    #[tokio::test]
18550    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
18551        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18552        let registry = Arc::new(Registry::default());
18553        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
18554        let router = Router::with_control_handler(Arc::clone(&control));
18555        let connection = router.begin_connection();
18556        let (ctx, mut rx) = route_ctx(connection.id());
18557
18558        router
18559            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
18560            .await
18561            .unwrap();
18562        let response = rx.recv().await.unwrap();
18563        let ack = parse_ack(&response);
18564        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18565        let channel = 1;
18566
18567        let goodbye =
18568            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
18569        router.route_for_connection(&ctx, goodbye).await.unwrap();
18570        assert!(rx.try_recv().is_err());
18571        assert!(registry.get_module("aft").unwrap().is_none());
18572
18573        router
18574            .route_for_connection(&ctx, channel_request(channel, 13))
18575            .await
18576            .unwrap();
18577        let error_frame = rx.recv().await.unwrap();
18578        assert_eq!(error_frame.header.ty, FrameType::Error);
18579        assert_eq!(error_frame.header.channel, channel);
18580        let captured = crate::router::test_log::captured_logs(&logs);
18581        assert_eq!(
18582            captured
18583                .lines()
18584                .filter(|line| {
18585                    line.contains("module_id=aft")
18586                        && line.contains("reason=explicit_goodbye")
18587                        && line.contains("module registration ended")
18588                })
18589                .count(),
18590            1,
18591            "unexpected GOODBYE registry log: {captured}"
18592        );
18593    }
18594
18595    #[tokio::test]
18596    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
18597        let registry = Arc::new(Registry::default());
18598        let handler = ControlHandler::new(registry).with_capability_config(
18599            [("prov".to_string(), true), ("cons".to_string(), true)],
18600            BTreeMap::new(),
18601        );
18602        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
18603        register_capability_manifest(
18604            &handler,
18605            &provider_ctx,
18606            &mut provider_rx,
18607            capability_manifest("prov", &["thing/v1"], &[]),
18608            1,
18609        )
18610        .await;
18611        let mut consumer = capability_manifest("cons", &[], &[]);
18612        consumer.capabilities.as_mut().unwrap().requires.push(
18613            subc_protocol::manifest::CapabilityRequirement {
18614                capability: "thing/v1".to_string(),
18615                need: subc_protocol::manifest::CapabilityNeed::Required,
18616            },
18617        );
18618        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
18619        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
18620        assert_eq!(
18621            handler.capability_evaluator.verdict("cons", "thing/v1"),
18622            Some(CapabilityVerdict::Provided)
18623        );
18624        let (mut client_rx, _) = open_route_for_capability_test(
18625            &handler,
18626            &provider_ctx,
18627            &mut provider_rx,
18628            703,
18629            3,
18630            "prov",
18631            None,
18632        )
18633        .await;
18634        let goodbye =
18635            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18636        handler
18637            .handle_control_frame(&provider_ctx, goodbye)
18638            .await
18639            .unwrap();
18640        assert_eq!(
18641            handler.capability_evaluator.verdict("cons", "thing/v1"),
18642            Some(CapabilityVerdict::NeverProvided)
18643        );
18644        let closed = client_rx
18645            .try_recv()
18646            .expect("GOODBYE pushes route.closed before route GOODBYE");
18647        assert!(
18648            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
18649            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
18650        );
18651        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
18652        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
18653    }
18654
18655    struct RouteClosedFixture {
18656        handler: ControlHandler,
18657        #[cfg(unix)]
18658        supervisor: Supervisor,
18659        module_ctx: RouteCtx,
18660        #[cfg(unix)]
18661        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18662        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18663        route: Arc<crate::forwarding::RouteBinding>,
18664    }
18665
18666    async fn route_closed_fixture() -> RouteClosedFixture {
18667        let registry = Arc::new(Registry::default());
18668        let handler = ControlHandler::new(Arc::clone(&registry));
18669        #[cfg(unix)]
18670        let supervisor = Supervisor::new(registry, RestartPolicy::default())
18671            .with_forwarding(Arc::clone(&handler.forwarding));
18672        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
18673        register_capability_manifest(
18674            &handler,
18675            &module_ctx,
18676            &mut module_rx,
18677            capability_manifest("prov", &[], &[]),
18678            1,
18679        )
18680        .await;
18681        let (client_rx, bind) = open_route_for_capability_test(
18682            &handler,
18683            &module_ctx,
18684            &mut module_rx,
18685            702,
18686            2,
18687            "prov",
18688            None,
18689        )
18690        .await;
18691        let ModuleControlRequest::RouteBind {
18692            route_channel: channel,
18693            epoch: route_epoch,
18694            ..
18695        } = bind
18696        else {
18697            panic!("expected route.bind");
18698        };
18699        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
18700            handler
18701                .forwarding
18702                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
18703                .unwrap()
18704        else {
18705            panic!("the acknowledged route must be bound");
18706        };
18707        RouteClosedFixture {
18708            handler,
18709            #[cfg(unix)]
18710            supervisor,
18711            module_ctx,
18712            #[cfg(unix)]
18713            module_rx,
18714            client_rx,
18715            route,
18716        }
18717    }
18718
18719    impl RouteClosedFixture {
18720        #[cfg(unix)]
18721        async fn cut(&mut self) {
18722            let notices = async {
18723                let mut draining = self.module_rx.recv().await.unwrap();
18724                assert!(matches!(
18725                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
18726                        &draining.body
18727                    )
18728                    .unwrap(),
18729                    subc_protocol::session::ModuleControlCommand::Draining {
18730                        reason: RouteCloseReason::Disable,
18731                        ..
18732                    }
18733                ));
18734                draining.flushed.take().unwrap().send(()).unwrap();
18735                let mut closing = self.client_rx.recv().await.unwrap();
18736                assert!(matches!(
18737                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
18738                    ClientControlPush::RouteClosing {
18739                        reason: RouteCloseReason::Restart,
18740                        ..
18741                    }
18742                ));
18743                closing.flushed.take().unwrap().send(()).unwrap();
18744            };
18745            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
18746            result.unwrap();
18747        }
18748
18749        fn disconnect(&mut self) -> ClientControlPush {
18750            self.handler
18751                .cleanup_connection(self.module_ctx.connection_id)
18752                .unwrap();
18753            let closed = self.client_rx.try_recv().unwrap();
18754            assert_eq!(closed.header.ty, FrameType::Push);
18755            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
18756            assert!(matches!(
18757                &push,
18758                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
18759                    if channels == &[self.route.client_channel]
18760            ));
18761            assert_eq!(
18762                self.client_rx.try_recv().unwrap().header.ty,
18763                FrameType::Goodbye
18764            );
18765            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
18766            push
18767        }
18768    }
18769
18770    #[cfg(unix)]
18771    #[tokio::test(start_paused = true)]
18772    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
18773        let mut fixture = route_closed_fixture().await;
18774        fixture.cut().await;
18775        assert!(matches!(
18776            fixture.disconnect(),
18777            ClientControlPush::RouteClosed {
18778                reason: RouteCloseReason::Restart,
18779                drained: true,
18780                ..
18781            }
18782        ));
18783    }
18784
18785    #[cfg(unix)]
18786    #[tokio::test(start_paused = true)]
18787    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
18788        let mut fixture = route_closed_fixture().await;
18789        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
18790        let started = tokio::time::Instant::now();
18791        fixture.cut().await;
18792        assert!(started.elapsed() >= Duration::from_secs(2));
18793        assert_eq!(fixture.route.flow.in_flight(), 1);
18794        assert!(matches!(
18795            fixture.disconnect(),
18796            ClientControlPush::RouteClosed {
18797                reason: RouteCloseReason::Restart,
18798                drained: false,
18799                ..
18800            }
18801        ));
18802    }
18803
18804    #[tokio::test]
18805    async fn module_connection_crash_reports_crash_and_not_drained() {
18806        let mut fixture = route_closed_fixture().await;
18807        // The server calls cleanup_connection on module EOF. Without a planned
18808        // daemon shutdown, that disconnect must remain a crash with no prior
18809        // route.closing notice, even when the route has no in-flight calls.
18810        assert!(matches!(
18811            fixture.disconnect(),
18812            ClientControlPush::RouteClosed {
18813                reason: RouteCloseReason::Crash,
18814                drained: false,
18815                ..
18816            }
18817        ));
18818    }
18819
18820    #[tokio::test]
18821    async fn dropping_router_connection_releases_registration() {
18822        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18823        let registry = Arc::new(Registry::default());
18824        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
18825        let router = Router::with_control_handler(Arc::clone(&control));
18826        let connection = router.begin_connection();
18827        let connection_id = connection.id();
18828        let (ctx, mut rx) = route_ctx(connection_id);
18829
18830        router
18831            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
18832            .await
18833            .unwrap();
18834        let response = rx.recv().await.unwrap();
18835        let ack = parse_ack(&response);
18836        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18837        assert!(registry.get_module("aft").unwrap().is_some());
18838
18839        drop(connection);
18840
18841        assert!(registry.get_module("aft").unwrap().is_none());
18842        assert_eq!(registry.active_registration_count().unwrap(), 0);
18843
18844        control
18845            .cleanup_connection(ConnectionId::new(u64::MAX))
18846            .unwrap();
18847        let captured = crate::router::test_log::captured_logs(&logs);
18848        println!("captured registry lifecycle logs:\n{captured}");
18849        let events: Vec<_> = captured
18850            .lines()
18851            .filter(|line| {
18852                line.contains("module registered module_id=aft ")
18853                    || line.contains("module registration ended")
18854            })
18855            .collect();
18856        assert_eq!(
18857            events.len(),
18858            2,
18859            "unexpected registry lifecycle logs: {captured}"
18860        );
18861        assert!(events[0].contains("module registered module_id=aft "));
18862        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
18863        assert!(events[1].contains(&format!(
18864            "module_id=aft connection_id={}",
18865            connection_id.get()
18866        )));
18867        assert!(events[1].contains("reason=connection_closed"));
18868        assert!(events[1].contains("module registration ended"));
18869        assert_eq!(
18870            captured
18871                .lines()
18872                .filter(|line| line.contains("module registered module_id=aft "))
18873                .count(),
18874            1,
18875            "legacy registration admission line must appear once: {captured}"
18876        );
18877    }
18878
18879    #[tokio::test]
18880    async fn hello_registration_keeps_legacy_module_registered_line_once() {
18881        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18882        let registry = Arc::new(Registry::default());
18883        let control = ControlHandler::new(Arc::clone(&registry));
18884        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
18885        hello_via_sink(
18886            &control,
18887            &ctx,
18888            &mut rx,
18889            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
18890        )
18891        .await;
18892
18893        let captured = crate::router::test_log::captured_logs(&logs);
18894        let admissions: Vec<_> = captured
18895            .lines()
18896            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
18897            .collect();
18898        assert_eq!(
18899            admissions.len(),
18900            1,
18901            "legacy admission line must remain exactly once: {captured}"
18902        );
18903        assert!(admissions[0].contains("routable_provider=true"));
18904        assert!(admissions[0].contains("connection_id=777"));
18905    }
18906
18907    fn capability_manifest(
18908        module_id: &str,
18909        provides: &[&str],
18910        must_never_reach: &[&str],
18911    ) -> ModuleManifest {
18912        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
18913        manifest.capabilities = Some(CapabilityDeclarations {
18914            provides: provides
18915                .iter()
18916                .map(|capability| (*capability).to_string())
18917                .collect(),
18918            requires: Vec::new(),
18919            must_never_reach: must_never_reach
18920                .iter()
18921                .map(|capability| (*capability).to_string())
18922                .collect(),
18923        });
18924        manifest
18925    }
18926
18927    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
18928        Frame::build(
18929            FrameType::Hello,
18930            control_flags(),
18931            0,
18932            0,
18933            corr,
18934            serde_json::to_vec(&ModuleHelloBody {
18935                protocol_ver: manifest.protocol_ver,
18936                manifest,
18937                control_ops: None,
18938                launch_nonce: None,
18939            })
18940            .expect("capability test HELLO serializes"),
18941        )
18942        .expect("capability test HELLO frame builds")
18943    }
18944
18945    fn catalog_update_with_capabilities_frame(
18946        corr: u64,
18947        capabilities: CapabilityDeclarations,
18948    ) -> Frame {
18949        Frame::build(
18950            FrameType::Request,
18951            control_flags(),
18952            0,
18953            0,
18954            corr,
18955            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
18956                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
18957                capabilities: Some(capabilities),
18958                ready: None,
18959            })
18960            .expect("capability catalog.update serializes"),
18961        )
18962        .expect("capability catalog.update frame builds")
18963    }
18964
18965    async fn register_capability_manifest(
18966        handler: &ControlHandler,
18967        ctx: &RouteCtx,
18968        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
18969        manifest: ModuleManifest,
18970        corr: u64,
18971    ) {
18972        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
18973    }
18974
18975    async fn open_route_for_capability_test(
18976        handler: &ControlHandler,
18977        target_ctx: &RouteCtx,
18978        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
18979        client_connection_id: u64,
18980        corr: u64,
18981        target_module_id: &str,
18982        consumer_identity: Option<ConsumerIdentity>,
18983    ) -> (
18984        mpsc::Receiver<crate::router::OutboundFrame>,
18985        ModuleControlRequest,
18986    ) {
18987        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
18988        let route_handler = handler.clone();
18989        let target_module_id = target_module_id.to_string();
18990        let route_task = tokio::spawn(async move {
18991            route_handler
18992                .handle_control_frame(
18993                    &client_ctx,
18994                    route_open_frame_with_admission_facts(
18995                        corr,
18996                        &target_module_id,
18997                        unique_project_root("admission-facts"),
18998                        consumer_identity,
18999                        None,
19000                    ),
19001                )
19002                .await
19003                .expect("capability test route.open succeeds")
19004        });
19005        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
19006            .await
19007            .expect("capability test route.open must reach route.bind")
19008            .expect("target control receiver stays open");
19009        let bind_request: ModuleControlRequest =
19010            serde_json::from_slice(&bind.body).expect("route.bind decodes");
19011        handler
19012            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
19013            .await
19014            .expect("capability test route.bind ACK succeeds");
19015        assert!(route_task.await.expect("route.open task joins").is_empty());
19016        let opened = client_rx
19017            .recv()
19018            .await
19019            .expect("successful route.open publishes a response");
19020        assert!(matches!(
19021            serde_json::from_slice::<ClientControlResponse>(&opened.body),
19022            Ok(ClientControlResponse::RouteOpen { .. })
19023        ));
19024        (client_rx, bind_request)
19025    }
19026
19027    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
19028        assert_eq!(frame.header.ty, FrameType::Push);
19029        assert_eq!(frame.header.channel, 0);
19030        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
19031            .expect("route.closed control push decodes");
19032        let ClientControlPush::RouteClosed { channels, .. } = &push else {
19033            panic!("expected route.closed");
19034        };
19035        assert_eq!(channels.len(), 1, "exactly one violating route closed");
19036        let closed_channel = channels[0];
19037        let channels = channels.clone();
19038        assert_eq!(
19039            push,
19040            ClientControlPush::RouteClosed {
19041                module_id: target_module_id.to_string(),
19042                channels,
19043                reason: RouteCloseReason::CapabilityDenied,
19044                drained: false,
19045                abandoned: 0,
19046                excluded_subscriptions: 0,
19047                terminal: Some(false),
19048            }
19049        );
19050        closed_channel
19051    }
19052
19053    #[tokio::test]
19054    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
19055        let registry = Arc::new(Registry::default());
19056        let forwarding = Arc::new(ForwardingTable::default());
19057        let supervisor = SupervisorHandle::new();
19058        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19059        let handler =
19060            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19061                .with_supervisor(supervisor);
19062        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
19063        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
19064        register_capability_manifest(
19065            &handler,
19066            &target_ctx,
19067            &mut target_rx,
19068            capability_manifest("target", &["credentials-provider/v1"], &[]),
19069            1,
19070        )
19071        .await;
19072        register_capability_manifest(
19073            &handler,
19074            &opener_ctx,
19075            &mut opener_rx,
19076            capability_manifest("opener", &[], &["credentials-provider/v1"]),
19077            2,
19078        )
19079        .await;
19080
19081        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
19082        let replies = handler
19083            .handle_control_frame(
19084                &client_ctx,
19085                route_open_frame_with_admission_facts(
19086                    3,
19087                    "target",
19088                    unique_project_root("admission-facts"),
19089                    Some(ConsumerIdentity {
19090                        module_id: "opener".to_string(),
19091                        launch_nonce: "opener-nonce".to_string(),
19092                    }),
19093                    None,
19094                ),
19095            )
19096            .await
19097            .expect("denied route.open returns a typed frame");
19098        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19099        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19100        assert!(
19101            target_rx.try_recv().is_err(),
19102            "forbidden route.open must not relay route.bind"
19103        );
19104    }
19105
19106    #[tokio::test]
19107    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
19108        let registry = Arc::new(Registry::default());
19109        let forwarding = Arc::new(ForwardingTable::default());
19110        let supervisor = SupervisorHandle::new();
19111        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19112        let handler =
19113            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19114                .with_supervisor(supervisor);
19115        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
19116        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
19117        register_capability_manifest(
19118            &handler,
19119            &target_ctx,
19120            &mut target_rx,
19121            capability_manifest("target", &["credentials-provider/v1"], &[]),
19122            1,
19123        )
19124        .await;
19125        register_capability_manifest(
19126            &handler,
19127            &old_opener_ctx,
19128            &mut old_opener_rx,
19129            capability_manifest("opener", &[], &[]),
19130            2,
19131        )
19132        .await;
19133        let (mut client_rx, _) = open_route_for_capability_test(
19134            &handler,
19135            &target_ctx,
19136            &mut target_rx,
19137            712,
19138            3,
19139            "target",
19140            Some(ConsumerIdentity {
19141                module_id: "opener".to_string(),
19142                launch_nonce: "opener-nonce".to_string(),
19143            }),
19144        )
19145        .await;
19146        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19147
19148        handler
19149            .cleanup_connection(old_opener_ctx.connection_id)
19150            .expect("old opener registration cleans up");
19151        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
19152        register_capability_manifest(
19153            &handler,
19154            &new_opener_ctx,
19155            &mut new_opener_rx,
19156            capability_manifest("opener", &[], &["credentials-provider/v1"]),
19157            4,
19158        )
19159        .await;
19160
19161        let closed_channel = assert_capability_denied_push(
19162            client_rx
19163                .try_recv()
19164                .expect("HELLO deny addition must emit route.closed")
19165                .frame,
19166            "target",
19167        );
19168        let goodbye = client_rx
19169            .try_recv()
19170            .expect("HELLO deny addition must send the client a route GOODBYE")
19171            .frame;
19172        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19173        assert_eq!(goodbye.header.channel, closed_channel);
19174        assert_ne!(goodbye.header.channel, 0);
19175        assert_ne!(goodbye.header.epoch, 0);
19176        assert!(client_rx.try_recv().is_err());
19177        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19178        assert!(matches!(
19179            target_rx.try_recv(),
19180            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19181        ));
19182    }
19183
19184    #[tokio::test]
19185    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
19186        let registry = Arc::new(Registry::default());
19187        let forwarding = Arc::new(ForwardingTable::default());
19188        let supervisor = SupervisorHandle::new();
19189        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19190        let handler =
19191            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19192                .with_supervisor(supervisor);
19193        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
19194        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
19195        register_capability_manifest(
19196            &handler,
19197            &target_ctx,
19198            &mut target_rx,
19199            capability_manifest("target", &[], &[]),
19200            1,
19201        )
19202        .await;
19203        register_capability_manifest(
19204            &handler,
19205            &opener_ctx,
19206            &mut opener_rx,
19207            capability_manifest("opener", &[], &["credentials-provider/v1"]),
19208            2,
19209        )
19210        .await;
19211        let (mut client_rx, _) = open_route_for_capability_test(
19212            &handler,
19213            &target_ctx,
19214            &mut target_rx,
19215            722,
19216            3,
19217            "target",
19218            Some(ConsumerIdentity {
19219                module_id: "opener".to_string(),
19220                launch_nonce: "opener-nonce".to_string(),
19221            }),
19222        )
19223        .await;
19224        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19225
19226        let replies = handler
19227            .handle_control_frame(
19228                &target_ctx,
19229                catalog_update_with_capabilities_frame(
19230                    4,
19231                    CapabilityDeclarations {
19232                        provides: vec!["credentials-provider/v1".to_string()],
19233                        requires: Vec::new(),
19234                        must_never_reach: Vec::new(),
19235                    },
19236                ),
19237            )
19238            .await
19239            .expect("claim catalog.update succeeds");
19240        assert!(matches!(
19241            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
19242            Ok(ModuleControlResponseToModule::CatalogUpdate {})
19243        ));
19244        let closed_channel = assert_capability_denied_push(
19245            client_rx
19246                .try_recv()
19247                .expect("claim addition must emit route.closed")
19248                .frame,
19249            "target",
19250        );
19251        let goodbye = client_rx
19252            .try_recv()
19253            .expect("claim addition must send the client a route GOODBYE")
19254            .frame;
19255        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19256        assert_eq!(goodbye.header.channel, closed_channel);
19257        assert_ne!(goodbye.header.channel, 0);
19258        assert_ne!(goodbye.header.epoch, 0);
19259        assert!(client_rx.try_recv().is_err());
19260        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19261        assert!(matches!(
19262            target_rx.try_recv(),
19263            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19264        ));
19265    }
19266
19267    #[tokio::test]
19268    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
19269        let registry = Arc::new(Registry::default());
19270        let forwarding = Arc::new(ForwardingTable::default());
19271        let supervisor = SupervisorHandle::new();
19272        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19273        let handler =
19274            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19275                .with_supervisor(supervisor);
19276        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
19277        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
19278        register_capability_manifest(
19279            &handler,
19280            &target_ctx,
19281            &mut target_rx,
19282            capability_manifest("target", &["credentials-provider/v1"], &[]),
19283            1,
19284        )
19285        .await;
19286        register_capability_manifest(
19287            &handler,
19288            &opener_ctx,
19289            &mut opener_rx,
19290            capability_manifest("opener", &[], &[]),
19291            2,
19292        )
19293        .await;
19294        let (mut client_rx, _) = open_route_for_capability_test(
19295            &handler,
19296            &target_ctx,
19297            &mut target_rx,
19298            732,
19299            3,
19300            "target",
19301            Some(ConsumerIdentity {
19302                module_id: "opener".to_string(),
19303                launch_nonce: "opener-nonce".to_string(),
19304            }),
19305        )
19306        .await;
19307        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19308
19309        handler
19310            .handle_control_frame(
19311                &target_ctx,
19312                catalog_update_with_capabilities_frame(
19313                    4,
19314                    CapabilityDeclarations {
19315                        provides: Vec::new(),
19316                        requires: Vec::new(),
19317                        must_never_reach: Vec::new(),
19318                    },
19319                ),
19320            )
19321            .await
19322            .expect("claim removal catalog.update succeeds");
19323        assert_eq!(
19324            forwarding.active_binding_count().unwrap(),
19325            1,
19326            "removing an attested target claim must leave the route census unchanged"
19327        );
19328        assert!(
19329            client_rx.try_recv().is_err(),
19330            "claim removal must not emit route.closed or a client route GOODBYE"
19331        );
19332        assert!(
19333            target_rx.try_recv().is_err(),
19334            "claim removal must not send the target a route GOODBYE"
19335        );
19336    }
19337
19338    /// A direct client may open a route to a denied capability provider; this
19339    /// policy applies only to attested supervised module origins, not to direct clients.
19340    #[tokio::test]
19341    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
19342        let registry = Arc::new(Registry::default());
19343        let forwarding = Arc::new(ForwardingTable::default());
19344        let supervisor = SupervisorHandle::new();
19345        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19346        let handler =
19347            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19348                .with_supervisor(supervisor);
19349        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
19350        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
19351        register_capability_manifest(
19352            &handler,
19353            &target_ctx,
19354            &mut target_rx,
19355            capability_manifest("target", &["credentials-provider/v1"], &[]),
19356            1,
19357        )
19358        .await;
19359        register_capability_manifest(
19360            &handler,
19361            &opener_ctx,
19362            &mut opener_rx,
19363            capability_manifest("opener", &[], &["credentials-provider/v1"]),
19364            2,
19365        )
19366        .await;
19367
19368        let (_client_rx, bind) = open_route_for_capability_test(
19369            &handler,
19370            &target_ctx,
19371            &mut target_rx,
19372            742,
19373            3,
19374            "target",
19375            None,
19376        )
19377        .await;
19378        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
19379            panic!("direct scope-honesty route must bind");
19380        };
19381        assert_eq!(principal, Some(Principal::Direct));
19382        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19383    }
19384
19385    /// A module that denies a capability receives no self-route exemption when it
19386    /// also attestedly provides that capability.
19387    #[tokio::test]
19388    async fn must_never_reach_self_route_is_capability_forbidden() {
19389        let registry = Arc::new(Registry::default());
19390        let forwarding = Arc::new(ForwardingTable::default());
19391        let supervisor = SupervisorHandle::new();
19392        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
19393        let handler =
19394            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19395                .with_supervisor(supervisor);
19396        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
19397        register_capability_manifest(
19398            &handler,
19399            &self_ctx,
19400            &mut self_rx,
19401            capability_manifest(
19402                "self-provider",
19403                &["credentials-provider/v1"],
19404                &["credentials-provider/v1"],
19405            ),
19406            1,
19407        )
19408        .await;
19409
19410        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
19411        let replies = handler
19412            .handle_control_frame(
19413                &client_ctx,
19414                route_open_frame_with_admission_facts(
19415                    2,
19416                    "self-provider",
19417                    unique_project_root("admission-facts"),
19418                    Some(ConsumerIdentity {
19419                        module_id: "self-provider".to_string(),
19420                        launch_nonce: "self-nonce".to_string(),
19421                    }),
19422                    None,
19423                ),
19424            )
19425            .await
19426            .expect("self-route refusal returns a typed frame");
19427        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19428        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19429        assert!(
19430            self_rx.try_recv().is_err(),
19431            "self denial must not relay route.bind"
19432        );
19433    }
19434
19435    #[test]
19436    fn unsupported_channel_zero_frame_returns_error() {
19437        let handler = ControlHandler::default();
19438        let request = Frame::build(
19439            FrameType::Request,
19440            control_flags(),
19441            0,
19442            0,
19443            21,
19444            b"opaque".to_vec(),
19445        )
19446        .unwrap();
19447
19448        let response = handler
19449            .handle_control(ConnectionId::new(1), request)
19450            .unwrap();
19451
19452        assert_eq!(response[0].header.ty, FrameType::Error);
19453        assert_eq!(
19454            parse_error(&response[0])["code"],
19455            "unsupported_control_frame"
19456        );
19457    }
19458
19459    /// Blue/green swap at the control-plane boundary. The supervisor that opens
19460    /// a swap is not wired yet, so the candidate is registered here directly
19461    /// into the registry and forwarding candidate slots, the way the swap's
19462    /// HELLO admission will.
19463    mod swap {
19464        use super::*;
19465
19466        const INCUMBENT: ConnectionId = ConnectionId::new(30);
19467        const CANDIDATE: ConnectionId = ConnectionId::new(40);
19468
19469        struct Swap {
19470            registry: Arc<Registry>,
19471            forwarding: Arc<ForwardingTable>,
19472            handler: ControlHandler,
19473            incumbent_ctx: RouteCtx,
19474            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19475            candidate_ctx: RouteCtx,
19476            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19477        }
19478
19479        async fn swap_with_incumbent() -> Swap {
19480            let registry = Arc::new(Registry::default());
19481            let forwarding = Arc::new(ForwardingTable::default());
19482            let handler =
19483                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
19484            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
19485            hello_via_sink(
19486                &handler,
19487                &incumbent_ctx,
19488                &mut incumbent_rx,
19489                hello_frame("aft", PROTOCOL_VERSION, 7),
19490            )
19491            .await;
19492            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
19493            Swap {
19494                registry,
19495                forwarding,
19496                handler,
19497                incumbent_ctx,
19498                incumbent_rx,
19499                candidate_ctx,
19500                candidate_rx,
19501            }
19502        }
19503
19504        fn register_candidate(swap: &Swap, ready: Option<bool>) {
19505            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
19506            candidate_manifest.ready = ready;
19507            let registration = swap
19508                .registry
19509                .register_candidate_with_control_ops(
19510                    candidate_manifest,
19511                    PROTOCOL_VERSION,
19512                    CANDIDATE,
19513                    module_baseline_control_ops(),
19514                )
19515                .unwrap();
19516            swap.forwarding
19517                .register_candidate_module_connection(
19518                    CANDIDATE,
19519                    "aft".to_string(),
19520                    PROTOCOL_VERSION,
19521                    manifest_concurrency(&registration.manifest),
19522                    swap.candidate_ctx.egress.clone(),
19523                )
19524                .unwrap();
19525        }
19526
19527        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
19528            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
19529            swap.registry.promote_candidate("aft").unwrap().unwrap();
19530            cutover.incumbent.unwrap()
19531        }
19532
19533        /// A swap candidate's HELLO reserves its candidate registry entry
19534        /// before installing its candidate forwarding endpoint. The swap
19535        /// supervisor promotes a ready candidate as soon as it reads it as
19536        /// registered, and promotion cuts forwarding over to the candidate
19537        /// endpoint, so the lookup the supervisor reads
19538        /// (`Registry::published_registration`) must not return the candidate
19539        /// until that endpoint exists. Otherwise the cutover finds no candidate
19540        /// endpoint and the swap fails as if the candidate had disconnected.
19541        #[tokio::test]
19542        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
19543            let mut swap = swap_with_incumbent().await;
19544            let hold = Arc::new(std::sync::Barrier::new(2));
19545            let handler = swap
19546                .handler
19547                .clone()
19548                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
19549            handler
19550                .supervisor
19551                .open_swap("aft", "candidate-nonce".to_string());
19552            let hello = {
19553                let handler = handler.clone();
19554                let candidate_ctx = swap.candidate_ctx.clone();
19555                std::thread::spawn(move || {
19556                    handler
19557                        .handle_hello(
19558                            candidate_ctx.connection_id,
19559                            Some(candidate_ctx.egress.clone()),
19560                            hello_frame_with_nonce(
19561                                "aft",
19562                                PROTOCOL_VERSION,
19563                                8,
19564                                Some("candidate-nonce"),
19565                            ),
19566                        )
19567                        .unwrap()
19568                })
19569            };
19570
19571            hold.wait();
19572            // Parked: the candidate slot is claimed but not published, and
19573            // forwarding has no candidate endpoint.
19574            assert!(swap
19575                .registry
19576                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
19577                .unwrap()
19578                .is_some());
19579            assert!(swap
19580                .registry
19581                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19582                .unwrap()
19583                .is_none());
19584            assert!(swap
19585                .forwarding
19586                .module_endpoint_for_connection(CANDIDATE)
19587                .unwrap()
19588                .is_none());
19589
19590            hold.wait();
19591            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
19592            assert_eq!(
19593                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19594                FrameType::HelloAck
19595            );
19596            let published = swap
19597                .registry
19598                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19599                .unwrap()
19600                .expect("candidate is published once its HELLO completes");
19601            assert_eq!(published.connection_id, CANDIDATE);
19602            assert!(swap
19603                .forwarding
19604                .module_endpoint_for_connection(CANDIDATE)
19605                .unwrap()
19606                .is_some());
19607        }
19608
19609        #[tokio::test]
19610        async fn candidate_lookup_after_hello_ack_waits_for_publication() {
19611            let mut swap = swap_with_incumbent().await;
19612            let hold = Arc::new(std::sync::Barrier::new(2));
19613            swap.handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
19614            swap.handler
19615                .supervisor
19616                .open_swap("aft", "candidate-nonce".to_string());
19617            let hello = {
19618                let handler = swap.handler.clone();
19619                let candidate_ctx = swap.candidate_ctx.clone();
19620                std::thread::spawn(move || {
19621                    handler
19622                        .handle_hello(
19623                            CANDIDATE,
19624                            Some(candidate_ctx.egress),
19625                            hello_frame_with_nonce(
19626                                "aft",
19627                                PROTOCOL_VERSION,
19628                                8,
19629                                Some("candidate-nonce"),
19630                            ),
19631                        )
19632                        .unwrap()
19633                })
19634            };
19635            hold.wait();
19636            assert_eq!(
19637                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19638                FrameType::HelloAck
19639            );
19640            let reader_registry = Arc::clone(&swap.registry);
19641            let published = lookup_during_hello_publication(&swap.registry, &hold, move || {
19642                reader_registry
19643                    .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19644                    .unwrap()
19645            });
19646            assert!(hello.join().unwrap().is_empty());
19647            assert_eq!(
19648                published
19649                    .expect("ACK makes candidate publication visible")
19650                    .connection_id,
19651                CANDIDATE
19652            );
19653            assert_eq!(
19654                swap.registry.generation().unwrap(),
19655                1,
19656                "candidate is not a second catalog entry"
19657            );
19658        }
19659
19660        #[tokio::test]
19661        async fn audit4_ungated_removed_candidate_publication_rolls_back_admission() {
19662            let mut swap = swap_with_incumbent().await;
19663            let hold = Arc::new(std::sync::Barrier::new(2));
19664            let handler = swap
19665                .handler
19666                .clone()
19667                .with_hello_registry_to_forwarding_hold(hold.clone());
19668            handler
19669                .supervisor
19670                .open_swap("aft", "candidate-nonce".into());
19671            let hello = {
19672                let handler = handler.clone();
19673                let ctx = swap.candidate_ctx.clone();
19674                std::thread::spawn(move || {
19675                    handler
19676                        .handle_hello(
19677                            CANDIDATE,
19678                            Some(ctx.egress),
19679                            hello_frame_with_nonce(
19680                                "aft",
19681                                PROTOCOL_VERSION,
19682                                8,
19683                                Some("candidate-nonce"),
19684                            ),
19685                        )
19686                        .unwrap()
19687                })
19688            };
19689            hold.wait();
19690            assert_eq!(
19691                swap.registry
19692                    .deregister_connection(CANDIDATE)
19693                    .unwrap()
19694                    .len(),
19695                1
19696            );
19697            hold.wait();
19698            assert!(hello.join().unwrap().is_empty());
19699            assert_eq!(
19700                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19701                FrameType::HelloAck
19702            );
19703            assert!(swap.registry.get_candidate("aft").unwrap().is_none());
19704            assert_eq!(
19705                handler
19706                    .supervisor
19707                    .swap_hello_admission("aft", Some("candidate-nonce")),
19708                SwapHelloAdmission::Candidate,
19709                "failed publication must roll back candidate_admitted for an ungated nonce swap"
19710            );
19711            handler.supervisor.close_swap("aft");
19712        }
19713
19714        fn keyed_total(counters: &Value, key: &str) -> u64 {
19715            counters[key]
19716                .as_object()
19717                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
19718                .unwrap_or(0)
19719        }
19720
19721        #[tokio::test]
19722        async fn replacement_logs_old_end_and_new_admission_once() {
19723            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
19724            let mut swap = swap_with_incumbent().await;
19725            swap.handler
19726                .supervisor
19727                .open_swap("aft", "candidate-nonce".to_string());
19728            hello_via_sink(
19729                &swap.handler,
19730                &swap.candidate_ctx,
19731                &mut swap.candidate_rx,
19732                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
19733            )
19734            .await;
19735            cutover(&swap);
19736            swap.handler.cleanup_connection(INCUMBENT).unwrap();
19737
19738            let captured = crate::router::test_log::captured_logs(&logs);
19739            println!("captured replacement registry logs:\n{captured}");
19740            let new_admissions: Vec<_> = captured
19741                .lines()
19742                .filter(|line| {
19743                    line.contains("connection_id=40")
19744                        && line.contains("swap candidate registered; not routable until cutover")
19745                })
19746                .collect();
19747            assert_eq!(
19748                new_admissions.len(),
19749                1,
19750                "unexpected admission logs: {captured}"
19751            );
19752            assert!(new_admissions[0].contains("module_id=aft"));
19753            assert!(new_admissions[0].contains("connection_id=40"));
19754            assert!(
19755                new_admissions[0].contains("swap candidate registered; not routable until cutover")
19756            );
19757            assert!(new_admissions[0].contains("ready=true"));
19758
19759            let old_admissions: Vec<_> = captured
19760                .lines()
19761                .filter(|line| {
19762                    line.contains("module registered module_id=aft ")
19763                        && line.contains("connection_id=30")
19764                })
19765                .collect();
19766            assert_eq!(
19767                old_admissions.len(),
19768                1,
19769                "unexpected incumbent admission: {captured}"
19770            );
19771
19772            let old_ends: Vec<_> = captured
19773                .lines()
19774                .filter(|line| {
19775                    line.contains("connection_id=30") && line.contains("module registration ended")
19776                })
19777                .collect();
19778            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
19779            assert!(old_ends[0].contains("module_id=aft"));
19780            assert!(old_ends[0].contains("connection_id=30"));
19781            assert!(old_ends[0].contains("reason=replaced"));
19782            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
19783            assert_eq!(
19784                captured
19785                    .lines()
19786                    .filter(|line| line.contains("module registration promoted"))
19787                    .count(),
19788                1,
19789                "unexpected promotion logs: {captured}"
19790            );
19791        }
19792
19793        /// An ack from the incumbent for a bind it was sent before cutover,
19794        /// arriving before the incumbent is drained. The incumbent is the live
19795        /// connection carrying every other client's routes, so the ack must
19796        /// not end it: the waiting client is told to retry, the reservation is
19797        /// given back, and the incumbent is told to drop just that binding.
19798        #[tokio::test]
19799        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
19800            let mut swap = swap_with_incumbent().await;
19801            let handler = swap.handler.clone();
19802
19803            // A co-tenant route, bound on the incumbent before the swap.
19804            let cotenant = ConnectionId::new(31);
19805            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
19806            let (cotenant_task, cotenant_bind) = relay_route_open(
19807                &handler,
19808                cotenant,
19809                &cotenant_ctx.egress,
19810                &mut swap.incumbent_rx,
19811                100,
19812                "aft",
19813                "swap-cotenant",
19814            )
19815            .await;
19816            handler
19817                .handle_control_frame(
19818                    &swap.incumbent_ctx,
19819                    route_bind_ack(cotenant_bind.header.corr),
19820                )
19821                .await
19822                .unwrap();
19823            assert!(cotenant_task.await.unwrap().is_empty());
19824            let (cotenant_channel, cotenant_epoch) =
19825                published_route(&cotenant_rx.recv().await.unwrap());
19826
19827            // A second route.open, relayed to the incumbent and not yet acked.
19828            let caller = ConnectionId::new(32);
19829            let (caller_ctx, mut caller_rx) = route_ctx(caller);
19830            let (caller_task, caller_bind) = relay_route_open(
19831                &handler,
19832                caller,
19833                &caller_ctx.egress,
19834                &mut swap.incumbent_rx,
19835                101,
19836                "aft",
19837                "swap-caller",
19838            )
19839            .await;
19840            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
19841
19842            register_candidate(&swap, None);
19843            cutover(&swap);
19844
19845            // The incumbent acks after promotion and before any drain.
19846            let ack = handler
19847                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
19848                .await;
19849            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
19850            if module_loop_error.is_some() {
19851                // What the connection loop does with an untranslated router
19852                // error: end the connection, releasing every route on it.
19853                handler.cleanup_connection(INCUMBENT).unwrap();
19854            }
19855
19856            // 1. The incumbent's other routes survive.
19857            assert!(
19858                cotenant_rx.try_recv().is_err(),
19859                "the co-tenant route on the incumbent was torn down by one late ack: \
19860                 {module_loop_error:?}"
19861            );
19862            assert!(matches!(
19863                swap.forwarding
19864                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
19865                    .unwrap(),
19866                DataRoute::Client(DataRouteState::Bound(_))
19867            ));
19868            assert_eq!(module_loop_error, None);
19869            assert!(swap
19870                .registry
19871                .get_module_by_connection(INCUMBENT)
19872                .unwrap()
19873                .is_some());
19874
19875            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
19876            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
19877                .await
19878                .expect("the incumbent is told to drop the abandoned binding")
19879                .unwrap()
19880                .frame;
19881            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19882            assert_eq!(goodbye.header.channel, abandoned_channel);
19883            assert_eq!(goodbye.header.epoch, abandoned_epoch);
19884            assert!(swap.incumbent_rx.try_recv().is_err());
19885
19886            // 3. The waiting client gets a retryable refusal and no route.
19887            let response = caller_task.await.unwrap();
19888            assert_eq!(response.len(), 1);
19889            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
19890            assert!(caller_rx.try_recv().is_err());
19891
19892            // 4. The reservation pair is given back, and the pending bind
19893            //    settled exactly once: one accepted open (the co-tenant) and one
19894            //    refused open (the caller), nothing counted twice.
19895            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
19896            let counters = handler.counters().snapshot();
19897            assert_eq!(
19898                keyed_total(&counters, "route_open_accepted_by_principal"),
19899                1
19900            );
19901            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
19902            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
19903        }
19904
19905        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
19906        /// id would resolve to the promoted candidate and every new route.open
19907        /// would be refused as reloading, leaving neither process routable.
19908        #[tokio::test]
19909        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
19910            let mut swap = swap_with_incumbent().await;
19911            register_candidate(&swap, None);
19912            let incumbent = cutover(&swap);
19913            swap.forwarding
19914                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
19915                .unwrap()
19916                .expect("the incumbent is still registered");
19917
19918            let client = ConnectionId::new(33);
19919            let (client_ctx, mut client_rx) = route_ctx(client);
19920            let route_handler = swap.handler.clone();
19921            let open_ctx = RouteCtx {
19922                connection_id: client,
19923                egress: client_ctx.egress.clone(),
19924            };
19925            let mut route_task = tokio::spawn(async move {
19926                route_handler
19927                    .handle_control_frame(
19928                        &open_ctx,
19929                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
19930                    )
19931                    .await
19932                    .unwrap()
19933            });
19934            let bind = tokio::select! {
19935                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
19936                response = &mut route_task => {
19937                    let response = response.unwrap();
19938                    panic!(
19939                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
19940                        parse_error(&response[0])["code"]
19941                    );
19942                }
19943            };
19944            swap.handler
19945                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
19946                .await
19947                .unwrap();
19948            assert!(route_task.await.unwrap().is_empty());
19949            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
19950            match swap
19951                .forwarding
19952                .lookup_data_route(client, channel, epoch)
19953                .unwrap()
19954            {
19955                DataRoute::Client(DataRouteState::Bound(route)) => {
19956                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
19957                }
19958                other => panic!("expected a bound route on the candidate, got {other:?}"),
19959            }
19960            assert!(swap.incumbent_rx.try_recv().is_err());
19961        }
19962
19963        /// A candidate declares itself ready with `catalog.update` on its own
19964        /// connection. If the connection-keyed registry lookups searched only the
19965        /// active slot, this would answer `not_registered` and the candidate
19966        /// would never become ready.
19967        #[tokio::test]
19968        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
19969            let swap = swap_with_incumbent().await;
19970            register_candidate(&swap, Some(false));
19971            let update = Frame::build(
19972                FrameType::Request,
19973                control_flags(),
19974                0,
19975                0,
19976                55,
19977                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
19978                    provides: manifest("aft", PROTOCOL_VERSION).provides,
19979                    capabilities: None,
19980                    ready: Some(true),
19981                })
19982                .unwrap(),
19983            )
19984            .unwrap();
19985
19986            let replies = swap
19987                .handler
19988                .handle_control_frame(&swap.candidate_ctx, update)
19989                .await
19990                .unwrap();
19991
19992            assert_eq!(replies.len(), 1);
19993            assert_eq!(
19994                replies[0].header.ty,
19995                FrameType::Response,
19996                "candidate catalog.update was refused: {:?}",
19997                serde_json::from_slice::<Value>(&replies[0].body).ok()
19998            );
19999            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
20000            assert_eq!(
20001                swap.registry
20002                    .get_module("aft")
20003                    .unwrap()
20004                    .unwrap()
20005                    .connection_id,
20006                INCUMBENT
20007            );
20008        }
20009    }
20010
20011    /// The HELLO gate while the supervisor has a swap open: only the nonce it
20012    /// minted for the candidate admits a second process, into the candidate
20013    /// slot, and that check runs ahead of the reserved-module gate.
20014    mod swap_admission {
20015        use super::*;
20016
20017        const INCUMBENT_NONCE: &str = "incumbent-nonce";
20018        const CANDIDATE_NONCE: &str = "candidate-nonce";
20019
20020        fn handler_with_incumbent(
20021            module_id: &str,
20022            reserved: bool,
20023        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
20024            let registry = Arc::new(Registry::default());
20025            let supervisor = SupervisorHandle::new();
20026            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
20027            if reserved {
20028                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
20029            }
20030            let handler =
20031                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
20032            let incumbent = handler
20033                .handle_control(
20034                    ConnectionId::new(1),
20035                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
20036                )
20037                .unwrap();
20038            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
20039            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
20040            (registry, supervisor, handler)
20041        }
20042
20043        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
20044        /// admits every nonce, so while a swap is open the swap gate is the only
20045        /// thing between a key-holder and the candidate slot. A nonce the
20046        /// supervisor did not mint, or none at all, is refused, and neither the
20047        /// incumbent's registration nor the candidate slot moves.
20048        #[test]
20049        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
20050            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
20051
20052            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
20053                let replies = handler
20054                    .handle_control(
20055                        ConnectionId::new(connection),
20056                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
20057                    )
20058                    .unwrap();
20059                assert_eq!(replies[0].header.ty, FrameType::Error);
20060                assert_eq!(
20061                    parse_error(&replies[0])["code"],
20062                    "swap_token_invalid",
20063                    "nonce {nonce:?}"
20064                );
20065            }
20066            assert!(registry.get_candidate("aft").unwrap().is_none());
20067            assert_eq!(
20068                registry.get_module("aft").unwrap().unwrap().connection_id,
20069                ConnectionId::new(1)
20070            );
20071
20072            // Control: the minted token is admitted, into the candidate slot,
20073            // and only once.
20074            let admitted = handler
20075                .handle_control(
20076                    ConnectionId::new(4),
20077                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
20078                )
20079                .unwrap();
20080            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
20081            assert_eq!(
20082                registry
20083                    .get_candidate("aft")
20084                    .unwrap()
20085                    .unwrap()
20086                    .connection_id,
20087                ConnectionId::new(4)
20088            );
20089            assert_eq!(
20090                registry.get_module("aft").unwrap().unwrap().connection_id,
20091                ConnectionId::new(1),
20092                "the candidate must not take the active slot"
20093            );
20094            let replayed = handler
20095                .handle_control(
20096                    ConnectionId::new(5),
20097                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
20098                )
20099                .unwrap();
20100            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
20101
20102            // The case only this gate covers: the incumbent has died mid-swap,
20103            // so its duplicate refusal is gone too, and without the gate a
20104            // key-holder would take the id's ACTIVE slot.
20105            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
20106            let squatter = handler
20107                .handle_control(
20108                    ConnectionId::new(6),
20109                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
20110                )
20111                .unwrap();
20112            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
20113            assert!(
20114                registry.get_module("aft").unwrap().is_none(),
20115                "a squatter took the active slot of an id being swapped"
20116            );
20117        }
20118
20119        /// Design mutation arm (iii). A reserved module's candidate presents a
20120        /// nonce the reserved gate has never seen (that gate holds the
20121        /// incumbent's), so the swap gate must run first or the candidate is
20122        /// refused `reserved_module` and a reserved module can never be swapped.
20123        #[test]
20124        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
20125            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
20126
20127            let replies = handler
20128                .handle_control(
20129                    ConnectionId::new(2),
20130                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20131                )
20132                .unwrap();
20133
20134            assert_eq!(
20135                replies[0].header.ty,
20136                FrameType::HelloAck,
20137                "reserved candidate refused: {:?}",
20138                serde_json::from_slice::<Value>(&replies[0].body).ok()
20139            );
20140            assert_eq!(
20141                registry
20142                    .get_candidate("vault")
20143                    .unwrap()
20144                    .unwrap()
20145                    .connection_id,
20146                ConnectionId::new(2)
20147            );
20148        }
20149
20150        /// With no swap open the gate is inert: the incumbent's reserved gate
20151        /// and duplicate refusal behave exactly as before.
20152        #[test]
20153        fn without_an_open_swap_the_ordinary_gates_decide() {
20154            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
20155            supervisor.close_swap("vault");
20156
20157            let candidate = handler
20158                .handle_control(
20159                    ConnectionId::new(2),
20160                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20161                )
20162                .unwrap();
20163            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
20164            let duplicate = handler
20165                .handle_control(
20166                    ConnectionId::new(3),
20167                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
20168                )
20169                .unwrap();
20170            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
20171            assert!(registry.get_candidate("vault").unwrap().is_none());
20172        }
20173    }
20174
20175    /// `scope.sync` and `scope.describe` through the real control handler: who
20176    /// may sync is decided by the registration and launch nonce of the module
20177    /// connection, never by the request body.
20178    mod scopes {
20179        mod close_reasons {
20180            include!("route_close_reason_tests.rs");
20181        }
20182        mod idle_tests {
20183            use super::*;
20184            include!("scope_idle_tests.rs");
20185        }
20186        mod route_open_timings {
20187            use super::*;
20188            include!("route_open_timing_tests.rs");
20189        }
20190        mod route_open_refusal_log {
20191            use super::*;
20192            include!("route_open_refusal_log_tests.rs");
20193        }
20194        mod activity {
20195            include!("activity_scope_tests.rs");
20196        }
20197        mod delta {
20198            include!("scope_handler_delta_tests.rs");
20199        }
20200        mod call_modes {
20201            include!("scope_call_mode_tests.rs");
20202        }
20203        mod federated {
20204            include!("scope_federated_tests.rs");
20205        }
20206        mod authorize {
20207            include!("route_authorize_tests.rs");
20208        }
20209        #[tokio::test]
20210        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
20211            let rig = rig().await;
20212            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
20213                "s".into(),
20214                1,
20215                "tool".into(),
20216            ))
20217            .unwrap();
20218            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
20219            let (direct, _rx) = wide_ctx(999);
20220            for (ctx, expected) in [
20221                (&rig.owner, FrameType::Response),
20222                (&direct, FrameType::Error),
20223            ] {
20224                let frame =
20225                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
20226                        .unwrap();
20227                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
20228                assert_eq!(replies[0].header.ty, expected);
20229                if expected == FrameType::Response {
20230                    let reply: subc_control::ScopeServedByReply =
20231                        serde_json::from_slice(&replies[0].body).unwrap();
20232                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
20233                } else {
20234                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
20235                }
20236            }
20237            let reply = call(
20238                &rig.handler,
20239                &rig.owner,
20240                &ModuleControlRequestFromModule::ScopeDescribe {
20241                    owner: Principal::Reserved {
20242                        module_id: OWNER.into(),
20243                    },
20244                    scope_ref: "s".into(),
20245                },
20246            )
20247            .await;
20248            assert!(matches!(
20249                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
20250                ModuleControlResponseToModule::ScopeDescribe { .. }
20251            ));
20252        }
20253
20254        use subc_protocol::scope::{
20255            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
20256            ScopeStatus,
20257        };
20258
20259        use super::*;
20260
20261        const OWNER: &str = "prefrontal-core";
20262
20263        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
20264            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
20265        }
20266
20267        async fn call(
20268            handler: &ControlHandler,
20269            ctx: &RouteCtx,
20270            request: &ModuleControlRequestFromModule,
20271        ) -> Frame {
20272            let body = serde_json::to_vec(request).unwrap();
20273            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
20274            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
20275            assert_eq!(replies.len(), 1, "{replies:?}");
20276            replies.pop().unwrap()
20277        }
20278
20279        async fn sync(
20280            handler: &ControlHandler,
20281            ctx: &RouteCtx,
20282            generation: u64,
20283            scopes: Vec<ScopeRecord>,
20284        ) -> Result<ModuleControlResponseToModule, String> {
20285            let reply = call(
20286                handler,
20287                ctx,
20288                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
20289            )
20290            .await;
20291            match reply.header.ty {
20292                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
20293                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
20294            }
20295        }
20296
20297        async fn describe(
20298            handler: &ControlHandler,
20299            ctx: &RouteCtx,
20300            owner: &str,
20301            scope_ref: &str,
20302        ) -> ModuleControlResponseToModule {
20303            let reply = call(
20304                handler,
20305                ctx,
20306                &ModuleControlRequestFromModule::ScopeDescribe {
20307                    owner: Principal::Reserved {
20308                        module_id: owner.to_string(),
20309                    },
20310                    scope_ref: scope_ref.to_string(),
20311                },
20312            )
20313            .await;
20314            assert_eq!(
20315                reply.header.ty,
20316                FrameType::Response,
20317                "{:?}",
20318                parse_error(&reply)
20319            );
20320            serde_json::from_slice(&reply.body).unwrap()
20321        }
20322
20323        #[tokio::test]
20324        async fn scope_describe_never_echoes_bind_proof_or_hash() {
20325            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
20326            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
20327            let mut rig = rig().await;
20328            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
20329                .await;
20330            let _bound = rig
20331                .bound(
20332                    Some(AFT),
20333                    PLEXUS,
20334                    Some(
20335                        rig.selector("s", Some(1))
20336                            .with_bind_proof(Some(PROOF.into())),
20337                    ),
20338                )
20339                .await;
20340            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
20341            let output = serde_json::to_string(&reply).unwrap();
20342            assert!(output.contains("\"status\":\"live\""));
20343            assert!(!output.contains(PROOF));
20344            assert!(!output.contains(HASH));
20345        }
20346
20347        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
20348        async fn module(
20349            handler: &ControlHandler,
20350            connection: u64,
20351            module_id: &str,
20352            nonce: Option<&str>,
20353        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20354            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
20355            hello_via_sink(
20356                handler,
20357                &ctx,
20358                &mut rx,
20359                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
20360            )
20361            .await;
20362            (ctx, rx)
20363        }
20364
20365        /// `direct` and every other client connection has no registration, so
20366        /// it can neither sync nor own a scope.
20367        #[tokio::test]
20368        async fn a_client_connection_cannot_sync_or_describe() {
20369            let handler = ControlHandler::new(Arc::new(Registry::default()));
20370            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
20371            for request in [
20372                ModuleControlRequestFromModule::ScopeSync {
20373                    generation: 1,
20374                    scopes: vec![head("s", 1)],
20375                },
20376                ModuleControlRequestFromModule::ScopeDescribe {
20377                    owner: Principal::Direct,
20378                    scope_ref: "s".to_string(),
20379                },
20380            ] {
20381                let reply = call(&handler, &ctx, &request).await;
20382                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
20383            }
20384            assert!(
20385                !handler
20386                    .scopes
20387                    .read()
20388                    .unwrap()
20389                    .describe(
20390                        &Principal::Reserved {
20391                            module_id: OWNER.to_string()
20392                        },
20393                        "s"
20394                    )
20395                    .owner_synced
20396            );
20397        }
20398
20399        /// A module the supervisor did not spawn registers without a launch
20400        /// nonce, so it is never an owner's current launch.
20401        #[tokio::test]
20402        async fn a_module_without_a_supervised_launch_cannot_sync() {
20403            let handler = ControlHandler::new(Arc::new(Registry::default()));
20404            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
20405            assert_eq!(
20406                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
20407                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20408            );
20409        }
20410
20411        #[tokio::test]
20412        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
20413            let supervisor = SupervisorHandle::new();
20414            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20415            let handler = ControlHandler::new(Arc::new(Registry::default()))
20416                .with_supervisor(supervisor.clone());
20417            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20418            sync(&handler, &incumbent, 1, vec![head("s", 1)])
20419                .await
20420                .expect("the current launch syncs");
20421
20422            // A swap candidate registers with the swap token and is refused
20423            // while the incumbent keeps syncing.
20424            supervisor.open_swap(OWNER, "n2".to_string());
20425            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
20426            assert_eq!(
20427                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20428                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20429            );
20430            sync(&handler, &incumbent, 2, vec![head("s", 1)])
20431                .await
20432                .expect("the serving owner syncs during the swap");
20433
20434            // The swap fails and is rolled back. The candidate never held sync
20435            // authority, and still cannot sync.
20436            supervisor.close_swap(OWNER);
20437            assert_eq!(
20438                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20439                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20440            );
20441            sync(&handler, &incumbent, 3, vec![head("s", 1)])
20442                .await
20443                .expect("the serving owner syncs after the rollback");
20444            handler.cleanup_connection(candidate.connection_id).unwrap();
20445
20446            // A swap that cuts over. Promotion records the candidate's nonce as
20447            // the module's spawn nonce, which is what `set_spawn_nonce` does
20448            // here; the promoted connection then takes authority at any
20449            // generation and the superseded incumbent is refused.
20450            supervisor.open_swap(OWNER, "n3".to_string());
20451            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
20452            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
20453            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
20454                .await
20455                .expect("the promoted launch takes authority");
20456            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
20457                panic!("unexpected reply {reply:?}");
20458            };
20459            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
20460            assert_eq!(
20461                sync(&handler, &incumbent, 4, Vec::new()).await,
20462                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20463            );
20464        }
20465
20466        /// Authority dies with its connection: the cleanup path releases it,
20467        /// so the owner's next connection takes it at any generation.
20468        #[tokio::test]
20469        async fn closing_the_authority_connection_frees_sync_authority() {
20470            let supervisor = SupervisorHandle::new();
20471            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20472            let handler = ControlHandler::new(Arc::new(Registry::default()))
20473                .with_supervisor(supervisor.clone());
20474            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20475            sync(&handler, &first, 10, vec![head("s", 1)])
20476                .await
20477                .unwrap();
20478            handler.cleanup_connection(first.connection_id).unwrap();
20479
20480            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
20481            sync(&handler, &second, 1, vec![head("s", 1)])
20482                .await
20483                .expect("the next connection takes the released authority");
20484        }
20485
20486        #[tokio::test]
20487        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
20488            let supervisor = SupervisorHandle::new();
20489            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20490            let handler =
20491                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
20492            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
20493            sync(&handler, &first, 10, vec![head("s", 1)])
20494                .await
20495                .unwrap();
20496            handler
20497                .handle_control_frame(
20498                    &first,
20499                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
20500                )
20501                .await
20502                .unwrap();
20503            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
20504            sync(&handler, &second, 1, vec![head("s", 1)])
20505                .await
20506                .expect("GOODBYE releases authority even if the old socket remains open");
20507        }
20508
20509        #[tokio::test]
20510        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
20511            let registry = Arc::new(Registry::default());
20512            let supervisor_handle = SupervisorHandle::new();
20513            let supervisor =
20514                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
20515                    .with_handle(supervisor_handle.clone())
20516                    .with_daemon_incarnation("incarnation-7".to_string());
20517            // Configured with enabled: false, so the supervisor lists the
20518            // module without spawning a process for it.
20519            supervisor
20520                .supervise_configured(
20521                    ModuleSpec {
20522                        script: None,
20523                        carry_containment: false,
20524                        module_id: OWNER.to_string(),
20525                        program: PathBuf::from("/nonexistent/prefrontal-core"),
20526                        args: Vec::new(),
20527                        env: Vec::new(),
20528                        reserved: false,
20529                        reserved_prefixes: Vec::new(),
20530                        protocol: ModuleProtocol::Subc,
20531                        overlap: Default::default(),
20532                    },
20533                    false,
20534                )
20535                .unwrap();
20536            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
20537            let handler =
20538                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
20539            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
20540
20541            // Configured but not yet synced: a reader waits for the owner.
20542            let ModuleControlResponseToModule::ScopeDescribe {
20543                status,
20544                daemon_incarnation,
20545                owner_synced,
20546                owner_configured,
20547                scope,
20548                ..
20549            } = describe(&handler, &reader, OWNER, "s").await
20550            else {
20551                panic!("not a describe reply");
20552            };
20553            assert_eq!(status, ScopeStatus::NotLive);
20554            assert_eq!(daemon_incarnation, "incarnation-7");
20555            assert!(!owner_synced);
20556            assert!(owner_configured);
20557            assert!(scope.is_none());
20558
20559            // Not a supervised module: the owner will never sync, and a reader
20560            // refuses rather than waits.
20561            let ModuleControlResponseToModule::ScopeDescribe {
20562                status,
20563                owner_configured,
20564                ..
20565            } = describe(&handler, &reader, "ghost", "s").await
20566            else {
20567                panic!("not a describe reply");
20568            };
20569            assert_eq!(status, ScopeStatus::NotLive);
20570            assert!(!owner_configured);
20571
20572            // Live, with the stamp fields and the computed owner_authorized.
20573            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
20574            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
20575            let ModuleControlResponseToModule::ScopeDescribe {
20576                status,
20577                scope_epoch,
20578                owner_synced,
20579                scope,
20580                ..
20581            } = describe(&handler, &reader, OWNER, "s").await
20582            else {
20583                panic!("not a describe reply");
20584            };
20585            assert_eq!(status, ScopeStatus::Live);
20586            assert_eq!(scope_epoch, Some(4));
20587            assert!(owner_synced);
20588            let stamp = scope.expect("a live scope carries its stamp");
20589            assert!(
20590                stamp.owner_authorized,
20591                "prefrontal-core is the default authority"
20592            );
20593            assert_eq!(stamp.kind, ScopeKind::Head);
20594        }
20595
20596        #[tokio::test]
20597        async fn scope_authority_owners_decides_owner_authorized() {
20598            let supervisor = SupervisorHandle::new();
20599            supervisor.set_spawn_nonce("broca", "b1".to_string());
20600            let handler = ControlHandler::new(Arc::new(Registry::default()))
20601                .with_supervisor(supervisor)
20602                .with_scope_authority_owners(vec!["broca".to_string()]);
20603            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
20604            let mut gated = head("s", 1);
20605            gated.attributes.agent_id = Some("agent".to_string());
20606            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
20607            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
20608                describe(&handler, &broca, "broca", "s").await
20609            else {
20610                panic!("not a describe reply");
20611            };
20612            assert!(scope.unwrap().owner_authorized);
20613        }
20614
20615        /// With route admission, the stamp, the commit re-check and drains in
20616        /// place, the feature is advertised: the module ops in HELLO_ACK, and
20617        /// `scopes/v1` in HELLO_ACK and `server.describe`.
20618        #[tokio::test]
20619        async fn scope_ops_and_the_scopes_capability_are_advertised() {
20620            let handler = ControlHandler::new(Arc::new(Registry::default()));
20621            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
20622            let ack = hello_via_sink(
20623                &handler,
20624                &ctx,
20625                &mut rx,
20626                hello_frame("m", PROTOCOL_VERSION, 1),
20627            )
20628            .await;
20629            let ack = parse_ack(&ack);
20630            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
20631                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
20632            }
20633            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
20634
20635            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
20636            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
20637            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
20638            let reply = handler
20639                .handle_control_frame(&client, frame)
20640                .await
20641                .unwrap()
20642                .pop()
20643                .unwrap();
20644            let ClientControlResponse::ServerDescribe { capabilities, .. } =
20645                serde_json::from_slice(&reply.body).unwrap()
20646            else {
20647                panic!("not a server.describe reply");
20648            };
20649            assert!(
20650                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
20651                "{capabilities:?}"
20652            );
20653        }
20654
20655        // ---- route admission, stamps, commit re-check and drains ----------
20656
20657        const PLEXUS: &str = "plexus";
20658        const OTHER: &str = "other";
20659        const AFT: &str = "aft";
20660        const BROCA: &str = "broca";
20661        const MAGIC: &str = "magic-context";
20662
20663        fn nonce(module_id: &str) -> String {
20664            format!("nonce-{module_id}")
20665        }
20666
20667        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20668            let (tx, rx) = mpsc::channel(64);
20669            (
20670                RouteCtx {
20671                    connection_id: ConnectionId::new(connection),
20672                    egress: FrameSink::new(tx),
20673                },
20674                rx,
20675            )
20676        }
20677
20678        /// A daemon with a configured owner (prefrontal-core) registered on its
20679        /// own module connection, two routable targets (plexus, other), and
20680        /// launch nonces minted for the modules that open routes as carriers.
20681        struct Rig {
20682            handler: ControlHandler,
20683            forwarding: Arc<ForwardingTable>,
20684            owner: RouteCtx,
20685            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20686            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
20687            generation: u64,
20688            next_connection: u64,
20689            _supervisor: Supervisor,
20690        }
20691
20692        async fn rig() -> Rig {
20693            rig_with_flow_support(true).await
20694        }
20695
20696        async fn rig_with_flow_support(flow_support: bool) -> Rig {
20697            rig_with_scope_capabilities(flow_support, false).await
20698        }
20699
20700        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
20701            let mut provides = Vec::new();
20702            if flow_support {
20703                provides.push("flow-scopes/v1");
20704            }
20705            if run_support {
20706                provides.push("agent-run-scopes/v1");
20707            }
20708            rig_with_target_capabilities(&provides).await
20709        }
20710
20711        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
20712            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
20713            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
20714            // Decoder versions alone cannot admit scoped calls: only a capability
20715            // declaration promises the required run or flow behaviour.
20716            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
20717            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
20718            Frame::build(
20719                FrameType::Hello,
20720                control_flags(),
20721                0,
20722                0,
20723                connection,
20724                serde_json::to_vec(&body).unwrap(),
20725            )
20726            .unwrap()
20727        }
20728
20729        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
20730            let registry = Arc::new(Registry::default());
20731            let forwarding = Arc::new(ForwardingTable::default());
20732            let supervisor_handle = SupervisorHandle::new();
20733            let supervisor =
20734                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
20735                    .with_handle(supervisor_handle.clone());
20736            supervisor
20737                .supervise_configured(
20738                    ModuleSpec {
20739                        script: None,
20740                        carry_containment: false,
20741                        module_id: OWNER.to_string(),
20742                        program: PathBuf::from("/nonexistent/prefrontal-core"),
20743                        args: Vec::new(),
20744                        env: Vec::new(),
20745                        reserved: false,
20746                        reserved_prefixes: Vec::new(),
20747                        protocol: ModuleProtocol::Subc,
20748                        overlap: Default::default(),
20749                    },
20750                    false,
20751                )
20752                .unwrap();
20753            for module_id in [OWNER, AFT, BROCA, MAGIC] {
20754                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
20755            }
20756            let handler =
20757                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
20758                    .with_supervisor(supervisor_handle);
20759            let (owner, mut owner_rx) = wide_ctx(1);
20760            hello_via_sink(
20761                &handler,
20762                &owner,
20763                &mut owner_rx,
20764                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
20765            )
20766            .await;
20767            let mut modules = BTreeMap::new();
20768            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
20769                let (ctx, mut rx) = wide_ctx(connection);
20770                let hello = hello_with_capabilities(module_id, connection, provides);
20771                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
20772                modules.insert(module_id.to_string(), (ctx, rx));
20773            }
20774            Rig {
20775                handler,
20776                forwarding,
20777                owner,
20778                _owner_rx: owner_rx,
20779                modules,
20780                generation: 0,
20781                next_connection: 100,
20782                _supervisor: supervisor,
20783            }
20784        }
20785
20786        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
20787            ScopeCarrier::new(Principal::Reserved {
20788                module_id: module_id.to_string(),
20789            })
20790            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
20791        }
20792
20793        /// The scope most tests open under: aft carries to any module, broca
20794        /// only to plexus and other, and the owner delegates as agent-1.
20795        fn session(scope_epoch: u64) -> ScopeRecord {
20796            let mut record = head("s", scope_epoch);
20797            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
20798            record.attributes.agent_id = Some("agent-1".to_string());
20799            record.attributes.delegates = true;
20800            record
20801        }
20802
20803        impl Rig {
20804            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
20805                self.generation += 1;
20806                sync(&self.handler, &self.owner, self.generation, scopes)
20807                    .await
20808                    .expect("the owner's sync is accepted");
20809            }
20810
20811            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
20812                ScopeSelector::new(
20813                    Principal::Reserved {
20814                        module_id: OWNER.to_string(),
20815                    },
20816                    scope_ref.to_string(),
20817                    scope_epoch,
20818                )
20819            }
20820
20821            fn open_frame(
20822                &mut self,
20823                opener: Option<&str>,
20824                target: &str,
20825                scope: Option<ScopeSelector>,
20826            ) -> (
20827                RouteCtx,
20828                mpsc::Receiver<crate::router::OutboundFrame>,
20829                Frame,
20830            ) {
20831                self.next_connection += 1;
20832                let (ctx, rx) = wide_ctx(self.next_connection);
20833                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
20834                    busy_refusals: None,
20835                    target: RouteTarget::ToolProvider {
20836                        module_id: target.to_string(),
20837                    },
20838                    identity: BindIdentity::new("unit".to_string(), "session".to_string()),
20839                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
20840                        module_id: module_id.to_string(),
20841                        launch_nonce: nonce(module_id),
20842                    }),
20843                    consumer_capabilities: None,
20844                    role_versions: None,
20845                    admission_facts: None,
20846                    scope,
20847                })
20848                .unwrap();
20849                let frame = Frame::build(
20850                    FrameType::Request,
20851                    control_flags(),
20852                    0,
20853                    0,
20854                    self.next_connection,
20855                    body,
20856                )
20857                .unwrap();
20858                (ctx, rx, frame)
20859            }
20860
20861            /// Open and expect a refusal before anything is relayed.
20862            async fn refused(
20863                &mut self,
20864                opener: Option<&str>,
20865                target: &str,
20866                scope: Option<ScopeSelector>,
20867            ) -> String {
20868                self.refusal_body(opener, target, scope).await["code"]
20869                    .as_str()
20870                    .unwrap()
20871                    .to_string()
20872            }
20873
20874            async fn refusal_body(
20875                &mut self,
20876                opener: Option<&str>,
20877                target: &str,
20878                scope: Option<ScopeSelector>,
20879            ) -> Value {
20880                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
20881                let replies = tokio::time::timeout(
20882                    Duration::from_secs(2),
20883                    self.handler.handle_control_frame(&ctx, frame),
20884                )
20885                .await
20886                .expect("the open must be refused before waiting for a bind ack")
20887                .unwrap();
20888                assert_eq!(replies.len(), 1, "{replies:?}");
20889                assert_eq!(replies[0].header.ty, FrameType::Error);
20890                let (_, module_rx) = self.modules.get_mut(target).unwrap();
20891                assert!(
20892                    module_rx.try_recv().is_err(),
20893                    "a refused open relays nothing"
20894                );
20895                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
20896                parse_error(&replies[0])
20897            }
20898
20899            /// Start an open and return its task and the bind the target got.
20900            async fn relayed(
20901                &mut self,
20902                opener: Option<&str>,
20903                target: &str,
20904                scope: Option<ScopeSelector>,
20905            ) -> Relayed {
20906                self.relayed_with_legacy_root(opener, target, scope, None)
20907                    .await
20908            }
20909
20910            async fn relayed_with_legacy_root(
20911                &mut self,
20912                opener: Option<&str>,
20913                target: &str,
20914                scope: Option<ScopeSelector>,
20915                legacy_root: Option<&std::path::Path>,
20916            ) -> Relayed {
20917                let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
20918                if let Some(root) = legacy_root {
20919                    let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
20920                    body["identity"]["project_root"] = serde_json::json!(root);
20921                    frame.body = serde_json::to_vec(&body).unwrap();
20922                }
20923                let handler = self.handler.clone();
20924                let task_ctx = ctx.clone();
20925                let task = tokio::spawn(async move {
20926                    handler
20927                        .handle_control_frame(&task_ctx, frame)
20928                        .await
20929                        .unwrap()
20930                });
20931                let (_, module_rx) = self.modules.get_mut(target).unwrap();
20932                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
20933                    .await
20934                    .expect("the target receives the relayed route.bind")
20935                    .unwrap()
20936                    .frame;
20937                Relayed {
20938                    target: target.to_string(),
20939                    client: ctx,
20940                    client_rx: rx,
20941                    task,
20942                    bind,
20943                }
20944            }
20945
20946            async fn ack(&self, relayed: &Relayed) {
20947                let (module, _) = &self.modules[&relayed.target];
20948                self.handler
20949                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
20950                    .await
20951                    .unwrap();
20952            }
20953
20954            /// Open, ack and return the bound route.
20955            async fn bound(
20956                &mut self,
20957                opener: Option<&str>,
20958                target: &str,
20959                scope: Option<ScopeSelector>,
20960            ) -> Bound {
20961                let relayed = self.relayed(opener, target, scope).await;
20962                self.commit_relayed(relayed).await
20963            }
20964
20965            async fn bound_with_legacy_root(
20966                &mut self,
20967                opener: Option<&str>,
20968                target: &str,
20969                scope: Option<ScopeSelector>,
20970                root: &std::path::Path,
20971            ) -> Bound {
20972                let relayed = self
20973                    .relayed_with_legacy_root(opener, target, scope, Some(root))
20974                    .await;
20975                self.commit_relayed(relayed).await
20976            }
20977
20978            async fn commit_relayed(&self, relayed: Relayed) -> Bound {
20979                self.ack(&relayed).await;
20980                let Relayed {
20981                    target,
20982                    client,
20983                    mut client_rx,
20984                    task,
20985                    bind,
20986                } = relayed;
20987                assert!(
20988                    task.await.unwrap().is_empty(),
20989                    "the open is answered by commit"
20990                );
20991                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
20992                Bound {
20993                    target,
20994                    client,
20995                    client_rx,
20996                    channel,
20997                    epoch,
20998                    bind,
20999                }
21000            }
21001
21002            fn live(&self, route: &Bound) -> bool {
21003                matches!(
21004                    self.forwarding
21005                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21006                        .unwrap(),
21007                    DataRoute::Client(DataRouteState::Bound(_))
21008                )
21009            }
21010        }
21011
21012        struct Relayed {
21013            target: String,
21014            client: RouteCtx,
21015            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21016            task: tokio::task::JoinHandle<Vec<Frame>>,
21017            bind: Frame,
21018        }
21019
21020        struct Bound {
21021            target: String,
21022            client: RouteCtx,
21023            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21024            channel: u16,
21025            epoch: u32,
21026            bind: Frame,
21027        }
21028
21029        impl Bound {
21030            /// The reason of the `route.closed` this client was sent, after
21031            /// checking it also got a GOODBYE on exactly this route.
21032            fn closed_reason(&mut self) -> RouteCloseReason {
21033                let mut reason = None;
21034                let mut goodbye = false;
21035                while let Ok(outbound) = self.client_rx.try_recv() {
21036                    let frame = outbound.frame;
21037                    match frame.header.ty {
21038                        FrameType::Goodbye => {
21039                            assert_eq!(
21040                                (frame.header.channel, frame.header.epoch),
21041                                (self.channel, self.epoch)
21042                            );
21043                            goodbye = true;
21044                        }
21045                        FrameType::Push => {
21046                            let ClientControlPush::RouteClosed {
21047                                reason: r,
21048                                module_id,
21049                                ..
21050                            } = serde_json::from_slice(&frame.body).unwrap()
21051                            else {
21052                                panic!("unexpected push");
21053                            };
21054                            assert_eq!(module_id, self.target);
21055                            reason = Some(r);
21056                        }
21057                        other => panic!("unexpected frame {other:?}"),
21058                    }
21059                }
21060                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
21061                reason.expect("the client is told why the route closed")
21062            }
21063
21064            fn untouched(&mut self) -> bool {
21065                self.client_rx.try_recv().is_err()
21066            }
21067
21068            fn stamp(&self) -> Option<ScopeStamp> {
21069                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
21070                    ModuleControlRequest::RouteBind { scope, .. } => scope,
21071                    other => panic!("expected a route.bind, got {other:?}"),
21072                }
21073            }
21074        }
21075
21076        #[tokio::test]
21077        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
21078        ) {
21079            let mut rig = rig().await;
21080            let mut record = session(1);
21081            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
21082            record.child_owners = vec![Principal::Reserved {
21083                module_id: MAGIC.to_string(),
21084            }];
21085            rig.sync(vec![record]).await;
21086            let scope = || Some(rig_selector("s", Some(1)));
21087
21088            // Admitted: the owner, a bare carrier to any module, a targeted
21089            // carrier to its listed module.
21090            rig.bound(Some(OWNER), PLEXUS, scope()).await;
21091            rig.bound(Some(AFT), OTHER, scope()).await;
21092            rig.bound(Some(BROCA), PLEXUS, scope()).await;
21093
21094            // Refused scope_not_carrier: a targeted carrier to an unlisted
21095            // module, a module that is not listed at all (a child owner is not
21096            // a carrier), and a direct key-holder.
21097            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
21098                assert_eq!(
21099                    rig.refused(opener, target, scope()).await,
21100                    error_codes::SCOPE_NOT_CARRIER,
21101                    "{opener:?} -> {target}"
21102                );
21103            }
21104        }
21105
21106        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
21107            ScopeSelector::new(
21108                Principal::Reserved {
21109                    module_id: OWNER.to_string(),
21110                },
21111                scope_ref.to_string(),
21112                scope_epoch,
21113            )
21114        }
21115
21116        #[tokio::test]
21117        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
21118            let mut rig = rig().await;
21119            rig.sync(vec![session(1)]).await;
21120            for opener in [OWNER, AFT] {
21121                assert_eq!(
21122                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
21123                        .await,
21124                    error_codes::SCOPE_EPOCH_REQUIRED,
21125                    "{opener}"
21126                );
21127            }
21128        }
21129
21130        #[tokio::test]
21131        async fn admission_separates_not_synced_not_live_and_ended() {
21132            let mut rig = rig().await;
21133            // Before the configured owner's first sync: retryable.
21134            let code = rig
21135                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21136                .await;
21137            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
21138            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
21139
21140            // An owner that is not configured will never sync: terminal.
21141            let ghost = ScopeSelector::new(
21142                Principal::Reserved {
21143                    module_id: "ghost".to_string(),
21144                },
21145                "s".to_string(),
21146                Some(1),
21147            );
21148            assert_eq!(
21149                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
21150                error_codes::SCOPE_NOT_LIVE
21151            );
21152
21153            rig.sync(vec![session(2)]).await;
21154            assert_eq!(
21155                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
21156                    .await,
21157                error_codes::SCOPE_NOT_LIVE
21158            );
21159            for epoch in [1, 3] {
21160                assert_eq!(
21161                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
21162                        .await,
21163                    error_codes::SCOPE_ENDED,
21164                    "epoch {epoch}"
21165                );
21166            }
21167            // Control: the live epoch is admitted.
21168            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
21169                .await;
21170        }
21171
21172        #[tokio::test]
21173        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
21174            let mut rig = rig().await;
21175            rig.sync(vec![session(1)]).await;
21176            let route = rig
21177                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21178                .await;
21179            let stamp = route.stamp().expect("a scoped bind carries the stamp");
21180            assert_eq!(stamp.scope_ref, "s");
21181            assert_eq!(stamp.scope_epoch, 1);
21182            assert_eq!(stamp.kind, ScopeKind::Head);
21183            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
21184            assert!(stamp.attributes.delegates);
21185            assert!(stamp.owner_authorized);
21186            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21187            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
21188
21189            // broca owns a scope of its own on its own module connection; it is
21190            // not in scope_authority_owners, so its stamp is not authorized.
21191            let (broca, mut broca_rx) = wide_ctx(50);
21192            hello_via_sink(
21193                &rig.handler,
21194                &broca,
21195                &mut broca_rx,
21196                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
21197            )
21198            .await;
21199            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
21200                .await
21201                .unwrap();
21202            let own = ScopeSelector::new(
21203                Principal::Reserved {
21204                    module_id: BROCA.to_string(),
21205                },
21206                "b".to_string(),
21207                Some(1),
21208            );
21209            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
21210            assert!(!route.stamp().unwrap().owner_authorized);
21211        }
21212
21213        #[tokio::test]
21214        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
21215            let mut rig = rig().await;
21216            let mut record = head("s", 1);
21217            record.carriers = vec![carrier(AFT, None)];
21218            let flow_id = "Flow:run-7/step_2!~";
21219            record.attributes.flow_id = Some(flow_id.to_string());
21220            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
21221                .await
21222                .unwrap();
21223            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
21224                panic!("not a sync reply");
21225            };
21226            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
21227            let route = rig
21228                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21229                .await;
21230            assert!(rig.live(&route), "the stamped bind committed");
21231            let stamp = route.stamp().expect("a flow scope carries a stamp");
21232            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
21233            assert_eq!(stamp.attributes.agent_id, None);
21234            assert!(!stamp.attributes.delegates);
21235            assert!(stamp.owner_authorized);
21236            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21237            assert_eq!(unscoped.stamp(), None);
21238        }
21239
21240        #[tokio::test]
21241        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
21242            let mut rig = rig_with_flow_support(false).await;
21243            let mut record = session(1);
21244            record.attributes.flow_id = Some("flow:7".to_string());
21245            rig.sync(vec![record]).await;
21246            for opener in [OWNER, AFT] {
21247                let body = rig
21248                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21249                    .await;
21250                assert_eq!(body["code"], "target_flow_unsupported");
21251                let message = body["message"].as_str().unwrap();
21252                for required in [PLEXUS, "flow-scopes/v1"] {
21253                    assert!(message.contains(required), "{message}");
21254                }
21255            }
21256        }
21257
21258        #[tokio::test]
21259        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
21260            let mut rig = rig_with_flow_support(true).await;
21261            let mut record = session(1);
21262            record.attributes.flow_id = Some("flow:7".to_string());
21263            rig.sync(vec![record]).await;
21264            for opener in [OWNER, AFT] {
21265                let route = rig
21266                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21267                    .await;
21268                assert!(rig.live(&route));
21269                assert_eq!(
21270                    route.stamp().unwrap().attributes.flow_id.as_deref(),
21271                    Some("flow:7")
21272                );
21273            }
21274        }
21275
21276        #[tokio::test]
21277        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
21278            use std::future::Future;
21279
21280            let mut rig = rig_with_flow_support(true).await;
21281            let mut record = session(1);
21282            record.attributes.flow_id = Some("flow:7".to_string());
21283            rig.sync(vec![record]).await;
21284            let (client, mut client_rx, frame) =
21285                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
21286            // Hold the route-open response permit so admission sees the first
21287            // target but relay reservation cannot capture an endpoint yet.
21288            for _ in 0..64 {
21289                client.egress.try_send(route_bind_ack(1)).unwrap();
21290            }
21291            let handler = rig.handler.clone();
21292            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21293            std::future::poll_fn(|cx| {
21294                assert!(open.as_mut().poll(cx).is_pending());
21295                std::task::Poll::Ready(())
21296            })
21297            .await;
21298            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21299
21300            let old_connection = rig.modules[PLEXUS].0.connection_id;
21301            rig.handler.cleanup_connection(old_connection).unwrap();
21302            let (replacement, mut replacement_rx) = wide_ctx(200);
21303            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
21304            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
21305            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
21306            let hello = Frame::build(
21307                FrameType::Hello,
21308                control_flags(),
21309                0,
21310                0,
21311                200,
21312                serde_json::to_vec(&body).unwrap(),
21313            )
21314            .unwrap();
21315            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
21316
21317            client_rx.try_recv().unwrap();
21318            let replies = tokio::time::timeout(Duration::from_secs(2), open)
21319                .await
21320                .expect("the replacement is refused without waiting for a bind ack")
21321                .unwrap();
21322            assert_eq!(replies.len(), 1);
21323            let body = parse_error(&replies[0]);
21324            assert_eq!(body["code"], "target_flow_unsupported");
21325            for required in [PLEXUS, "flow-scopes/v1"] {
21326                assert!(body["message"].as_str().unwrap().contains(required));
21327            }
21328            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
21329            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21330            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21331            assert!(rig
21332                .handler
21333                .registry
21334                .get_module_by_connection(replacement.connection_id)
21335                .unwrap()
21336                .is_some());
21337        }
21338
21339        #[tokio::test]
21340        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
21341            use std::future::Future;
21342            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
21343            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21344            const NEXT_HASH: &str =
21345                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21346            let mut rig = rig().await;
21347            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
21348            record.carriers.push(ScopeCarrier::new(Principal::Direct));
21349            rig.sync(vec![record.clone()]).await;
21350            let (client, mut rx, frame) = rig.open_frame(
21351                None,
21352                PLEXUS,
21353                Some(
21354                    rig.selector("s", Some(1))
21355                        .with_bind_proof(Some(PROOF.into())),
21356                ),
21357            );
21358            // Hold egress reservation while the scope owner changes its bind-proof
21359            // hash, so a proof valid at admission must be checked again at relay.
21360            for _ in 0..64 {
21361                client.egress.try_send(route_bind_ack(1)).unwrap();
21362            }
21363            let handler = rig.handler.clone();
21364            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21365            std::future::poll_fn(|cx| {
21366                assert!(open.as_mut().poll(cx).is_pending());
21367                std::task::Poll::Ready(())
21368            })
21369            .await;
21370            record.bind_proof_sha256 = Some(NEXT_HASH.into());
21371            rig.sync(vec![record]).await;
21372            rx.try_recv().unwrap();
21373            let replies = tokio::time::timeout(Duration::from_secs(2), open)
21374                .await
21375                .unwrap()
21376                .unwrap();
21377            assert_eq!(
21378                parse_error(&replies[0])["code"],
21379                error_codes::SCOPE_BIND_PROOF_MISMATCH
21380            );
21381            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
21382            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21383            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21384        }
21385
21386        #[tokio::test]
21387        async fn scope_bind_proof_reserved_openers_ignore_proof() {
21388            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21389            let mut rig = rig().await;
21390            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21391                .await;
21392            for opener in [OWNER, AFT] {
21393                for proof in [None, Some("malformed+proof=")] {
21394                    let route = rig
21395                        .bound(
21396                            Some(opener),
21397                            PLEXUS,
21398                            Some(
21399                                rig.selector("s", Some(1))
21400                                    .with_bind_proof(proof.map(str::to_string)),
21401                            ),
21402                        )
21403                        .await;
21404                    assert!(rig.live(&route));
21405                }
21406            }
21407        }
21408
21409        #[tokio::test]
21410        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
21411            use std::future::Future;
21412            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21413            const NEXT_HASH: &str =
21414                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21415            let mut rig = rig().await;
21416            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21417                .await;
21418            let (client, mut rx, frame) =
21419                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
21420            for _ in 0..64 {
21421                client.egress.try_send(route_bind_ack(1)).unwrap();
21422            }
21423            let handler = rig.handler.clone();
21424            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21425            std::future::poll_fn(|cx| {
21426                assert!(open.as_mut().poll(cx).is_pending());
21427                std::task::Poll::Ready(())
21428            })
21429            .await;
21430            rig.sync(vec![
21431                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
21432            ])
21433            .await;
21434            rx.try_recv().unwrap();
21435            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21436            let bind = std::future::poll_fn(|cx| {
21437                assert!(open.as_mut().poll(cx).is_pending());
21438                module_rx.poll_recv(cx)
21439            })
21440            .await
21441            .unwrap()
21442            .frame;
21443            handler
21444                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
21445                .await
21446                .unwrap();
21447            let replies = tokio::time::timeout(Duration::from_secs(2), open)
21448                .await
21449                .unwrap()
21450                .unwrap();
21451            assert_eq!(replies.len(), 1);
21452            // Proof rechecking allowed the relay. Independent tag settlement
21453            // still asks the opener to retry after any same-epoch content change.
21454            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
21455            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21456            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
21457            let route = rig
21458                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
21459                .await;
21460            assert!(rig.live(&route));
21461        }
21462
21463        #[tokio::test]
21464        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
21465        {
21466            let mut rig = rig_with_flow_support(false).await;
21467            rig.sync(vec![session(1)]).await;
21468            let route = rig
21469                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21470                .await;
21471            assert!(rig.live(&route));
21472            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
21473            let mut record = session(1);
21474            record.attributes.flow_id = Some("flow:7".to_string());
21475            rig.sync(vec![record]).await;
21476            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
21477            assert!(rig.live(&unscoped));
21478            assert_eq!(unscoped.stamp(), None);
21479        }
21480
21481        #[tokio::test]
21482        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
21483            let mut rig = rig().await;
21484            let mut record = session(1);
21485            record.attributes.flow_id = Some("flow:7".to_string());
21486            rig.sync(vec![record.clone()]).await;
21487            let before = rig.forwarding.published_scope_tag(OWNER, "s");
21488            let mut owner_route = rig
21489                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21490                .await;
21491            let mut carrier_route = rig
21492                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
21493                .await;
21494            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21495            rig.sync(vec![record.clone()]).await;
21496            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21497            assert!(rig.live(&owner_route) && owner_route.untouched());
21498            assert!(rig.live(&carrier_route) && carrier_route.untouched());
21499
21500            record.attributes.flow_id = Some("flow:8".to_string());
21501            rig.sync(vec![record]).await;
21502            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
21503            let before = before.unwrap();
21504            assert_eq!(after.scope_epoch, before.scope_epoch);
21505            assert!(after.version > before.version);
21506            for route in [&mut owner_route, &mut carrier_route] {
21507                assert!(!rig.live(route));
21508                assert_eq!(
21509                    route.closed_reason(),
21510                    RouteCloseReason::ScopeDelegationChanged
21511                );
21512            }
21513            assert!(rig.live(&unscoped) && unscoped.untouched());
21514            // Each provider also receives a GOODBYE for its drained route;
21515            // consume it before expecting the next route.bind on that sink.
21516            for target in [PLEXUS, OTHER] {
21517                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
21518                let goodbye = module_rx
21519                    .try_recv()
21520                    .expect("the provider sees the drain")
21521                    .frame;
21522                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21523                assert!(module_rx.try_recv().is_err());
21524            }
21525            let rebound = rig
21526                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21527                .await;
21528            assert_eq!(
21529                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
21530                Some("flow:8")
21531            );
21532        }
21533
21534        /// The owner's sync lands between admission and the module's ack. The
21535        /// open is refused by name, the module's other routes stay up, and the
21536        /// reserved pair is released. Changed content is retryable; an ended
21537        /// scope is not.
21538        #[tokio::test]
21539        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
21540            let mut rig = rig().await;
21541            rig.sync(vec![session(1)]).await;
21542            let mut cotenant = rig
21543                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21544                .await;
21545
21546            let mut changed = session(1);
21547            changed.child_owners.push(Principal::Reserved {
21548                module_id: MAGIC.to_string(),
21549            });
21550            let mut ended = None;
21551            for (code, next) in [
21552                (error_codes::SCOPE_CHANGED, vec![changed]),
21553                (error_codes::SCOPE_ENDED, Vec::new()),
21554            ] {
21555                let relayed = rig
21556                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21557                    .await;
21558                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
21559                ended = Some(next.is_empty());
21560                rig.sync(next).await;
21561                rig.ack(&relayed).await;
21562                let replies = relayed.task.await.unwrap();
21563                assert_eq!(replies.len(), 1, "{replies:?}");
21564                assert_eq!(parse_error(&replies[0])["code"], code);
21565                assert_eq!(
21566                    subc_protocol::error_codes::is_retryable_route_open(code),
21567                    code == error_codes::SCOPE_CHANGED
21568                );
21569                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21570                // The module is told to drop just the binding it created.
21571                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21572                // Collected, because ending the scope also closes the co-tenant
21573                // route, whose GOODBYE comes first.
21574                let mut goodbyes = Vec::new();
21575                while let Ok(outbound) = plexus_rx.try_recv() {
21576                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
21577                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
21578                }
21579                assert!(
21580                    goodbyes.contains(&(bind_channel, bind_epoch)),
21581                    "{goodbyes:?}"
21582                );
21583                assert!(rig
21584                    .handler
21585                    .registry
21586                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
21587                    .unwrap()
21588                    .is_some());
21589            }
21590            assert_eq!(ended, Some(true));
21591            // The co-tenant stayed up through the change, and closed only when
21592            // the scope ended, by the drain rule rather than by the commit.
21593            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
21594        }
21595
21596        /// Each row of the drain table on one set of routes: the owner's, a
21597        /// bare carrier's, and a targeted carrier's to each of its targets.
21598        #[tokio::test]
21599        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
21600            struct Case {
21601                name: &'static str,
21602                change: fn(&mut ScopeRecord),
21603                /// Closed routes by index: owner->plexus, aft->plexus,
21604                /// broca->plexus, broca->other.
21605                closed: [Option<RouteCloseReason>; 4],
21606            }
21607            use RouteCloseReason::*;
21608            let cases = [
21609                Case {
21610                    name: "a carrier entry removed",
21611                    change: |r| {
21612                        r.carriers.retain(|c| {
21613                            c.principal
21614                                != Principal::Reserved {
21615                                    module_id: AFT.to_string(),
21616                                }
21617                        })
21618                    },
21619                    closed: [None, Some(ScopeCarrierRemoved), None, None],
21620                },
21621                Case {
21622                    name: "a target removed from a carrier",
21623                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
21624                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
21625                },
21626                Case {
21627                    name: "a bare carrier narrowed to targets",
21628                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
21629                    closed: [None, Some(ScopeCarrierRemoved), None, None],
21630                },
21631                Case {
21632                    name: "delegates turned off",
21633                    change: |r| r.attributes.delegates = false,
21634                    closed: [Some(ScopeDelegationChanged); 4],
21635                },
21636                Case {
21637                    name: "agent_id changed",
21638                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
21639                    closed: [Some(ScopeDelegationChanged); 4],
21640                },
21641                Case {
21642                    name: "a carrier added, child owners changed, the record re-sent",
21643                    change: |r| {
21644                        r.carriers.push(carrier(MAGIC, None));
21645                        r.child_owners.push(Principal::Reserved {
21646                            module_id: MAGIC.to_string(),
21647                        });
21648                    },
21649                    closed: [None; 4],
21650                },
21651                Case {
21652                    name: "a target added",
21653                    change: |r| {
21654                        r.carriers[1]
21655                            .targets
21656                            .as_mut()
21657                            .unwrap()
21658                            .push("third".to_string())
21659                    },
21660                    closed: [None; 4],
21661                },
21662                Case {
21663                    name: "delegates turned on",
21664                    change: |r| r.attributes.delegates = true,
21665                    closed: [None; 4],
21666                },
21667            ];
21668            for case in cases {
21669                let mut rig = rig().await;
21670                rig.sync(vec![session(1)]).await;
21671                let scope = || Some(rig_selector("s", Some(1)));
21672                let mut routes = [
21673                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
21674                    rig.bound(Some(AFT), PLEXUS, scope()).await,
21675                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
21676                    rig.bound(Some(BROCA), OTHER, scope()).await,
21677                ];
21678                let mut record = session(1);
21679                (case.change)(&mut record);
21680                rig.sync(vec![record]).await;
21681                for (index, expected) in case.closed.iter().enumerate() {
21682                    let route = &mut routes[index];
21683                    match expected {
21684                        Some(reason) => {
21685                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
21686                            assert_eq!(
21687                                route.closed_reason(),
21688                                *reason,
21689                                "{}: route {index}",
21690                                case.name
21691                            );
21692                        }
21693                        None => {
21694                            assert!(rig.live(route), "{}: route {index} closed", case.name);
21695                            assert!(
21696                                route.untouched(),
21697                                "{}: route {index} was told something",
21698                                case.name
21699                            );
21700                        }
21701                    }
21702                }
21703            }
21704        }
21705
21706        #[tokio::test]
21707        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
21708            // Removed, and replaced by a higher epoch.
21709            for next in [Vec::new(), vec![session(2)]] {
21710                let mut rig = rig().await;
21711                rig.sync(vec![session(1)]).await;
21712                let mut route = rig
21713                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21714                    .await;
21715                rig.sync(next).await;
21716                assert!(!rig.live(&route));
21717                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
21718            }
21719
21720            // A child whose parent ends: its routes close as parent-ended, the
21721            // child stays live, and routes under the parent close as ended.
21722            let mut rig = rig().await;
21723            let mut child = session(1);
21724            child.scope_ref = "child".to_string();
21725            child.kind = ScopeKind::Worker;
21726            child.parent = Some(ScopeParent::new(
21727                Principal::Reserved {
21728                    module_id: OWNER.to_string(),
21729                },
21730                "s".to_string(),
21731                1,
21732            ));
21733            rig.sync(vec![session(1), child.clone()]).await;
21734            let mut child_route = rig
21735                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
21736                .await;
21737            assert_eq!(
21738                child_route.stamp().unwrap().parent_state,
21739                Some(ParentState::Linked)
21740            );
21741            rig.sync(vec![child]).await;
21742            assert!(!rig.live(&child_route));
21743            assert_eq!(
21744                child_route.closed_reason(),
21745                RouteCloseReason::ScopeParentEnded
21746            );
21747        }
21748
21749        #[tokio::test]
21750        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
21751        ) {
21752            let mut rig = rig().await;
21753            rig.sync(vec![session(1)]).await;
21754            let mut route = rig
21755                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21756                .await;
21757            let before = rig.forwarding.published_scope_tag(OWNER, "s");
21758            rig.sync(vec![session(1)]).await;
21759            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21760            assert!(rig.live(&route) && route.untouched());
21761
21762            // A call in flight on the route when another carrier is added. A
21763            // forwarded REQUEST holds one credit on the route's flow until the
21764            // module answers; the router takes it exactly like this.
21765            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
21766                .forwarding
21767                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21768                .unwrap()
21769            else {
21770                panic!("the route is bound");
21771            };
21772            binding.flow.acquire_tagged(9, false).await.unwrap();
21773            let mut widened = session(1);
21774            widened.carriers.push(carrier(MAGIC, None));
21775            rig.sync(vec![widened]).await;
21776            assert!(rig.live(&route) && route.untouched());
21777            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21778            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
21779            // The call's credit is still held on an open flow, so its answer
21780            // will be delivered: closing the route would have closed the flow.
21781            assert_eq!(binding.flow.in_flight(), 1);
21782            binding
21783                .flow
21784                .acquire_tagged(10, false)
21785                .await
21786                .expect("the flow is still open");
21787        }
21788
21789        /// A swap's superseded endpoint keeps its routes until drained; ending
21790        /// the scope closes them there too.
21791        #[tokio::test]
21792        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
21793            let mut rig = rig().await;
21794            rig.sync(vec![session(1)]).await;
21795            let mut on_incumbent = rig
21796                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21797                .await;
21798
21799            // Swap plexus: register a candidate and cut over, leaving the
21800            // incumbent superseded with the route still on it.
21801            let (candidate, _candidate_rx) = wide_ctx(9);
21802            let registration = rig
21803                .handler
21804                .registry
21805                .register_candidate_with_control_ops(
21806                    manifest(PLEXUS, PROTOCOL_VERSION),
21807                    PROTOCOL_VERSION,
21808                    candidate.connection_id,
21809                    module_baseline_control_ops(),
21810                )
21811                .unwrap();
21812            rig.forwarding
21813                .register_candidate_module_connection(
21814                    candidate.connection_id,
21815                    PLEXUS.to_string(),
21816                    PROTOCOL_VERSION,
21817                    manifest_concurrency(&registration.manifest),
21818                    candidate.egress.clone(),
21819                )
21820                .unwrap();
21821            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
21822            rig.handler
21823                .registry
21824                .promote_candidate(PLEXUS)
21825                .unwrap()
21826                .unwrap();
21827            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
21828
21829            rig.sync(Vec::new()).await;
21830            assert!(!rig.live(&on_incumbent));
21831            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
21832            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21833            let goodbye = incumbent_rx
21834                .try_recv()
21835                .expect("the superseded endpoint is told")
21836                .frame;
21837            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21838        }
21839    }
21840}
21841
21842#[cfg(test)]
21843mod concurrency_default_exposure_tests {
21844    use super::*;
21845
21846    fn hello_body(role_json: &str) -> Vec<u8> {
21847        format!(
21848            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
21849        )
21850        .into_bytes()
21851    }
21852
21853    fn manifest_from(body: &[u8]) -> ModuleManifest {
21854        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
21855        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
21856            .expect("manifest parses")
21857    }
21858
21859    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
21860
21861    #[test]
21862    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
21863        let body = hello_body(&format!(
21864            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
21865        ));
21866        let manifest = manifest_from(&body);
21867        // Precondition: serde really resolved it to the default, so the typed
21868        // manifest alone cannot answer the question this probe exists for.
21869        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21870        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
21871    }
21872
21873    #[test]
21874    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
21875        let body = hello_body(&format!(
21876            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
21877        ));
21878        let manifest = manifest_from(&body);
21879        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21880        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21881    }
21882
21883    #[test]
21884    fn non_management_roles_are_never_reported() {
21885        let body = hello_body(
21886            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
21887        );
21888        let manifest = manifest_from(&body);
21889        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21890    }
21891}