Skip to main content

subc_daemon/
control.rs

1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6    fmt,
7    net::SocketAddr,
8    path::{Path, PathBuf},
9    sync::{Arc, Mutex, RwLock},
10    time::{Duration, Instant as StdInstant},
11};
12
13use crate::scopes::admission::denied_capability;
14use serde::{Deserialize, Serialize};
15use subc_control::{
16    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
17    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
18    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
19    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
20    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
21    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
22    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
23};
24use subc_protocol::{
25    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
26    error_codes,
27    manifest::{
28        validate_hello_capability_grammar, validate_hello_confirm_operations,
29        validate_hello_display_name, validate_hello_event_declarations,
30        validate_hello_flow_operations, validate_hello_operation_deferral,
31        validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
32        validate_hello_self_signal_declarations, validate_role_declarations_value,
33        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
34        ProviderRole,
35    },
36    peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
37    provider_declarations::ProviderDeclarations,
38    scope::{
39        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
40        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
41        SCOPE_SYNC_OP,
42    },
43    session::{
44        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
45        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
46        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
47        MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS, ROLE_VERSIONS_FIELD,
48    },
49    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
50    Priority, RouteTarget, PROTOCOL_VERSION,
51};
52use tokio::time::{timeout_at, Instant};
53use tracing::{debug, info, instrument::WithSubscriber, warn};
54
55use crate::{
56    caller_identity::CallerIdentity,
57    capability_requirements::{
58        log_duplicate_claim_events, log_manifest_change, log_requirement_events,
59        CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
60        RequirementStatus, RuntimeModule,
61    },
62    child_roster::DaemonShutdownFlag,
63    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
64    forwarding::{
65        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
66        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
67        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
68    },
69    observability::{
70        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
71    },
72    privileged_prompt::{prompt_text, PrivilegedOperation},
73    provenance::{
74        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
75    },
76    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
77    router::{daemon_error_flags, RouteCtx, RouterError},
78    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
79    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
80    stderr_tail::{CaptureState, TailEntry},
81    supervise::{
82        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
83        SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
84    },
85    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
86};
87
88/// Lowest envelope version this subc build will negotiate.
89///
90/// Module HELLO negotiation is exact: peers must use the daemon's locked
91/// protocol version. Older and newer peers receive `version_unsupported` and
92/// are not registered.
93pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
94
95const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
96const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
97const CAP_PING_PONG: &str = "ping_pong_v1";
98const CAP_SESSION_ATTACH: &str = "session_attach_v1";
99const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
100
101const SUBC_CONTROL_OPS: &[&str] = &[
102    "claim.mint",
103    "claim.present",
104    ops::DAEMON_SHUTDOWN_INTENT,
105    ops::ACTIVITY_RECORD,
106    ops::ACTIVITY_QUERY,
107    ops::ACTIVITY_SUBSCRIBE,
108    ops::SERVER_DESCRIBE,
109    ops::CATALOG_LIST,
110    ops::CATALOG_WATCH,
111    ops::ROUTE_OPEN,
112    ops::ROUTE_POLL,
113    ops::ROUTE_CLOSING,
114    ops::ROUTE_CLOSED,
115    ops::SUPERVISOR_LIST,
116    ops::SUPERVISOR_RESTART,
117    ops::SUPERVISOR_SWAP,
118    ops::SUPERVISOR_RELOAD,
119    ops::SUPERVISOR_RESCAN,
120    ops::SUPERVISOR_RELEASE_RESERVED,
121    ops::SUPERVISOR_SET_ENABLED,
122    ops::SUPERVISOR_HEALTH_PROBE,
123    ops::SUPERVISOR_HEALTH,
124    ops::SUPERVISOR_STDERR_TAIL,
125    ops::SUPERVISOR_TERMINALS,
126    ops::SUPERVISOR_ROUTES,
127    ops::SUPERVISOR_PROVENANCE,
128    ops::SUPERVISOR_SPAWN_SNAPSHOT,
129    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
130];
131
132/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
133/// announced by a capability in `server.describe` instead, and clients check
134/// that capability before sending it. `ck daemon --json` prints `subc_ops`
135/// verbatim and scripts read that output, so a new name there would change it.
136/// A malformed body for one of these is still refused `invalid_control_body`
137/// rather than as an unknown op.
138const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[
139    ops::SUPERVISOR_RESCAN_CHECKED,
140    "config.status",
141    "config.plan",
142    "config.apply",
143    "config.stamp",
144    "config.unstamp",
145    "config.history",
146    "config.rollback",
147];
148
149mod caller_attestation;
150mod channel_operations;
151#[cfg(all(test, unix, feature = "test-support"))]
152#[path = "control/channel_operations_tests.rs"]
153mod channel_operations_tests;
154pub(crate) mod claims;
155#[cfg(test)]
156#[path = "control/config_advertisement_tests.rs"]
157mod config_advertisement_tests;
158mod config_gate;
159use caller_attestation::{caller_for_target, scope_acts_for_another};
160#[cfg(all(test, unix, feature = "test-support"))]
161#[path = "control/caller_attestation_tests.rs"]
162mod caller_attestation_tests;
163mod scope_expiry;
164
165mod agent_commands;
166pub(crate) mod catalog;
167mod rescan_checked;
168use rescan_checked::{RescanMode, RescanPlan};
169
170const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
171    "claim.mint",
172    "claim.present",
173    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
174    MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS,
175    "supervisor.live_roots",
176    "supervisor.agent_commands",
177    SCOPE_SYNC_OP,
178    SCOPE_APPLY_OP,
179    SCOPE_DESCRIBE_OP,
180    subc_protocol::scope::ROUTE_AUTHORIZE_OP,
181    "operator.confirm",
182];
183
184/// Module-originated ops the daemon answers but does not advertise in
185/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
186/// belongs to is incomplete, so no module is told it works before it does.
187const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
188
189const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
190
191/// How long subc waits for a module to ack a relayed route.bind before returning
192/// `module_timeout`. The ack waits on the module's own configure, which for AFT
193/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
194/// and DB-open work — on a cold page cache or a large repo that legitimately
195/// exceeds a couple of seconds. The default is generous because rejecting a VALID
196/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
197/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
198pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
199
200/// How many CONSECUTIVE full-budget relay timeouts against one target module
201/// open that module's bind-relay breaker.
202///
203/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
204/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
205/// so three consecutive full-budget timeouts take ~36s to observe: every client
206/// whose open contributed to opening the breaker had already given up on its
207/// own. That is what makes opening the breaker unable to turn a call that would
208/// have succeeded into a refusal — it can only make an already-failing module
209/// fail faster.
210///
211/// Two would be reachable inside one default deadline. One would convict a
212/// module on a single cold-cache bind, which is exactly the valid-but-slow case
213/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
214pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
215
216/// How long a module's bind-relay breaker stays open before exactly one
217/// `route.open` is let through as a probe.
218///
219/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
220/// re-pays a full-budget stall almost continuously, and the breaker stops being
221/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
222/// deadline: a client that starts retrying after the module recovers has to get
223/// a probe opportunity inside its own deadline, or the breaker converts a
224/// recovered module into a failed call — the failure it exists to prevent,
225/// pointed the other way.
226///
227/// 20s sits between those with room on both sides, and it caps what a wedged
228/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
229/// rather than one per `route.open` per connection. The stall that motivated
230/// this, with its measurements, is written up in
231/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
232/// waited the whole budget out.
233pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
234
235const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
236const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
237
238fn reload_verdict(
239    configured: &Path,
240    spawned_from: Option<&Path>,
241    image: subc_control::RunningImageAgreement,
242) -> PendingReloadVerdict {
243    let path = match spawned_from {
244        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
245            ReloadPathAgreement::Match
246        }
247        Some(spawned_from) => ReloadPathAgreement::Mismatch {
248            configured: configured.to_path_buf(),
249            spawned_from: spawned_from.to_path_buf(),
250        },
251        None => ReloadPathAgreement::Unavailable {
252            reason: if matches!(
253                image,
254                subc_control::RunningImageAgreement::Unavailable {
255                    reason: subc_control::RunningImageUnavailableReason::NotRunning
256                }
257            ) {
258                ReloadPathUnavailableReason::NotRunning
259            } else {
260                ReloadPathUnavailableReason::SpawnedPathUnavailable
261            },
262        },
263    };
264    PendingReloadVerdict::new(path, image)
265}
266
267fn launch_spec_agreement(
268    configured: &crate::supervise::ModuleSpec,
269    running: bool,
270    spawned: Option<&crate::supervise::LaunchSpecDigest>,
271) -> subc_control::LaunchSpecAgreement {
272    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
273    if !running {
274        return Agreement::Unavailable {
275            reason: Reason::NotRunning,
276        };
277    }
278    match spawned {
279        None => Agreement::Unavailable {
280            reason: Reason::SpawnedSpecUnavailable,
281        },
282        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
283            Agreement::Match
284        }
285        Some(_) => Agreement::Mismatch,
286    }
287}
288
289fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
290    if configured == spawned_from {
291        return true;
292    }
293    #[cfg(windows)]
294    {
295        // Windows spawn records the resolved verbatim path (\\?\...), while
296        // configuration keeps its original spelling. Resolve both with the OS
297        // instead of stripping prefixes or folding case, which could conflate
298        // distinct names in case-sensitive directories. Different resolved
299        // paths still differ even if their files have identical content.
300        match (
301            std::fs::canonicalize(configured),
302            std::fs::canonicalize(spawned_from),
303        ) {
304            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
305            _ => false,
306        }
307    }
308    #[cfg(not(windows))]
309    {
310        false
311    }
312}
313
314#[derive(Clone)]
315struct DaemonProvenanceFacts {
316    build: DaemonBuildProvenance,
317    pid: Option<u32>,
318    started_at_ms: Option<u64>,
319    start_clock: Option<crate::clock::StartClock>,
320    executable_path: Option<PathBuf>,
321    executable_identity: Option<SpawnedFileIdentity>,
322    process_start_time: Option<u64>,
323    probe: ExecutableIdentityProbe,
324}
325
326impl Default for DaemonProvenanceFacts {
327    fn default() -> Self {
328        Self {
329            build: DaemonBuildProvenance {
330                build_git_sha: None,
331                build_lock_digest: None,
332            },
333            pid: None,
334            started_at_ms: None,
335            start_clock: None,
336            executable_path: None,
337            executable_identity: None,
338            process_start_time: None,
339            probe: ExecutableIdentityProbe::default(),
340        }
341    }
342}
343
344#[derive(Debug, Clone)]
345struct SupervisorRescanContext {
346    supervisor: Supervisor,
347    config_path: PathBuf,
348    configured_port: Option<u16>,
349    storage_config: Option<crate::daemon_config::StorageConfig>,
350    admission_facts_carrier_module_id: Option<String>,
351    admission_facts_targets: Option<Vec<String>>,
352    scope_authority_owners: Vec<String>,
353    federated_scope_owner: String,
354    privileged_operator_prompt: PrivilegedOperatorPrompt,
355    /// Counts executed rescans that set out to change the supervised modules'
356    /// staged or applied configuration, counted before the first change so a
357    /// rescan that fails partway still counts. With the daemon incarnation it
358    /// names the state a `supervisor.rescan_checked` preview was computed
359    /// against.
360    generation: Arc<std::sync::atomic::AtomicU64>,
361}
362
363/// Refusal labels passed to `observe_route_open_refusal` that mean the target
364/// module is not serving right now, and so open or extend an outage in the
365/// route outage tracker. Every one of them is only reachable after the target
366/// was found in the registry, which is what keeps an arbitrary client-chosen
367/// id from ever creating tracker state.
368///
369/// Deliberately absent: `not_registered` and `removed` (the id may be
370/// anything a client sent, and a removed module is gone on purpose),
371/// `protocol_none` (such a module never serves routes, so nothing is out),
372/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
373/// capability and admission-facts refusals (they refuse the caller, not a
374/// module outage), and `relay_reservation_failed` (its code ranges over
375/// capacity limits as well as a vanished connection). Capacity, breaker,
376/// relay-timeout and module-rejection refusals do not pass through that
377/// function at all; the breaker logs its own transitions.
378///
379/// The two not-serving refusals that bypass that function record themselves
380/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
381/// `required_capability_unprovided` is not tracked: the module itself is up,
382/// and the outage belongs to the missing provider.
383const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
384    "reloading",
385    "supervisor_not_live",
386    "registration_not_active",
387    "no_forwarding_connection",
388    "relay_send_failed",
389];
390
391/// Observe test request cancellation relative to the connection's cleanup.
392#[cfg(test)]
393struct ActivityRequestPause {
394    connection: ConnectionId,
395    started: tokio::sync::Notify,
396    cancelled: tokio::sync::Notify,
397    cleanup_started: std::sync::atomic::AtomicBool,
398    cancelled_after_cleanup: std::sync::atomic::AtomicBool,
399}
400
401#[cfg(test)]
402struct ActivityRequestCancellation(Arc<ActivityRequestPause>);
403
404#[cfg(test)]
405impl Drop for ActivityRequestCancellation {
406    fn drop(&mut self) {
407        self.0.cancelled_after_cleanup.store(
408            self.0
409                .cleanup_started
410                .load(std::sync::atomic::Ordering::Acquire),
411            std::sync::atomic::Ordering::Release,
412        );
413        self.0.cancelled.notify_one();
414    }
415}
416
417/// A test hold: signal arrival, then wait for release on the holding thread.
418#[cfg(test)]
419type TestPause = (
420    tokio::sync::oneshot::Sender<()>,
421    std::sync::mpsc::Receiver<()>,
422);
423
424/// A test hook run once at a fixed point in a handler.
425#[cfg(test)]
426type TestHook = Box<dyn FnOnce() + Send>;
427
428/// A test hold for async code: signal arrival, then await release.
429#[cfg(test)]
430type AsyncTestPause = (
431    tokio::sync::oneshot::Sender<()>,
432    tokio::sync::oneshot::Receiver<()>,
433);
434
435/// Real channel-0 control handler for subc itself.
436#[derive(Clone)]
437pub struct ControlHandler {
438    #[cfg(test)]
439    activity_request_pause: Arc<Mutex<Option<Arc<ActivityRequestPause>>>>,
440    /// Holds connection cleanup at a numbered point: 0 is just after the agent
441    /// declaration queue closes, 1 just after the registration channel is
442    /// forgotten.
443    #[cfg(test)]
444    cleanup_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
445    /// Runs once in `route.open`'s launch-nonce attestation, right after the
446    /// nonce is resolved, to interleave a nonce withdrawal there.
447    #[cfg(test)]
448    after_nonce_attestation: Arc<Mutex<Option<TestHook>>>,
449    /// Holds a registration channel's HELLO after its reader resolved the
450    /// channel and before the blocking HELLO handler starts.
451    #[cfg(test)]
452    hello_ingress_pause: Arc<Mutex<Option<AsyncTestPause>>>,
453    /// Holds an agent command declaration at a numbered point in its handler.
454    #[cfg(test)]
455    agent_commands_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
456    claims: Arc<claims::Claims>,
457    registry: Arc<Registry>,
458    config_runtime: Option<Arc<crate::config_gate::runtime::RuntimeOwner>>,
459    config_review_admission: Arc<dyn crate::config_gate::ReviewAdmission>,
460    config_review_admission_is_default: bool,
461    activity: Option<crate::activity::ActivityLedger>,
462    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
463    catalog_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
464    forwarding: Arc<ForwardingTable>,
465    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
466    supervisor: SupervisorHandle,
467    pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
468    agent_declarations: agent_commands::Queues,
469    list_admission: Arc<tokio::sync::Semaphore>,
470    subc_capabilities: Arc<[String]>,
471    /// Daemon-wide route.bind relay budget. Used as the fallback when the
472    /// target module has no per-module override in
473    /// `route_bind_relay_timeouts`.
474    route_bind_relay_timeout: Duration,
475    /// Per-module route.bind relay budget overrides, keyed by module id. When
476    /// `handle_route_open` resolves the deadline for a target module, a
477    /// per-module entry wins over the daemon-wide value above.
478    route_bind_relay_timeouts: BTreeMap<String, Duration>,
479    /// Per-target-module bind-relay breaker state. Shared with the forwarding
480    /// table, which is where a new module connection resets it.
481    route_bind_breakers: RouteBindBreakers,
482    /// Live relay admissions keyed by target module. Shared through the
483    /// forwarding table so cloned or separately built handlers enforce one cap.
484    route_bind_concurrency: RouteBindConcurrency,
485    /// Start and end of each module's not-serving period as seen by
486    /// `route.open`, so an outage gets one line at each edge instead of only
487    /// the per-refusal INFO lines. Taken from the forwarding table, so every
488    /// handler built over one table shares it.
489    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
490    /// Consecutive relay timeouts that open a module's breaker.
491    route_bind_breaker_threshold: u32,
492    /// How long a breaker stays open before one probe is admitted.
493    route_bind_breaker_cooldown: Duration,
494    health_probe_timeout: Duration,
495    /// Central storage policy. When set, each registering module receives its
496    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
497    storage_config: Option<crate::daemon_config::StorageConfig>,
498    /// The machine id established at boot, served on every HELLO_ACK and on
499    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
500    /// changes the file, never this value. `None` serves no id.
501    machine_id: Option<crate::machine_id::MachineId>,
502    admission_facts_carrier_module_id: Option<String>,
503    admission_facts_targets: Option<Vec<String>>,
504    /// Scope records with their sync authorities and tombstones; see
505    /// `crate::scopes`. Shared by clones of this handler, so every connection
506    /// reads and writes one table.
507    scopes: Arc<RwLock<ScopeTable>>,
508    /// Wakes the expiry loop when an accepted scope change may have added a
509    /// deadline, so the loop can wait without ticking while no live scope has
510    /// one. A permit is stored when nothing waits, so no wake is lost.
511    scope_deadline_added: Arc<tokio::sync::Notify>,
512    /// The configured `scope_authority_owners`, kept so a rescan can report a
513    /// changed value as needing a daemon restart; rescan never applies it.
514    scope_authority_owners: Vec<String>,
515    federated_scope_owner: String,
516    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
517    privileged_operator_prompt: PrivilegedOperatorPrompt,
518    /// The local and peer address of each accepted TCP client connection, read
519    /// from the daemon's own socket, so a privileged request can name its caller.
520    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
521    build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
522    observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
523    captured_peers: Arc<Mutex<HashMap<ConnectionId, subc_os::local_peer::CapturedPeer>>>,
524    config_desktop_stamps:
525        Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
526    local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
527    pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
528    pub(crate) operator_apps:
529        Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
530    /// One launch entry per app, behind a leaf lock: no other lock is taken
531    /// while it is held. Entries are memory-only, so a daemon or
532    /// installed-service restart drops them and a relaunched daemon never
533    /// mistakes an older process for its own launch.
534    pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
535    /// Serializes `app.open` launches: the liveness check in the decision and
536    /// the later publish must not interleave across concurrent calls, or two
537    /// calls could both launch and the second publish would orphan the first
538    /// process. Refusals and `already_running` answers share the same lock;
539    /// `app.open` is rare, so the cost is negligible.
540    pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
541    // Portable wire samples exercise dispatch without starting an OS process.
542    #[cfg(test)]
543    pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
544    #[cfg(test)]
545    pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
546    #[cfg(test)]
547    recovery_exit_test_after_step: Option<fn(usize) -> Result<(), crate::config_gate::GateError>>,
548    #[cfg(test)]
549    recovery_exit_test_completed: Option<fn(u64)>,
550    #[cfg(test)]
551    apply_reconcile_pause: Arc<Mutex<Option<AsyncTestPause>>>,
552    #[cfg(test)]
553    recovery_exit_test_quiet_retry: bool,
554    #[cfg(test)]
555    recovery_exit_test_retry_pause: Option<fn(u64) -> Option<PathBuf>>,
556    #[cfg(test)]
557    config_delivery_test_wakes:
558        Option<tokio::sync::mpsc::UnboundedSender<(&'static str, tokio::time::Instant)>>,
559    #[cfg(all(test, unix))]
560    config_delivery_test_idle:
561        Option<tokio::sync::mpsc::UnboundedSender<tokio::sync::oneshot::Sender<()>>>,
562    /// Test-only Windows signature and flash hooks. Production builds carry
563    /// neither: with the publisher unset, the production table never reaches
564    /// verification, and the default attention hook only logs.
565    #[allow(dead_code)]
566    #[cfg(windows)]
567    pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
568    #[allow(dead_code)]
569    #[cfg(windows)]
570    pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
571    /// Test-only tracking fault hook: makes opening the tracking handle on
572    /// the launched pid fail, so a test can prove an untrackable launch is
573    /// stopped rather than left running.
574    #[allow(dead_code)]
575    #[cfg(windows)]
576    pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
577    /// Test-only count of Windows spawn attempts, so a test can prove a
578    /// refused launch started no process. Unit tests only.
579    #[cfg(all(test, windows))]
580    pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
581    #[cfg(test)]
582    caller_lookup: Option<TestCallerLookup>,
583    /// The launch nonce each module connection presented at HELLO, which is how
584    /// a `scope.sync` is matched to the owner's current launch.
585    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
586    rescan: Option<SupervisorRescanContext>,
587    connected_clients: ConnectedClients,
588    counters: DaemonCounters,
589    /// Why this daemon serves no local endpoint, when binding one failed. `None`
590    /// when the endpoint is served, and also on a handler that was never told.
591    local_endpoint_unavailable: Option<String>,
592    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
593    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
594    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
595    /// their usual severity (see [`log_requirement_events`]). A handler built
596    /// without one has a flag that is never set.
597    daemon_shutdown: DaemonShutdownFlag,
598    daemon_provenance: DaemonProvenanceFacts,
599    #[cfg(test)]
600    control_dispatch_delay: Option<Duration>,
601    #[cfg(test)]
602    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
603    /// Holds `route.open` after the route.bind is queued to the module and
604    /// before it is marked enqueued, so a test can make the module's reply (or
605    /// a client teardown) land exactly inside that window.
606    #[cfg(test)]
607    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
608    /// Holds a HELLO after its registry entry is reserved and before its
609    /// forwarding endpoint is installed, so a test can read the catalog and
610    /// open routes while the module is registered but not yet routable. A
611    /// blocking barrier because the HELLO handler is synchronous; the test
612    /// runs the HELLO on its own thread.
613    #[cfg(test)]
614    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
615    #[cfg(test)]
616    hello_ack_to_publication_hold: Option<Arc<std::sync::Barrier>>,
617    #[cfg(test)]
618    hello_before_publication_hold: Option<Arc<std::sync::Barrier>>,
619    #[cfg(test)]
620    hello_after_launch_check_hold: Option<Arc<std::sync::Barrier>>,
621    #[cfg(test)]
622    hello_inside_publication_hold: Option<Arc<std::sync::Barrier>>,
623}
624
625impl fmt::Debug for ControlHandler {
626    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
627        f.debug_struct("ControlHandler")
628            .field("registry", &self.registry)
629            .field("forwarding", &self.forwarding)
630            .field("process_liveness", &self.process_liveness.is_some())
631            .field("supervisor", &self.supervisor)
632            .field("subc_capabilities", &self.subc_capabilities)
633            .finish()
634    }
635}
636
637struct RouteOpenRequest {
638    target: RouteTarget,
639    identity: BindIdentity,
640    consumer_identity: Option<ConsumerIdentity>,
641    consumer_capabilities: Option<Vec<String>>,
642    role_versions: Option<BTreeMap<String, String>>,
643    busy_refusals: Option<bool>,
644    admission_facts: Option<serde_json::Value>,
645    scope: Option<ScopeSelector>,
646    accepts_close_reasons: Option<Vec<String>>,
647}
648
649#[derive(Clone, Copy)]
650enum RouteOpenPhase {
651    Admission,
652    ProjectRoot,
653    EgressReservation,
654    BindSend,
655    BindAckWait,
656}
657
658/// Store elapsed time for each route-open phase, including attempts that end
659/// in an early refusal or backend error. Admission also includes rechecks and
660/// commit bookkeeping performed between the other timed phases.
661struct RouteOpenTimings {
662    module_id: String,
663    principal: String,
664    admission: Duration,
665    project_root: Duration,
666    egress_reservation: Duration,
667    bind_send: Duration,
668    bind_ack_wait: Duration,
669    phase: Option<RouteOpenPhase>,
670    phase_started_at: Instant,
671}
672
673impl RouteOpenTimings {
674    fn new() -> Self {
675        Self {
676            module_id: String::new(),
677            // Refusals before attestation must not claim a verified principal.
678            principal: "unresolved".to_string(),
679            admission: Duration::ZERO,
680            project_root: Duration::ZERO,
681            egress_reservation: Duration::ZERO,
682            bind_send: Duration::ZERO,
683            bind_ack_wait: Duration::ZERO,
684            phase: Some(RouteOpenPhase::Admission),
685            phase_started_at: Instant::now(),
686        }
687    }
688
689    fn finish(&mut self) {
690        let elapsed = self.phase_started_at.elapsed();
691        match self.phase.take() {
692            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
693            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
694            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
695            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
696            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
697            None => {}
698        }
699    }
700
701    fn enter(&mut self, phase: RouteOpenPhase) {
702        self.finish();
703        self.phase = Some(phase);
704        self.phase_started_at = Instant::now();
705    }
706}
707
708struct RouteBindReservationGuard {
709    forwarding: Arc<ForwardingTable>,
710    endpoint: ModuleEndpointId,
711    relay_corr: u64,
712    armed: bool,
713}
714
715struct ModuleControlRpcGuard {
716    forwarding: Arc<ForwardingTable>,
717    endpoint: ModuleEndpointId,
718    corr: u64,
719    armed: bool,
720}
721
722impl ModuleControlRpcGuard {
723    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
724        Self {
725            forwarding,
726            endpoint,
727            corr,
728            armed: true,
729        }
730    }
731
732    fn disarm(&mut self) {
733        self.armed = false;
734    }
735}
736
737impl Drop for ModuleControlRpcGuard {
738    fn drop(&mut self) {
739        if self.armed {
740            let _ = self
741                .forwarding
742                .cancel_module_control_rpc(self.endpoint, self.corr);
743        }
744    }
745}
746
747impl RouteBindReservationGuard {
748    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
749        Self {
750            forwarding,
751            endpoint,
752            relay_corr,
753            armed: true,
754        }
755    }
756
757    fn release_and_disarm(&mut self) {
758        if !self.armed {
759            return;
760        }
761        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
762            self.endpoint,
763            self.relay_corr,
764            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
765        ) {
766            send_goodbye_target_best_effort(
767                &self.forwarding.counters(),
768                &target,
769                "canceled route.bind",
770            );
771        }
772        self.armed = false;
773    }
774
775    fn disarm(&mut self) {
776        self.armed = false;
777    }
778}
779
780impl Drop for RouteBindReservationGuard {
781    fn drop(&mut self) {
782        self.release_and_disarm();
783    }
784}
785
786/// Per-target-module circuit breaker around the `route.bind` relay.
787///
788/// The connection reader is serial per connection, so a module whose `on_bind`
789/// sits on the ack blocks every LATER frame on the connections that call it,
790/// including calls to unrelated modules. This does not make any module's bind
791/// fast; it stops the daemon paying the full budget again and again for a
792/// condition it has already observed.
793///
794/// State is keyed by TARGET MODULE and shared by every connection: a wedged
795/// module wedges everyone, so what one connection learned should protect the
796/// rest.
797///
798/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
799/// relay to that module has actually timed out, and is removed again when a
800/// relay is accepted or the module reconnects, so it cannot grow with traffic
801/// or with modules that behave.
802///
803/// # Why a `std` mutex here is not the head-of-line defect again
804///
805/// Acquisition never awaits. The critical section is a hash lookup plus a few
806/// integer updates, with no I/O and no `.await` inside it, so a reader task
807/// cannot be descheduled behind it the way it can behind
808/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
809/// primitive, held for the same kind of work, as the refusal counter this very
810/// path already increments.
811///
812/// It is also NOT on the data-plane splice path: only `route.open` and module
813/// registration touch it, so bound-route frames gain no state check and no
814/// contention.
815#[derive(Debug, Clone, Default)]
816pub(crate) struct RouteBindBreakers {
817    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
818}
819
820#[derive(Debug, Clone, Default)]
821pub(crate) struct RouteBindConcurrency {
822    modules: Arc<Mutex<HashMap<String, usize>>>,
823}
824
825struct RouteBindConcurrencyGuard {
826    concurrency: RouteBindConcurrency,
827    module_id: String,
828}
829
830impl RouteBindConcurrency {
831    /// Admit without waiting. Waiting here would move the bind stall from the
832    /// module reply to a semaphore and restore reader head-of-line blocking.
833    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
834        let mut modules = self
835            .modules
836            .lock()
837            .expect("route.bind concurrency mutex poisoned");
838        let in_flight = modules.entry(module_id.to_string()).or_default();
839        if *in_flight >= limit {
840            return Err(*in_flight);
841        }
842        *in_flight += 1;
843        Ok(RouteBindConcurrencyGuard {
844            concurrency: self.clone(),
845            module_id: module_id.to_string(),
846        })
847    }
848}
849
850impl Drop for RouteBindConcurrencyGuard {
851    fn drop(&mut self) {
852        let mut modules = self
853            .concurrency
854            .modules
855            .lock()
856            .expect("route.bind concurrency mutex poisoned");
857        let remove = {
858            let in_flight = modules
859                .get_mut(&self.module_id)
860                .expect("admitted route.bind has a concurrency entry");
861            *in_flight -= 1;
862            *in_flight == 0
863        };
864        if remove {
865            modules.remove(&self.module_id);
866        }
867    }
868}
869
870#[derive(Debug, Default)]
871struct ModuleBreakerState {
872    /// Relay timeouts observed with no accepted relay in between.
873    consecutive_timeouts: u32,
874    /// `Some` while the breaker is open: the instant the cooldown expires and
875    /// the next arrival may probe. `None` means closed.
876    cooldown_until: Option<Instant>,
877    /// A half-open probe has been admitted and has not settled yet. This is
878    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
879    /// that read the cooldown, so concurrent opens arriving at the moment the
880    /// cooldown expires cannot all decide that they are the probe.
881    probe_in_flight: Option<Arc<()>>,
882}
883
884/// What the breaker decided for one `route.open`, before any relay work.
885enum RouteBindAdmission<'a> {
886    Admitted {
887        guard: RouteBindBreakerGuard<'a>,
888        /// This open is the single half-open probe, so the transition is worth
889        /// one log line.
890        probe: bool,
891    },
892    Refused {
893        consecutive_timeouts: u32,
894        /// What is left of the cooldown. Zero when the refusal is because the
895        /// one probe is already in flight rather than because the cooldown has
896        /// not elapsed.
897        retry_in: Duration,
898        probe_in_flight: bool,
899    },
900}
901
902/// An outstanding admission, which must be told how its relay settled.
903///
904/// `Drop` settles it as inconclusive, so an early return between admission and
905/// the relay -- or the whole handler being cancelled when the client
906/// disconnects -- releases a half-open probe slot instead of leaving the
907/// breaker wedged half-open with no further probes.
908struct RouteBindBreakerGuard<'a> {
909    breakers: RouteBindBreakers,
910    module_id: &'a str,
911    probe_token: Option<Arc<()>>,
912    settled: bool,
913}
914
915impl RouteBindBreakerGuard<'_> {
916    /// The module answered within the budget and took the bind. THE ONLY
917    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
918    /// breaker, which is a transition worth logging.
919    fn record_accepted(&mut self) -> bool {
920        self.settled = true;
921        self.breakers.record_accepted(self.module_id)
922    }
923
924    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
925    /// COUNTS TOWARD OPENING.
926    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
927        self.settled = true;
928        self.breakers.record_timeout(
929            self.module_id,
930            self.probe_token.as_ref(),
931            threshold,
932            cooldown,
933        )
934    }
935
936    /// Everything else: the module REJECTED the bind, its connection went away
937    /// mid-relay, or the waiter was cancelled.
938    ///
939    /// None of these is evidence that a module is slow, and each already has
940    /// its own refusal with its own code. A module that rejects a bind in
941    /// microseconds is healthy and must never be convicted for it; a module
942    /// that died has said nothing about the module that replaces it. So these
943    /// neither increment nor reset the count -- they only release a probe slot.
944    fn record_inconclusive(&mut self) {
945        self.settled = true;
946        self.breakers
947            .record_inconclusive(self.module_id, self.probe_token.as_ref());
948    }
949}
950
951impl Drop for RouteBindBreakerGuard<'_> {
952    fn drop(&mut self) {
953        if !self.settled {
954            self.breakers
955                .record_inconclusive(self.module_id, self.probe_token.as_ref());
956        }
957    }
958}
959
960/// The breaker moved to open, reported so the caller can log it outside the
961/// lock. Opening is rare and load-bearing; the refusals that follow are
962/// frequent and are counted rather than logged.
963struct BreakerOpened {
964    consecutive_timeouts: u32,
965    /// True when a failed probe re-opened an already-open breaker, which reads
966    /// very differently in a log from a first opening.
967    reopened_after_probe: bool,
968}
969
970impl RouteBindBreakers {
971    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
972        self.modules
973            .lock()
974            .expect("route.bind breaker mutex poisoned")
975    }
976
977    /// Decide whether this `route.open` may attempt its relay. Takes the map
978    /// lock and nothing else, and never awaits.
979    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
980        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
981            probe: probe_token.is_some(),
982            guard: RouteBindBreakerGuard {
983                breakers: self.clone(),
984                module_id,
985                probe_token,
986                settled: false,
987            },
988        };
989
990        let mut modules = self.lock();
991        let Some(state) = modules.get_mut(module_id) else {
992            return admitted(None);
993        };
994        let Some(cooldown_until) = state.cooldown_until else {
995            return admitted(None);
996        };
997        if state.probe_in_flight.is_some() {
998            return RouteBindAdmission::Refused {
999                consecutive_timeouts: state.consecutive_timeouts,
1000                retry_in: Duration::ZERO,
1001                probe_in_flight: true,
1002            };
1003        }
1004        let now = Instant::now();
1005        if now < cooldown_until {
1006            return RouteBindAdmission::Refused {
1007                consecutive_timeouts: state.consecutive_timeouts,
1008                retry_in: cooldown_until - now,
1009                probe_in_flight: false,
1010            };
1011        }
1012        let token = Arc::new(());
1013        state.probe_in_flight = Some(Arc::clone(&token));
1014        admitted(Some(token))
1015    }
1016
1017    fn record_accepted(&self, module_id: &str) -> bool {
1018        self.lock()
1019            .remove(module_id)
1020            .is_some_and(|state| state.cooldown_until.is_some())
1021    }
1022
1023    fn record_timeout(
1024        &self,
1025        module_id: &str,
1026        probe_token: Option<&Arc<()>>,
1027        threshold: u32,
1028        cooldown: Duration,
1029    ) -> Option<BreakerOpened> {
1030        let mut modules = self.lock();
1031        let state = modules.entry(module_id.to_string()).or_default();
1032        let was_open = state.cooldown_until.is_some();
1033        let was_probe = Self::owns_probe(state, probe_token);
1034        if was_probe {
1035            state.probe_in_flight = None;
1036        }
1037        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
1038        if state.consecutive_timeouts < threshold {
1039            return None;
1040        }
1041        state.cooldown_until = Some(Instant::now() + cooldown);
1042        Some(BreakerOpened {
1043            consecutive_timeouts: state.consecutive_timeouts,
1044            reopened_after_probe: was_open && was_probe,
1045        })
1046    }
1047
1048    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
1049        // A relay can finish after the breaker was reset or after it opened
1050        // again and started a new probe. Only the guard whose token matches the
1051        // active probe may release it, so a late relay never frees a newer probe.
1052        state
1053            .probe_in_flight
1054            .as_ref()
1055            .zip(token)
1056            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
1057    }
1058
1059    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
1060        if let Some(state) = self.lock().get_mut(module_id) {
1061            if Self::owns_probe(state, probe_token) {
1062                state.probe_in_flight = None;
1063            }
1064        }
1065    }
1066
1067    /// Discard what was learned about a module, because the process it was
1068    /// learned about is gone. Returns the discarded count when it was non-zero.
1069    ///
1070    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
1071    /// `module_id` is a configuration identity that outlives any particular
1072    /// child; what the breaker observed was the process behind the module
1073    /// connection of the moment. When a new connection registers under that id
1074    /// the verdict's subject no longer exists, so the verdict is stale by
1075    /// construction rather than merely likely to be wrong. Keeping it would
1076    /// apply a dead process's record to a live one, which is the same defect
1077    /// class this breaker exists to stop the daemon committing.
1078    ///
1079    /// A half-open probe in flight is discarded with the rest: it was a
1080    /// question about the old process.
1081    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
1082        self.lock()
1083            .remove(module_id)
1084            .map(|state| state.consecutive_timeouts)
1085            .filter(|discarded| *discarded > 0)
1086    }
1087
1088    /// Open breakers, for the `server.describe` counters object. `None` when
1089    /// none is open, so the key stays absent rather than present-and-empty.
1090    ///
1091    /// This is the operator's answer to "is this module refusing instantly or
1092    /// is it fine?", which look identical from a client that retries and then
1093    /// succeeds.
1094    fn open_snapshot(&self) -> Option<serde_json::Value> {
1095        let now = Instant::now();
1096        let modules = self.lock();
1097        let open = modules
1098            .iter()
1099            .filter_map(|(module_id, state)| {
1100                let cooldown_until = state.cooldown_until?;
1101                Some((
1102                    module_id.clone(),
1103                    serde_json::json!({
1104                        "consecutive_timeouts": state.consecutive_timeouts,
1105                        "cooldown_remaining_ms":
1106                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
1107                        "probe_in_flight": state.probe_in_flight.is_some(),
1108                    }),
1109                ))
1110            })
1111            .collect::<serde_json::Map<String, serde_json::Value>>();
1112        (!open.is_empty()).then_some(serde_json::Value::Object(open))
1113    }
1114}
1115
1116impl ControlHandler {
1117    pub(crate) fn install_registration_service(
1118        &self,
1119        router: std::sync::Weak<crate::Router>,
1120        auth: crate::server::ServerAuth,
1121    ) {
1122        self.supervisor.install_registration_service(router, auth);
1123    }
1124    pub fn new(registry: Arc<Registry>) -> Self {
1125        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
1126    }
1127
1128    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
1129        let counters = forwarding.counters();
1130        // Taken from the forwarding table rather than created here, so that the
1131        // breaker a `route.open` consults is the same one a module's
1132        // registration resets, however many handlers are built over one table.
1133        let route_bind_breakers = forwarding.route_bind_breakers();
1134        let route_bind_concurrency = forwarding.route_bind_concurrency();
1135        let route_outages = forwarding.route_outages();
1136        Self {
1137            registry,
1138            #[cfg(test)]
1139            activity_request_pause: Arc::new(Mutex::new(None)),
1140            #[cfg(test)]
1141            cleanup_pause: Arc::new(Mutex::new(None)),
1142            #[cfg(test)]
1143            after_nonce_attestation: Arc::new(Mutex::new(None)),
1144            #[cfg(test)]
1145            hello_ingress_pause: Arc::new(Mutex::new(None)),
1146            #[cfg(test)]
1147            agent_commands_pause: Arc::new(Mutex::new(None)),
1148            claims: Arc::new(claims::Claims::default()),
1149            config_runtime: None,
1150            config_review_admission: Arc::new(crate::config_gate::MissingReviewAdmission),
1151            config_review_admission_is_default: true,
1152            activity: None,
1153            activity_streams: Arc::new(Mutex::new(HashMap::new())),
1154            catalog_streams: Arc::new(Mutex::new(HashMap::new())),
1155            forwarding,
1156            process_liveness: None,
1157            supervisor: SupervisorHandle::new(),
1158            resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1159                env!("CARGO_PKG_VERSION").to_owned(),
1160            )),
1161            agent_declarations: agent_commands::Queues::default(),
1162            list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1163            subc_capabilities: Arc::from([
1164                CAP_MANIFEST_REGISTRATION.to_string(),
1165                CAP_CHANNEL_LIFECYCLE.to_string(),
1166                CAP_PING_PONG.to_string(),
1167                CAP_SESSION_ATTACH.to_string(),
1168                CAP_ADMISSION_FACTS_RELAY.to_string(),
1169                CAP_SCOPES_V1.to_string(),
1170                subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1171                subc_protocol::peer::CALLER_ATTESTATION_CAPABILITY.to_string(),
1172                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1173                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1174                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1175                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1176                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1177                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1178                subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1179                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1180                subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1181                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1182                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1183                subc_protocol::ROUTE_CLOSE_REASONS_CAPABILITY.to_string(),
1184            ]),
1185            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1186            route_bind_relay_timeouts: BTreeMap::new(),
1187            route_bind_breakers,
1188            route_bind_concurrency,
1189            route_outages,
1190            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1191            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1192            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1193            storage_config: None,
1194            machine_id: None,
1195            admission_facts_carrier_module_id: None,
1196            admission_facts_targets: None,
1197            scopes: Arc::new(RwLock::new(ScopeTable::new(
1198                crate::daemon_config::default_scope_authority_owners(),
1199            ))),
1200            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1201            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1202            federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1203            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1204            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1205            build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1206            observed_peers: Arc::new(Mutex::new(HashMap::new())),
1207            captured_peers: Arc::new(Mutex::new(HashMap::new())),
1208            config_desktop_stamps: Arc::new(Mutex::new(HashMap::new())),
1209            local_connections: Arc::new(Mutex::new(HashSet::new())),
1210            operator_app_policy: Default::default(),
1211            operator_apps: Arc::new(Mutex::new(HashMap::new())),
1212            launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1213            app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1214            #[cfg(test)]
1215            app_open_test_platform: None,
1216            #[cfg(test)]
1217            app_open_test_launch: None,
1218            #[cfg(test)]
1219            recovery_exit_test_after_step: None,
1220            #[cfg(test)]
1221            recovery_exit_test_completed: None,
1222            #[cfg(test)]
1223            apply_reconcile_pause: Arc::new(Mutex::new(None)),
1224            #[cfg(test)]
1225            recovery_exit_test_quiet_retry: false,
1226            #[cfg(test)]
1227            recovery_exit_test_retry_pause: None,
1228            #[cfg(test)]
1229            config_delivery_test_wakes: None,
1230            #[cfg(all(test, unix))]
1231            config_delivery_test_idle: None,
1232            #[cfg(windows)]
1233            windows_verify: None,
1234            #[cfg(windows)]
1235            windows_flash: None,
1236            #[cfg(windows)]
1237            windows_track: None,
1238            #[cfg(all(test, windows))]
1239            windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1240            #[cfg(test)]
1241            caller_lookup: None,
1242            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1243            rescan: None,
1244            connected_clients: ConnectedClients::new(),
1245            local_endpoint_unavailable: None,
1246            counters,
1247            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1248            daemon_shutdown: DaemonShutdownFlag::default(),
1249            daemon_provenance: DaemonProvenanceFacts::default(),
1250            #[cfg(test)]
1251            control_dispatch_delay: None,
1252            #[cfg(test)]
1253            provenance_probe_override: None,
1254            #[cfg(test)]
1255            route_bind_relay_sent_hold: None,
1256            #[cfg(test)]
1257            hello_registry_to_forwarding_hold: None,
1258            #[cfg(test)]
1259            hello_ack_to_publication_hold: None,
1260            #[cfg(test)]
1261            hello_before_publication_hold: None,
1262            #[cfg(test)]
1263            hello_after_launch_check_hold: None,
1264            #[cfg(test)]
1265            hello_inside_publication_hold: None,
1266        }
1267    }
1268
1269    /// Set the central storage policy: registering modules then receive their
1270    /// resolved storage descriptor in HELLO_ACK.
1271    pub fn with_storage_config(
1272        mut self,
1273        storage_config: Option<crate::daemon_config::StorageConfig>,
1274    ) -> Self {
1275        self.storage_config = storage_config;
1276        self
1277    }
1278
1279    /// Open the ledger on its own writer thread; an unavailable sink does not
1280    /// prevent the daemon from serving routing or supervisor requests.
1281    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1282        self.activity = Some(crate::activity::ActivityLedger::open(
1283            run_dir.join("activity.jsonl"),
1284            incarnation,
1285        ));
1286        self.supervisor.set_activity(self.activity.clone());
1287        self
1288    }
1289
1290    /// Set the machine id served to every registering module (HELLO_ACK) and on
1291    /// `server.describe`.
1292    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1293        self.machine_id = machine_id;
1294        self
1295    }
1296
1297    /// Configure the exact reserved module and target ids permitted to relay
1298    /// opaque admission facts. Config-file loading validates this authority;
1299    /// this builder keeps the same policy available to embedded test daemons.
1300    pub fn with_admission_facts_config(
1301        mut self,
1302        carrier_module_id: Option<String>,
1303        targets: Option<Vec<String>>,
1304    ) -> Self {
1305        self.admission_facts_carrier_module_id = carrier_module_id;
1306        self.admission_facts_targets = targets;
1307        self
1308    }
1309
1310    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1311    /// Replaces the scope table with an empty one under the new list, so call it
1312    /// while building the handler, before any module can sync.
1313    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1314        self.scopes = Arc::new(RwLock::new(
1315            ScopeTable::new(owners.iter().cloned())
1316                .with_federated_owner(self.federated_scope_owner.clone()),
1317        ));
1318        self.scope_authority_owners = owners;
1319        self
1320    }
1321
1322    /// Set the exclusive federated owner while building the handler, before syncs.
1323    pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1324        self.scopes = Arc::new(RwLock::new(
1325            ScopeTable::new(self.scope_authority_owners.iter().cloned())
1326                .with_federated_owner(owner.clone()),
1327        ));
1328        self.federated_scope_owner = owner;
1329        self
1330    }
1331
1332    /// Set the startup policy before installing the rescan context or serving requests.
1333    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1334        self.privileged_operator_prompt = mode;
1335        self
1336    }
1337
1338    /// Override the route.bind relay timeout. Used by tests that assert the
1339    /// timeout path so they don't block on the production-safe default.
1340    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1341        self.route_bind_relay_timeout = timeout;
1342        self
1343    }
1344
1345    /// Install per-module route.bind relay budget overrides. A module id
1346    /// listed here wins over the daemon-wide default set via
1347    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1348    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1349    /// the same `Duration` the bind path will use.
1350    pub fn with_route_bind_relay_timeouts(
1351        mut self,
1352        timeouts: impl IntoIterator<Item = (String, Duration)>,
1353    ) -> Self {
1354        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1355        self
1356    }
1357
1358    /// Resolve the route.bind relay budget for a specific target module id.
1359    /// Per-module overrides win; the daemon-wide value (set via
1360    /// `with_route_bind_relay_timeout` or the built-in default) is the
1361    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1362    /// the same resolution `handle_route_open` will use.
1363    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1364        self.route_bind_relay_timeouts
1365            .get(module_id)
1366            .copied()
1367            .unwrap_or(self.route_bind_relay_timeout)
1368    }
1369
1370    /// Override the per-module bind-relay breaker policy.
1371    ///
1372    /// Used by tests, which cannot spend three production budgets opening a
1373    /// breaker or twenty seconds waiting for its cooldown. The production
1374    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1375    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1376    /// reasoning for the numbers.
1377    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1378        self.route_bind_breaker_threshold = threshold.max(1);
1379        self.route_bind_breaker_cooldown = cooldown;
1380        self
1381    }
1382
1383    #[cfg(test)]
1384    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1385        self.health_probe_timeout = timeout;
1386        self
1387    }
1388
1389    #[cfg(test)]
1390    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1391        self.control_dispatch_delay = Some(delay);
1392        self
1393    }
1394
1395    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1396    /// announce it is parked, once to be released. Build the barrier for two
1397    /// parties, the test being the other one.
1398    #[cfg(test)]
1399    pub(crate) fn with_route_bind_relay_sent_hold(
1400        mut self,
1401        hold: Arc<tokio::sync::Barrier>,
1402    ) -> Self {
1403        self.route_bind_relay_sent_hold = Some(hold);
1404        self
1405    }
1406
1407    /// A HELLO waits on `hold` twice between reserving its registry entry and
1408    /// installing its forwarding endpoint: once to announce it is parked, once
1409    /// to be released. Build the barrier for two parties, the test being the
1410    /// other one, and run the HELLO on a thread of its own.
1411    #[cfg(test)]
1412    pub(crate) fn with_hello_registry_to_forwarding_hold(
1413        mut self,
1414        hold: Arc<std::sync::Barrier>,
1415    ) -> Self {
1416        self.hello_registry_to_forwarding_hold = Some(hold);
1417        self
1418    }
1419
1420    #[cfg(test)]
1421    fn hold_between_registry_and_forwarding(&self) {
1422        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1423            hold.wait();
1424            hold.wait();
1425        }
1426    }
1427
1428    /// Park after the ACK is queued and forwarding is installed, until a test
1429    /// has read the ACK and issued a registry lookup from another connection.
1430    #[cfg(test)]
1431    fn hold_between_ack_and_publication(&self) {
1432        if let Some(hold) = &self.hello_ack_to_publication_hold {
1433            hold.wait();
1434            hold.wait();
1435        }
1436    }
1437
1438    #[cfg(all(test, unix))]
1439    pub(crate) fn with_hello_before_publication_hold(
1440        mut self,
1441        hold: Arc<std::sync::Barrier>,
1442    ) -> Self {
1443        self.hello_before_publication_hold = Some(hold);
1444        self
1445    }
1446
1447    #[cfg(all(test, unix))]
1448    pub(crate) fn with_hello_after_launch_check_hold(
1449        mut self,
1450        hold: Arc<std::sync::Barrier>,
1451    ) -> Self {
1452        self.hello_after_launch_check_hold = Some(hold);
1453        self
1454    }
1455
1456    #[cfg(all(test, unix))]
1457    pub(crate) fn with_hello_inside_publication_hold(
1458        mut self,
1459        hold: Arc<std::sync::Barrier>,
1460    ) -> Self {
1461        self.hello_inside_publication_hold = Some(hold);
1462        self
1463    }
1464
1465    pub fn with_process_liveness(
1466        mut self,
1467        process_liveness: Arc<dyn ModuleProcessLiveness>,
1468    ) -> Self {
1469        self.process_liveness = Some(process_liveness);
1470        self
1471    }
1472
1473    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1474        self.supervisor = supervisor;
1475        if let Some(owner) = &self.config_runtime {
1476            self.supervisor.install_binding_retirement(owner);
1477        }
1478        self.supervisor.set_activity(self.activity.clone());
1479        self
1480    }
1481
1482    pub fn with_daemon_provenance(
1483        mut self,
1484        pid: u32,
1485        started_at_ms: u64,
1486        executable_path: Option<PathBuf>,
1487        build_git_sha: Option<String>,
1488        build_lock_digest: Option<String>,
1489    ) -> Self {
1490        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1491        let process_start_time = process_start_time(pid);
1492        self.daemon_provenance = DaemonProvenanceFacts {
1493            build: DaemonBuildProvenance {
1494                build_git_sha,
1495                build_lock_digest,
1496            },
1497            pid: Some(pid),
1498            started_at_ms: Some(started_at_ms),
1499            start_clock: None,
1500            executable_path,
1501            executable_identity,
1502            process_start_time,
1503            probe: ExecutableIdentityProbe::default(),
1504        };
1505        self
1506    }
1507
1508    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1509        self.daemon_provenance.start_clock = Some(clock);
1510        self
1511    }
1512
1513    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1514    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1515        self.scopes
1516            .write()
1517            .unwrap_or_else(|poisoned| poisoned.into_inner())
1518            .set_wall_clock(Arc::new(clock));
1519        self
1520    }
1521
1522    #[cfg(test)]
1523    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1524        self.provenance_probe_override = Some(result);
1525        self
1526    }
1527
1528    /// Install the configured module set and its reserved capability bindings.
1529    /// Bindings are configuration-scoped and may point at a provider that has not
1530    /// been installed yet, so this does not require the bound module to exist.
1531    pub fn with_capability_config(
1532        self,
1533        modules: impl IntoIterator<Item = (String, bool)>,
1534        reserved_capabilities: BTreeMap<String, String>,
1535    ) -> Self {
1536        self.capability_evaluator
1537            .configure(modules, reserved_capabilities);
1538        self
1539    }
1540
1541    /// Share the supervisor's daemon-shutdown flag with this handler, so
1542    /// requirement changes after the daemon has begun its announced shutdown
1543    /// are not reported as faults.
1544    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1545        self.daemon_shutdown = flag;
1546        self
1547    }
1548
1549    pub fn with_supervisor_rescan(
1550        mut self,
1551        supervisor: Supervisor,
1552        config_path: impl Into<PathBuf>,
1553        configured_port: Option<u16>,
1554    ) -> Self {
1555        self.rescan = Some(SupervisorRescanContext {
1556            supervisor,
1557            config_path: config_path.into(),
1558            configured_port,
1559            storage_config: self.storage_config.clone(),
1560            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1561            admission_facts_targets: self.admission_facts_targets.clone(),
1562            scope_authority_owners: self.scope_authority_owners.clone(),
1563            federated_scope_owner: self.federated_scope_owner.clone(),
1564            privileged_operator_prompt: self.privileged_operator_prompt,
1565            generation: Arc::default(),
1566        });
1567        self
1568    }
1569
1570    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1571        self.connected_clients = connected_clients;
1572        self
1573    }
1574
1575    /// Report why this daemon serves no local endpoint, in `server.describe`, so
1576    /// an operator can see a bind failure without reading the daemon log.
1577    pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1578        self.local_endpoint_unavailable = reason;
1579        self
1580    }
1581
1582    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1583        Arc::clone(&self.forwarding)
1584    }
1585
1586    pub(crate) fn counters(&self) -> DaemonCounters {
1587        self.counters.clone()
1588    }
1589
1590    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1591    /// requirement event without depending on an operator polling a status command.
1592    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1593        let handler = Arc::downgrade(&self);
1594        let evaluator = self.capability_evaluator.clone();
1595        tokio::spawn(async move {
1596            loop {
1597                evaluator.wait_for_change_or_deadline().await;
1598                let Some(handler) = handler.upgrade() else {
1599                    break;
1600                };
1601                handler.refresh_capability_requirements();
1602            }
1603        });
1604    }
1605
1606    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1607    /// deadline passed during sleep is enforced on the first tick after wake.
1608    /// While no live scope carries a deadline the loop waits for an accepted
1609    /// scope change instead of ticking, so a daemon without deadlines never
1610    /// wakes for expiry.
1611    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1612        let context = scope_expiry::ScopeExpiryContext::new(&self);
1613        let deadline_added = self.scope_deadline_added.clone();
1614        tokio::spawn(async move {
1615            loop {
1616                if !context.has_deadlines() {
1617                    deadline_added.notified().await;
1618                    continue;
1619                }
1620                tokio::time::sleep(Duration::from_secs(1)).await;
1621                if let Err(error) = context.sweep() {
1622                    warn!(%error, "scope expiry sweep failed");
1623                }
1624            }
1625        });
1626    }
1627
1628    #[cfg(test)]
1629    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1630        scope_expiry::ScopeExpiryContext::new(self).sweep()
1631    }
1632
1633    fn publish_scope_changes(
1634        &self,
1635        changes: &[crate::scopes::ScopeTagChange],
1636        expired: &[crate::scopes::ScopeExpired],
1637    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1638        scope_expiry::ScopeExpiryContext::new(self).publish_scope_changes(changes, expired)
1639    }
1640
1641    fn runtime_capability_snapshot(
1642        &self,
1643    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1644        let runtime = self
1645            .supervisor
1646            .list()
1647            .into_iter()
1648            .map(|module| {
1649                let status = module.status().map_err(|err| {
1650                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1651                })?;
1652                Ok(RuntimeModule {
1653                    module_id: status.module_id,
1654                    state: status.state,
1655                    enabled: status.enabled,
1656                })
1657            })
1658            .collect::<Result<Vec<_>, RouterError>>()?;
1659        // Include every registration, published or not. The capability
1660        // evaluator counts a module as providing its capabilities as soon as it
1661        // has registered, even before `catalog.list` shows it, so a HELLO in
1662        // progress does not change other modules' required-capability
1663        // verdicts. See `Registry::list_registered_modules`.
1664        let registrations = self.registry.list_registered_modules().map_err(|err| {
1665            RouterError::backend(
1666                0,
1667                0,
1668                format!("failed to list capability registrations: {err}"),
1669            )
1670        })?;
1671        let registrations = registrations
1672            .into_iter()
1673            .map(|registration| RegisteredModule {
1674                module_id: registration.manifest.module_id,
1675                module_version: registration.manifest.module_version,
1676                capabilities: registration.manifest.capabilities,
1677            })
1678            .collect();
1679        Ok((runtime, registrations))
1680    }
1681
1682    /// The capability side effects of a module becoming the active registration
1683    /// for its id: cache its manifest (logging how it differs from the cached
1684    /// one), run the deny census when its declarations call for one, and recompute the
1685    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1686    /// candidate's does not, and the supervisor does it at promotion instead,
1687    /// through [`crate::supervise::SwapPromotionObserver`].
1688    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1689        let cached_registration = RegisteredModule {
1690            module_id: registration.manifest.module_id.clone(),
1691            module_version: registration.manifest.module_version.clone(),
1692            capabilities: registration.manifest.capabilities.clone(),
1693        };
1694        let change = self.capability_evaluator.record_hello(&cached_registration);
1695        log_manifest_change(&cached_registration.module_id, &change);
1696        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1697            self.enforce_capability_denies();
1698        }
1699        self.refresh_capability_requirements();
1700    }
1701
1702    /// Point the shared supervisor handle at this handler for swap promotions.
1703    /// Called wherever a handler is put behind the `Arc` the router serves, so
1704    /// it can be held weakly.
1705    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1706        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1707            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1708        self.supervisor.set_swap_promotion_observer(observer);
1709    }
1710
1711    pub fn refresh_capability_requirements(&self) {
1712        match self.runtime_capability_snapshot() {
1713            Ok((runtime, registrations)) => {
1714                log_requirement_events(
1715                    self.capability_evaluator
1716                        .evaluate_now(&runtime, &registrations),
1717                    &self.daemon_shutdown,
1718                );
1719            }
1720            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1721        }
1722    }
1723
1724    /// Reconcile only live, attested route bindings after a capability deny edge
1725    /// or target claim was added. This is deliberately a control-plane census:
1726    /// the opaque forwarding hot path must not grow a per-frame capability check.
1727    fn enforce_capability_denies(&self) {
1728        let registrations = match self.registry.list_registered_modules() {
1729            Ok(snapshot) => snapshot,
1730            Err(err) => {
1731                warn!(error = %err, "failed to read registrations for capability deny census");
1732                return;
1733            }
1734        };
1735        let manifests = registrations
1736            .into_iter()
1737            .map(|registration| {
1738                (
1739                    registration.manifest.module_id.clone(),
1740                    registration.manifest,
1741                )
1742            })
1743            .collect::<BTreeMap<_, _>>();
1744        let census = match self.forwarding.route_census(None) {
1745            Ok(census) => census,
1746            Err(err) => {
1747                warn!(error = %err, "failed to read route census for capability deny enforcement");
1748                return;
1749            }
1750        };
1751
1752        for (target_module_id, routes) in census {
1753            let Some(target_manifest) = manifests.get(&target_module_id) else {
1754                continue;
1755            };
1756            let mut closed_routes = Vec::new();
1757            let mut goodbyes = Vec::new();
1758            for route in routes {
1759                let Principal::Reserved {
1760                    module_id: opening_module_id,
1761                } = &route.principal
1762                else {
1763                    continue;
1764                };
1765                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1766                    continue;
1767                };
1768                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1769                    continue;
1770                };
1771
1772                match self.forwarding.release_client_route(
1773                    route.goodbye_target.connection_id,
1774                    route.goodbye_target.channel,
1775                    route.goodbye_target.epoch,
1776                ) {
1777                    Ok(RouteRelease::Removed(module_goodbye)) => {
1778                        warn!(
1779                            opening_module_id,
1780                            target_module_id,
1781                            capability,
1782                            "force-closing route because an attested capability deny edge now matches"
1783                        );
1784                        goodbyes.push(module_goodbye);
1785                        goodbyes.push(route.goodbye_target.clone());
1786                        closed_routes.push(route);
1787                    }
1788                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1789                    Err(err) => warn!(
1790                        opening_module_id,
1791                        target_module_id,
1792                        capability,
1793                        error = %err,
1794                        "failed to force-close capability-denied route"
1795                    ),
1796                }
1797            }
1798
1799            if closed_routes.is_empty() {
1800                continue;
1801            }
1802            send_route_control_pushes(
1803                &self.forwarding,
1804                closed_routes,
1805                ClientControlPush::RouteClosed {
1806                    module_id: target_module_id,
1807                    channels: Vec::new(),
1808                    reason: RouteCloseReason::CapabilityDenied,
1809                    drained: false,
1810                    abandoned: 0,
1811                    excluded_subscriptions: 0,
1812                    terminal: Some(false),
1813                },
1814            );
1815            // route.closed records the reason; the route GOODBYE ends the
1816            // client's handle and settles any calls still pending on it.
1817            self.emit_route_goodbyes(goodbyes);
1818        }
1819    }
1820
1821    /// Why a registered module is not accepting new route binds, or `None` when
1822    /// it is. This is the module's effective readiness: its declared readiness
1823    /// first, then every `need: required` capability it declares evaluating to
1824    /// `provided`. `route.open` and `catalog.list` both read it here so the
1825    /// catalog never reports a module routable that `route.open` would refuse.
1826    fn not_ready_reason(
1827        &self,
1828        registration: &crate::registry::ModuleRegistration,
1829    ) -> Option<NotReadyReason> {
1830        if !registration.ready {
1831            return Some(NotReadyReason {
1832                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1833                capability: None,
1834            });
1835        }
1836        self.first_unprovided_required_capability(registration)
1837            .map(|capability| NotReadyReason {
1838                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1839                capability: Some(capability),
1840            })
1841    }
1842
1843    /// The lexicographically first capability this registration declares
1844    /// `need: required` whose evaluator verdict is not `provided`.
1845    ///
1846    /// The verdicts are the capability evaluator's own; nothing here decides
1847    /// what "provided" means. The evaluator counts a capability provided as
1848    /// soon as a module claiming it has REGISTERED, not once that module is
1849    /// ready. That distinction is what keeps two modules that require each
1850    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1851    /// is ready", each would wait for the other to become ready first and
1852    /// neither ever would. Do not tighten it to readiness.
1853    ///
1854    /// A required capability with no verdict at all means this registration's
1855    /// HELLO or catalog.update landed after the last recompute; recompute once
1856    /// rather than let a missing verdict read as either answer. If it is still
1857    /// missing (the recompute itself failed) the capability counts as
1858    /// unprovided: the refusal is retryable, and routing a module whose
1859    /// required provider is unknown is the outcome this check exists to stop.
1860    fn first_unprovided_required_capability(
1861        &self,
1862        registration: &crate::registry::ModuleRegistration,
1863    ) -> Option<String> {
1864        let required = registration
1865            .manifest
1866            .capabilities
1867            .iter()
1868            .flat_map(|declarations| declarations.requires.iter())
1869            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1870            .map(|requirement| requirement.capability.as_str())
1871            .collect::<BTreeSet<_>>();
1872        if required.is_empty() {
1873            return None;
1874        }
1875        let module_id = registration.manifest.module_id.as_str();
1876        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1877        if required
1878            .iter()
1879            .any(|capability| verdict(capability).is_none())
1880        {
1881            self.refresh_capability_requirements();
1882        }
1883        required
1884            .into_iter()
1885            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1886            .map(str::to_string)
1887    }
1888
1889    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1890        self.capability_evaluator
1891            .statuses()
1892            .into_iter()
1893            .map(capability_requirement_status)
1894            .collect()
1895    }
1896
1897    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1898    /// registration-release watch. The signal is what the supervisor waits on
1899    /// before spawning a replacement, so it must only fire once forwarding
1900    /// teardown is also done (see [`Self::cleanup_connection`] /
1901    /// [`Self::handle_goodbye`]). Unpublished HELLO rollback uses the separate
1902    /// registry-only path below because publication already owns the spawn lock.
1903    fn deregister_connection(
1904        &self,
1905        connection_id: ConnectionId,
1906        reason: RegistrationEndReason,
1907    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1908        if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
1909            spawn.end_primary_registration();
1910        }
1911        self.registry
1912            .deregister_connection_with_reason(connection_id, reason)
1913    }
1914
1915    // HELLO publication owns the spawn transition guard. claim.mint is allowed
1916    // only after HELLO acceptance, so a failed first HELLO has no bearer tokens
1917    // or claim-bound consumers to revoke. Remove only its unpublished registry
1918    // entry without re-entering primary-registration teardown.
1919    fn rollback_unpublished_hello(
1920        &self,
1921        connection: ConnectionId,
1922    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1923        self.registry.deregister_connection_with_reason(
1924            connection,
1925            RegistrationEndReason::RegistrationFailed,
1926        )
1927    }
1928
1929    fn request_refusal_close(
1930        &self,
1931        connection: ConnectionId,
1932        sink: Option<&crate::FrameSink>,
1933        reason: CloseReason,
1934    ) {
1935        if let Some(sink) = sink {
1936            sink.terminate_ingress();
1937        }
1938        if self.claims.close_notifications_enabled() {
1939            self.forwarding.request_connection_close(connection, reason);
1940        }
1941    }
1942
1943    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1944        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1945            return None;
1946        }
1947        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1948            parse_client_control_request(&frame.body)
1949        else {
1950            return None;
1951        };
1952        Some(target_module_id(&target).to_string())
1953    }
1954
1955    pub(crate) fn route_open_capacity_refusal(
1956        &self,
1957        ctx: &RouteCtx,
1958        frame: &Frame,
1959        target_module_id: &str,
1960        in_flight: usize,
1961        limit: usize,
1962    ) -> Result<Frame, RouterError> {
1963        // Sent before the request reaches `handle_route_open`, so the caller
1964        // fields come from the raw frame here.
1965        let caller = self.route_open_caller_from_frame(ctx, frame);
1966        self.route_open_admission_refusal_frame(
1967            ctx,
1968            &caller,
1969            frame,
1970            target_module_id,
1971            "open_admission_full",
1972            (in_flight, limit),
1973            format!(
1974                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1975            ),
1976        )
1977    }
1978
1979    fn route_open_target_capacity_refusal(
1980        &self,
1981        ctx: &RouteCtx,
1982        caller: &RouteOpenCaller,
1983        frame: &Frame,
1984        target_module_id: &str,
1985        in_flight: usize,
1986    ) -> Result<Frame, RouterError> {
1987        self.route_open_admission_refusal_frame(
1988            ctx,
1989            caller,
1990            frame,
1991            target_module_id,
1992            "target_binds_full",
1993            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1994            format!(
1995                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1996            ),
1997        )
1998    }
1999
2000    /// Admission pressure clears as existing binds settle, so its refusal must
2001    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
2002    /// `module_reloading`, `module_warming`, `target_unavailable`, or
2003    /// `module_timeout`. `target_unavailable` is honest for an attempt that
2004    /// cannot currently reach its target; `module_timeout` would falsely claim
2005    /// that a wait expired. A new, cleaner code would be terminal to deployed
2006    /// clients, so it requires a client-tolerance rollout before daemon emission.
2007    #[allow(clippy::too_many_arguments)]
2008    fn route_open_admission_refusal_frame(
2009        &self,
2010        ctx: &RouteCtx,
2011        caller: &RouteOpenCaller,
2012        frame: &Frame,
2013        target_module_id: &str,
2014        reason: &'static str,
2015        (in_flight, limit): (usize, usize),
2016        message: impl Into<String>,
2017    ) -> Result<Frame, RouterError> {
2018        let code = error_codes::TARGET_UNAVAILABLE;
2019        self.counters.increment_route_open_refused(code);
2020        info!(
2021            target: "control",
2022            code,
2023            reason,
2024            module_id = ?target_module_id,
2025            connection_id = ctx.connection_id.get(),
2026            principal = %caller.principal,
2027            scope = %caller.scope,
2028            harness = %caller.harness,
2029            in_flight,
2030            limit,
2031            "route.open refused"
2032        );
2033        control_error_frame(frame, code, message.into())
2034    }
2035
2036    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
2037    ///
2038    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
2039    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
2040    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
2041    #[cfg(test)]
2042    pub fn handle_control(
2043        &self,
2044        connection_id: ConnectionId,
2045        frame: Frame,
2046    ) -> Result<Vec<Frame>, RouterError> {
2047        match frame.header.ty {
2048            FrameType::Ping => Ok(vec![pong(&frame)?]),
2049            FrameType::Hello => self.handle_hello(connection_id, None, frame),
2050            FrameType::Goodbye => self.handle_goodbye(connection_id),
2051            ty => Ok(vec![control_error_frame(
2052                &frame,
2053                "unsupported_control_frame",
2054                format!("unsupported channel-0 frame {ty:?}"),
2055            )?]),
2056        }
2057    }
2058
2059    pub async fn handle_control_frame(
2060        &self,
2061        ctx: &RouteCtx,
2062        frame: Frame,
2063    ) -> Result<Vec<Frame>, RouterError> {
2064        self.handle_control_frame_timed(ctx, frame, None).await
2065    }
2066
2067    #[cfg(any(test, feature = "test-support"))]
2068    pub fn with_test_operator_app_policy(
2069        mut self,
2070        policy: crate::operator_apps::TestOperatorAppPolicy,
2071    ) -> Self {
2072        self.operator_app_policy = policy.into();
2073        self
2074    }
2075
2076    /// Inject the Windows signature check for tests. The injected check
2077    /// answers the verified version string or the rejection reason; the
2078    /// daemon enforces the floor itself on the returned version.
2079    #[cfg(all(windows, any(test, feature = "test-support")))]
2080    pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
2081        self.windows_verify = Some(verify);
2082        self
2083    }
2084
2085    /// Record which pids the daemon would flash for `already_running`.
2086    #[cfg(all(windows, any(test, feature = "test-support")))]
2087    pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
2088        self.windows_flash = Some(flash);
2089        self
2090    }
2091
2092    /// Inject a tracking failure for tests: makes `track_launched_process`
2093    /// fail so a test can prove a launch that cannot be tracked is stopped,
2094    /// never left running.
2095    #[cfg(all(windows, any(test, feature = "test-support")))]
2096    pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
2097        self.windows_track = Some(track);
2098        self
2099    }
2100
2101    pub(crate) fn record_captured_peer(
2102        &self,
2103        connection: ConnectionId,
2104        peer: subc_os::local_peer::CapturedPeer,
2105    ) {
2106        self.record_observed_peer(connection, peer.observation().clone());
2107        let identity = self.operator_app_policy.config_identity(peer.clone());
2108        #[cfg(windows)]
2109        let identity = identity.or_else(|| self.config_windows_launch_identity(&peer));
2110        if let Some(identity) = identity {
2111            // Keep the connect-time launch incarnation. Looking it up afresh
2112            // from a numeric pipe-client PID could match a later reused PID.
2113            self.config_desktop_stamps
2114                .lock()
2115                .unwrap_or_else(|p| p.into_inner())
2116                .insert(connection, identity);
2117        }
2118        self.captured_peers
2119            .lock()
2120            .unwrap_or_else(|p| p.into_inner())
2121            .insert(connection, peer.clone());
2122        #[cfg(windows)]
2123        if let Some(stamp) = self.stamp_windows_launch(&peer) {
2124            self.operator_apps
2125                .lock()
2126                .unwrap_or_else(|p| p.into_inner())
2127                .insert(connection, stamp);
2128        }
2129        if let Some(stamp) = self.operator_app_policy.stamp(peer) {
2130            self.operator_apps
2131                .lock()
2132                .unwrap_or_else(|p| p.into_inner())
2133                .insert(connection, stamp);
2134        }
2135    }
2136
2137    pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
2138        self.local_connections
2139            .lock()
2140            .unwrap_or_else(|p| p.into_inner())
2141            .insert(connection);
2142    }
2143
2144    // Keep the connection-level delivery decision shared with route.bind. The
2145    // target's capability opt-in is separate from recognition of the caller.
2146    fn connection_operator_app(
2147        &self,
2148        stamp: Option<&crate::operator_apps::OperatorAppStamp>,
2149    ) -> Option<subc_protocol::peer::OperatorApp> {
2150        stamp
2151            .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
2152            .map(|stamp| stamp.app.clone())
2153    }
2154
2155    fn handle_connection_describe(
2156        &self,
2157        ctx: &RouteCtx,
2158        frame: Frame,
2159    ) -> Result<Vec<Frame>, RouterError> {
2160        if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
2161            return Ok(vec![control_error_frame(
2162                &frame,
2163                "invalid_control_body",
2164                "malformed connection.describe body",
2165            )?]);
2166        }
2167        let local = self
2168            .local_connections
2169            .lock()
2170            .unwrap_or_else(|p| p.into_inner())
2171            .contains(&ctx.connection_id);
2172        let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
2173        let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
2174        Ok(vec![control_response_body_frame(
2175            &frame,
2176            &subc_control::ConnectionDescribeReply::new(local, operator_app),
2177            ops::CONNECTION_DESCRIBE,
2178        )?])
2179    }
2180
2181    pub(crate) fn record_observed_peer(
2182        &self,
2183        connection: ConnectionId,
2184        peer: subc_os::local_peer::ObservedPeer,
2185    ) {
2186        use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
2187        let peer = match peer {
2188            OsPeer::Unobserved => ObservedPeer::unobserved(),
2189            OsPeer::Observed(peer) => {
2190                let observed = match peer.user {
2191                    PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
2192                    PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
2193                };
2194                observed
2195                    .with_exe(
2196                        peer.exe
2197                            .and_then(|path| path.into_os_string().into_string().ok()),
2198                    )
2199                    .with_signing_id(peer.signing_id)
2200            }
2201        };
2202        self.observed_peers
2203            .lock()
2204            .unwrap_or_else(|p| p.into_inner())
2205            .insert(connection, peer);
2206    }
2207
2208    pub(crate) fn record_caller_endpoints(
2209        &self,
2210        connection: ConnectionId,
2211        endpoints: Option<(SocketAddr, SocketAddr)>,
2212    ) {
2213        let mut callers = self
2214            .caller_endpoints
2215            .lock()
2216            .unwrap_or_else(|p| p.into_inner());
2217        callers.remove(&connection);
2218        if let Some(endpoints) = endpoints {
2219            callers.insert(connection, endpoints);
2220        }
2221    }
2222
2223    #[cfg(test)]
2224    pub(crate) fn with_caller_lookup_for_test(
2225        mut self,
2226        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2227    ) -> Self {
2228        self.caller_lookup = Some(Arc::new(lookup));
2229        self
2230    }
2231
2232    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2233        let endpoints = self
2234            .caller_endpoints
2235            .lock()
2236            .unwrap_or_else(|p| p.into_inner())
2237            .get(&connection)
2238            .copied();
2239        #[cfg(test)]
2240        let lookup = self.caller_lookup.clone();
2241        tokio::task::spawn_blocking(move || {
2242            #[cfg(test)]
2243            if let Some(lookup) = lookup {
2244                return lookup(endpoints);
2245            }
2246            endpoints
2247                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2248                .unwrap_or_default()
2249        })
2250        .await
2251        .unwrap_or_default()
2252    }
2253
2254    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2255        frame.header.channel == 0
2256            && frame.header.ty == FrameType::Request
2257            && self
2258                .forwarding
2259                .module_endpoint_for_connection(ctx.connection_id)
2260                .is_ok_and(|endpoint| endpoint.is_none())
2261            && parse_client_control_request(&frame.body)
2262                .ok()
2263                .is_some_and(|request| privileged_operation(&request).is_some())
2264    }
2265
2266    pub(crate) async fn handle_control_frame_timed(
2267        &self,
2268        ctx: &RouteCtx,
2269        frame: Frame,
2270        dispatch_started_at: Option<StdInstant>,
2271    ) -> Result<Vec<Frame>, RouterError> {
2272        let first = ctx.egress.first_ingress_frame();
2273        self.handle_control_frame_ingress(ctx, frame, dispatch_started_at, first)
2274            .await
2275    }
2276
2277    pub(crate) async fn handle_control_frame_ingress(
2278        &self,
2279        ctx: &RouteCtx,
2280        frame: Frame,
2281        dispatch_started_at: Option<StdInstant>,
2282        first: bool,
2283    ) -> Result<Vec<Frame>, RouterError> {
2284        if ctx.egress.ingress_terminal() {
2285            return Ok(Vec::new());
2286        }
2287        #[cfg(test)]
2288        self.claims.record_dispatch(frame.header.corr);
2289        let claim_binding = self.claims.binding(ctx.connection_id);
2290        let mut prefix_attempt = (frame.header.ty == FrameType::Hello
2291            && claim_binding
2292                .as_ref()
2293                .is_some_and(|binding| binding.prefix_module_id().is_some()))
2294        .then(|| self.prefix_hello_attempt(ctx));
2295        if claim_binding
2296            .as_ref()
2297            .is_some_and(|binding| binding.prefix_module_id().is_some())
2298        {
2299            let registered = self
2300                .forwarding
2301                .module_endpoint_for_connection(ctx.connection_id)
2302                .map_err(RouterError::Forwarding)?
2303                .is_some();
2304            if !registered && frame.header.ty != FrameType::Hello {
2305                return self
2306                    .refuse_claim_connection(ctx, &frame, "claim_hello_required")
2307                    .await;
2308            }
2309            if registered && frame.header.ty == FrameType::Goodbye {
2310                ctx.egress.terminate_ingress();
2311                let result = self.handle_goodbye(ctx.connection_id);
2312                self.claims.connection_closed(ctx.connection_id);
2313                self.request_refusal_close(
2314                    ctx.connection_id,
2315                    Some(&ctx.egress),
2316                    CloseReason::new("claim_registration_ended", "prefix registration ended"),
2317                );
2318                return result;
2319            }
2320            if registered && frame.header.ty == FrameType::Request {
2321                let allowed =
2322                    serde_json::from_slice::<ControlOpProbe>(&frame.body).is_ok_and(|probe| {
2323                        matches!(
2324                            probe.op.as_str(),
2325                            "catalog.update"
2326                                | "catalog.update_role_declarations"
2327                                | "scope.describe"
2328                                | "supervisor.live_roots"
2329                                | "activity.record"
2330                        )
2331                    });
2332                if !allowed {
2333                    return Ok(vec![control_error_frame(
2334                        &frame,
2335                        channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2336                        "prefix registrations have only brokered module operations",
2337                    )?]);
2338                }
2339            }
2340            if registered
2341                && frame.header.ty == FrameType::Push
2342                && !serde_json::from_slice::<ControlOpProbe>(&frame.body)
2343                    .is_ok_and(|probe| probe.op == "route.status")
2344            {
2345                return Ok(vec![control_error_frame(
2346                    &frame,
2347                    channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2348                    "prefix registrations may push only route.status for routes they serve",
2349                )?]);
2350            }
2351        }
2352        match frame.header.ty {
2353            FrameType::Ping => Ok(vec![pong(&frame)?]),
2354            FrameType::Hello => {
2355                if claim_binding
2356                    .as_ref()
2357                    .is_some_and(|binding| binding.prefix_module_id().is_none())
2358                {
2359                    return self
2360                        .refuse_claim_connection(ctx, &frame, "claim_hello_not_allowed")
2361                        .await;
2362                }
2363                // Resolved once, here: the blocking HELLO below may outlive this
2364                // reader and run after cleanup has forgotten the channel, and it
2365                // must still be judged as this channel's HELLO.
2366                let channel_spawn = self.supervisor.registration_spawn(ctx.connection_id);
2367                if let Some(spawn) = &channel_spawn {
2368                    spawn.wait_launch().await;
2369                    // Refused launches close the channel. Leave their pending
2370                    // HELLO unanswered instead of racing a reply against shutdown.
2371                    if !spawn.launch_ready() {
2372                        ctx.egress.terminate_ingress();
2373                        return Ok(Vec::new());
2374                    }
2375                    #[cfg(test)]
2376                    self.pause_hello_ingress_for_test().await;
2377                } else if claim_binding.is_none() {
2378                    if let Ok(hello) = serde_json::from_slice::<serde_json::Value>(&frame.body) {
2379                        if let (Some(module), Some(nonce)) = (
2380                            hello
2381                                .get("manifest")
2382                                .and_then(|value| value.get("module_id"))
2383                                .and_then(|value| value.as_str()),
2384                            hello.get("launch_nonce").and_then(|value| value.as_str()),
2385                        ) {
2386                            if !self
2387                                .config_runtime
2388                                .as_ref()
2389                                .is_some_and(|owner| owner.module_is_gated(module))
2390                                && !self.supervisor.wait_program_binding(module, nonce).await
2391                            {
2392                                self.request_refusal_close(
2393                                    ctx.connection_id,
2394                                    Some(&ctx.egress),
2395                                    CloseReason::new(
2396                                        "program_refused",
2397                                        "the spawned program binding failed",
2398                                    ),
2399                                );
2400                                return Ok(vec![control_error_frame(
2401                                    &frame,
2402                                    if self
2403                                        .config_runtime
2404                                        .as_ref()
2405                                        .is_some_and(|owner| owner.is_recovering())
2406                                    {
2407                                        "config_gated"
2408                                    } else {
2409                                        "program_refused"
2410                                    },
2411                                    "the spawned program binding failed",
2412                                )?]);
2413                            }
2414                        }
2415                    }
2416                }
2417                let handler = self.clone();
2418                let connection = ctx.connection_id;
2419                let egress = ctx.egress.clone();
2420                let correlation = frame.header.corr;
2421                let dispatcher = tracing::dispatcher::get_default(|dispatcher| dispatcher.clone());
2422                let span = tracing::Span::current();
2423                let hello_binding = claim_binding.clone();
2424                let replies = tokio::task::spawn_blocking(move || {
2425                    tracing::dispatcher::with_default(&dispatcher, || {
2426                        span.in_scope(|| {
2427                            handler.handle_hello_checked(
2428                                connection,
2429                                Some(egress),
2430                                frame,
2431                                true,
2432                                hello_binding,
2433                                channel_spawn,
2434                            )
2435                        })
2436                    })
2437                })
2438                .await
2439                .map_err(|error| RouterError::backend(0, correlation, error.to_string()))??;
2440                #[cfg(test)]
2441                if prefix_attempt.is_some() {
2442                    self.claims.pause_prefix_commit_for_test(false).await;
2443                }
2444                if let Some(attempt) = &mut prefix_attempt {
2445                    let accepted = !replies
2446                        .iter()
2447                        .any(|reply| reply.header.ty == FrameType::Error)
2448                        && self
2449                            .registry
2450                            .get_module_by_connection(ctx.connection_id)
2451                            .map_err(|err| RouterError::backend(0, correlation, err.to_string()))?
2452                            .is_some();
2453                    if accepted {
2454                        attempt.accepted = true;
2455                    } else {
2456                        ctx.egress.terminate_ingress();
2457                        for reply in replies {
2458                            ctx.egress.send_flushed(reply).await?;
2459                        }
2460                        return Ok(Vec::new());
2461                    }
2462                }
2463                Ok(replies)
2464            }
2465            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2466            FrameType::Cancel => {
2467                // Channel-0 cancellation withdraws approvals and subscriptions
2468                // and fences pending agent declarations;
2469                // successful cancellation has no ordinary reply.
2470                // A config Apply may also own an OS prompt. Cancel that prompt
2471                // before config cancellation can report the request handled.
2472                if self
2473                    .agent_declarations
2474                    .cancel(ctx.connection_id, frame.header.corr)
2475                    || self
2476                        .forwarding
2477                        .operator_confirms()
2478                        .cancel(ctx.connection_id, frame.header.corr)
2479                    || self.cancel_config_approval(ctx.connection_id, frame.header.corr)
2480                    || self
2481                        .supervisor
2482                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2483                    || self.cancel_activity_subscription(ctx, &frame)?
2484                    || self.cancel_catalog_subscription(ctx.connection_id, frame.header.corr)
2485                {
2486                    Ok(Vec::new())
2487                } else {
2488                    Ok(vec![control_error_frame(
2489                        &frame,
2490                        "unknown_subscription",
2491                        "no subscription has this correlation id",
2492                    )?])
2493                }
2494            }
2495            FrameType::Request => {
2496                // This additive operation is not part of the existing exhaustive
2497                // module-control enum. Probe the op before decoding that enum.
2498                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2499                let claim_present = op.as_ref().is_some_and(|probe| probe.op == "claim.present");
2500                if let Some(refusal) = self
2501                    .recovery_request_refusal(
2502                        ctx,
2503                        &frame,
2504                        op.as_ref().map(|probe| probe.op.as_str()),
2505                    )
2506                    .await?
2507                {
2508                    if claim_present {
2509                        self.reclaim_presented_expiry(&frame);
2510                        return self
2511                            .refuse_claim_connection(ctx, &frame, "config_gated")
2512                            .await;
2513                    }
2514                    return Ok(vec![refusal]);
2515                }
2516                if claim_present {
2517                    return self.handle_claim_present(ctx, &frame, first).await;
2518                }
2519                if op.as_ref().is_some_and(|probe| probe.op == "claim.mint") {
2520                    return self.handle_claim_mint(ctx, &frame).await;
2521                }
2522                if let Some(refusal) = self.registration_operation_refusal(
2523                    ctx,
2524                    &frame,
2525                    op.as_ref().map(|probe| probe.op.as_str()),
2526                )? {
2527                    return Ok(vec![refusal]);
2528                }
2529                if op
2530                    .as_ref()
2531                    .is_some_and(|probe| probe.op == ops::CATALOG_WATCH)
2532                {
2533                    return self.handle_catalog_watch(ctx, frame);
2534                }
2535                if op
2536                    .as_ref()
2537                    .is_some_and(|probe| probe.op == MODULE_TO_SUBC_OP_CATALOG_UPDATE)
2538                    && subc_protocol::provider_declarations::catalog_update_has_provider_declarations(&frame.body)
2539                        .unwrap_or(false)
2540                {
2541                    return Ok(vec![control_error_frame(
2542                        &frame,
2543                        error_codes::CATALOG_UPDATE_FROZEN_DECLARATIONS,
2544                        "provider_declarations are fixed at HELLO; restart the module to change them",
2545                    )?]);
2546                }
2547                if let Some(probe) = op
2548                    .as_ref()
2549                    .filter(|probe| config_gate::is_module_config_op(&probe.op))
2550                {
2551                    return self
2552                        .handle_module_config_request(ctx, frame, &probe.op)
2553                        .await;
2554                }
2555                if op
2556                    .as_ref()
2557                    .is_some_and(|probe| config_gate::is_config_op(&probe.op))
2558                {
2559                    return self.handle_config_gate_request(ctx, frame).await;
2560                }
2561                if op
2562                    .as_ref()
2563                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2564                {
2565                    return self.handle_activity_record(ctx, frame).await;
2566                }
2567                if op.as_ref().is_some_and(|probe| {
2568                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2569                }) {
2570                    return self.handle_activity_read(ctx, frame).await;
2571                }
2572                if op
2573                    .as_ref()
2574                    .is_some_and(|probe| probe.op == "operator.confirm")
2575                {
2576                    return self.handle_operator_confirm(ctx, frame);
2577                }
2578                if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2579                    return self
2580                        .observe_control_wait(
2581                            ctx,
2582                            None,
2583                            ops::APP_OPEN,
2584                            self.handle_app_open(ctx, frame),
2585                        )
2586                        .await;
2587                }
2588                if op
2589                    .as_ref()
2590                    .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2591                {
2592                    return self.handle_connection_describe(ctx, frame);
2593                }
2594                if op
2595                    .as_ref()
2596                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2597                {
2598                    return self.handle_scope_served_by(ctx.connection_id, frame);
2599                }
2600                if self
2601                    .forwarding
2602                    .module_endpoint_for_connection(ctx.connection_id)
2603                    .map_err(RouterError::Forwarding)?
2604                    .is_some()
2605                {
2606                    if !is_known_module_request_op(&frame.body) {
2607                        return Ok(vec![control_error_frame(
2608                            &frame,
2609                            "unsupported_control_frame",
2610                            "module-originated channel-0 REQUEST is not supported",
2611                        )?]);
2612                    }
2613                    let request = match parse_module_control_request_from_module(&frame.body) {
2614                        Ok(request) => request,
2615                        Err((err, ControlRequestBodyError::UnknownOp)) => {
2616                            return Ok(vec![control_error_frame(
2617                                &frame,
2618                                "unsupported_control_frame",
2619                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
2620                            )?])
2621                        }
2622                        Err((err, ControlRequestBodyError::InvalidBody)) => {
2623                            return Ok(vec![control_error_frame(
2624                                &frame,
2625                                "invalid_control_body",
2626                                format!("malformed module control body: {err}"),
2627                            )?])
2628                        }
2629                    };
2630                    let op = module_control_request_op(&request);
2631                    let corr = frame.header.corr;
2632                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
2633                    if let ModuleControlRequestFromModule::AgentCommands {
2634                        started,
2635                        finished,
2636                        known,
2637                    } = request
2638                    {
2639                        return self
2640                            .agent_declarations
2641                            .enqueue(
2642                                self,
2643                                ctx,
2644                                frame,
2645                                crate::resource_totals::Declaration {
2646                                    started,
2647                                    finished,
2648                                    known,
2649                                },
2650                            )
2651                            .await;
2652                    }
2653                    let result =
2654                        self.handle_module_control_request(ctx.connection_id, frame, request);
2655                    log_slow_control_dispatch(
2656                        dispatch_started_at,
2657                        op,
2658                        ctx.connection_id,
2659                        corr,
2660                        None,
2661                    );
2662                    return result;
2663                }
2664
2665                if is_known_module_request_op(&frame.body) {
2666                    let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2667                        .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2668                    let code = if is_agents {
2669                        "not_supervised"
2670                    } else if op
2671                        .as_ref()
2672                        .is_some_and(|probe| probe.op == subc_protocol::scope::ROUTE_AUTHORIZE_OP)
2673                    {
2674                        error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED
2675                    } else {
2676                        "not_registered"
2677                    };
2678                    return Ok(vec![control_error_frame(
2679                        &frame,
2680                        code,
2681                        "module-originated control requests require the module's own registered connection",
2682                    )?]);
2683                }
2684
2685                let request = match parse_client_control_request(&frame.body) {
2686                    Ok(request) => request,
2687                    Err((err, ControlRequestBodyError::UnknownOp)) => {
2688                        return Ok(vec![control_error_frame(
2689                            &frame,
2690                            "unknown_control_op",
2691                            format!("unknown client control op: {err}"),
2692                        )?])
2693                    }
2694                    Err((err, ControlRequestBodyError::InvalidBody)) => {
2695                        return Ok(vec![control_error_frame(
2696                            &frame,
2697                            "invalid_control_body",
2698                            format!("malformed client control body: {err}"),
2699                        )?])
2700                    }
2701                };
2702                let op = client_control_request_op(&request);
2703                let corr = frame.header.corr;
2704                log_control_dispatch_arrival(op, ctx.connection_id, corr);
2705                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2706                #[cfg(test)]
2707                if let Some(delay) = self.control_dispatch_delay {
2708                    tokio::time::sleep(delay).await;
2709                }
2710                let result = self
2711                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2712                    .await;
2713                if let Some(timings) = route_open_timings.as_mut() {
2714                    timings.finish();
2715                }
2716                log_slow_control_dispatch(
2717                    dispatch_started_at,
2718                    op,
2719                    ctx.connection_id,
2720                    corr,
2721                    route_open_timings.as_ref(),
2722                );
2723                result
2724            }
2725            FrameType::Push => {
2726                let Some(endpoint) = self
2727                    .forwarding
2728                    .module_endpoint_for_connection(ctx.connection_id)
2729                    .map_err(RouterError::Forwarding)?
2730                else {
2731                    return Ok(vec![control_error_frame(
2732                        &frame,
2733                        "unsupported_control_frame",
2734                        "client-originated channel-0 PUSH is not supported",
2735                    )?]);
2736                };
2737                self.handle_status_update(endpoint, frame)
2738            }
2739            FrameType::Response | FrameType::Error
2740                if self
2741                    .forwarding
2742                    .module_endpoint_for_connection(ctx.connection_id)
2743                    .map_err(RouterError::Forwarding)?
2744                    .is_some() =>
2745            {
2746                self.handle_module_relay_response(ctx.connection_id, frame)
2747            }
2748            ty => Ok(vec![control_error_frame(
2749                &frame,
2750                "unsupported_control_frame",
2751                format!("unsupported channel-0 frame {ty:?}"),
2752            )?]),
2753        }
2754    }
2755
2756    #[cfg(test)]
2757    fn pause_cleanup_for_test(&self, stage: u8) {
2758        let mut slot = self.cleanup_pause.lock().unwrap_or_else(|p| p.into_inner());
2759        if slot.as_ref().is_some_and(|(at, _)| *at == stage) {
2760            let (_, (reached, release)) = slot.take().unwrap();
2761            drop(slot);
2762            let _ = reached.send(());
2763            let _ = release.recv_timeout(Duration::from_secs(5));
2764        }
2765    }
2766
2767    /// Run `hook` once, in the next `route.open` that attests a launch nonce,
2768    /// right after the nonce is resolved.
2769    #[cfg(all(test, unix))]
2770    pub(crate) fn after_nonce_attestation_for_test(&self, hook: impl FnOnce() + Send + 'static) {
2771        *self
2772            .after_nonce_attestation
2773            .lock()
2774            .unwrap_or_else(|p| p.into_inner()) = Some(Box::new(hook));
2775    }
2776
2777    #[cfg(test)]
2778    pub(super) fn pause_after_nonce_attestation_for_test(&self) {
2779        let hook = self
2780            .after_nonce_attestation
2781            .lock()
2782            .unwrap_or_else(|p| p.into_inner())
2783            .take();
2784        if let Some(hook) = hook {
2785            hook();
2786        }
2787    }
2788
2789    #[cfg(test)]
2790    async fn pause_hello_ingress_for_test(&self) {
2791        let pause = self
2792            .hello_ingress_pause
2793            .lock()
2794            .unwrap_or_else(|p| p.into_inner())
2795            .take();
2796        if let Some((reached, release)) = pause {
2797            let _ = reached.send(());
2798            let _ = release.await;
2799        }
2800    }
2801
2802    pub fn cleanup_connection(
2803        &self,
2804        connection_id: ConnectionId,
2805    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2806        // The server loop joins its offloaded request tasks (route opens,
2807        // privileged requests and activity reads) before this cleanup runs, and
2808        // inline dispatch has ended. Two kinds of work are not joined. Agent
2809        // command declarations run on their queue's detached worker, so the
2810        // queue is closed first: every declaration still queued or validating
2811        // has its commit fence cancelled before anything below can change how
2812        // the connection resolves. A HELLO whose reader was dropped keeps
2813        // running on its blocking thread. It carries the channel spawn its
2814        // reader resolved at ingress instead of re-reading the index forgotten
2815        // below, so it is still judged as that channel's HELLO: the retired
2816        // spawn refuses it before it records anything, and a channel HELLO
2817        // never records a launch nonce, so it cannot leave this connection any
2818        // nonce authority.
2819        self.agent_declarations.close(connection_id);
2820        self.close_catalog_subscriptions(connection_id);
2821        #[cfg(test)]
2822        self.pause_cleanup_for_test(0);
2823        if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
2824            spawn.retire_registration_channel();
2825        }
2826        self.supervisor
2827            .forget_registration_connection(connection_id);
2828        #[cfg(test)]
2829        self.pause_cleanup_for_test(1);
2830        self.local_connections
2831            .lock()
2832            .unwrap_or_else(|p| p.into_inner())
2833            .remove(&connection_id);
2834        self.caller_endpoints
2835            .lock()
2836            .unwrap_or_else(|p| p.into_inner())
2837            .remove(&connection_id);
2838        self.build_class_warned
2839            .lock()
2840            .unwrap_or_else(|p| p.into_inner())
2841            .remove(&connection_id);
2842        self.observed_peers
2843            .lock()
2844            .unwrap_or_else(|p| p.into_inner())
2845            .remove(&connection_id);
2846        self.captured_peers
2847            .lock()
2848            .unwrap_or_else(|p| p.into_inner())
2849            .remove(&connection_id);
2850        self.config_desktop_stamps
2851            .lock()
2852            .unwrap_or_else(|p| p.into_inner())
2853            .remove(&connection_id);
2854        self.operator_apps
2855            .lock()
2856            .unwrap_or_else(|p| p.into_inner())
2857            .remove(&connection_id);
2858        self.forwarding
2859            .operator_confirms()
2860            .connection_closed(connection_id);
2861        let cleanup = self.cleanup_connection_with_end_reason(
2862            connection_id,
2863            RegistrationEndReason::ConnectionClosed,
2864        );
2865        // A draining writer can still be reached by a route until forwarding
2866        // cleanup finishes. Keep its claim sink visible to launch retirement.
2867        self.claims.connection_closed(connection_id);
2868        cleanup
2869    }
2870
2871    fn cleanup_connection_with_end_reason(
2872        &self,
2873        connection_id: ConnectionId,
2874        requested_reason: RegistrationEndReason,
2875    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2876        #[cfg(test)]
2877        if let Some(pause) = self
2878            .activity_request_pause
2879            .lock()
2880            .unwrap()
2881            .as_ref()
2882            .filter(|pause| pause.connection == connection_id)
2883        {
2884            pause
2885                .cleanup_started
2886                .store(true, std::sync::atomic::Ordering::Release);
2887        }
2888        if let Some(runtime) = &self.config_runtime {
2889            runtime.disconnected(connection_id);
2890        }
2891        self.agent_declarations.close(connection_id);
2892        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2893            self.registry
2894                .get_module_by_connection(connection_id)?
2895                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2896                .and_then(|module| module.registration_end_reason().ok().flatten())
2897                .unwrap_or(requested_reason)
2898        } else {
2899            requested_reason
2900        };
2901        let crash_closed = self
2902            .registry
2903            .get_module_by_connection(connection_id)?
2904            .and_then(|registration| {
2905                self.forwarding
2906                    .module_endpoint_for_connection(connection_id)
2907                    .ok()
2908                    .flatten()
2909                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2910                    .map(|routes| (registration.manifest.module_id, routes))
2911            });
2912        let crash_closed = crash_closed.map(|(module_id, routes)| {
2913            let terminal = match self.supervisor.get(&module_id) {
2914                None => false,
2915                Some(module) => match module.will_recover_after_connection_loss() {
2916                    Ok(will_recover) => !will_recover,
2917                    Err(err) => {
2918                        warn!(
2919                            %module_id,
2920                            error = %err,
2921                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2922                        );
2923                        false
2924                    }
2925                },
2926            };
2927            // The forwarding table gates all providers at the start of daemon
2928            // shutdown, before their connections are closed. An ordinary
2929            // module disconnect still reports crash if that gate is not set.
2930            let (reason, drained) = match self
2931                .forwarding
2932                .daemon_drain_result_for_connection(connection_id)
2933            {
2934                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2935                Ok(None) => (RouteCloseReason::Crash, false),
2936                Err(err) => {
2937                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2938                    (RouteCloseReason::Crash, false)
2939                }
2940            };
2941            (module_id, routes, reason, drained, terminal)
2942        });
2943        let registrations = self.deregister_connection(connection_id, end_reason);
2944        // Read before teardown: the spawn whose registration this connection
2945        // was. Its registration ending is an authority change for it.
2946        let ended_authority = self
2947            .forwarding
2948            .target_authority_for_connection(connection_id)
2949            .ok()
2950            .flatten();
2951        let cleanup = if crash_closed.is_some() {
2952            self.forwarding.cleanup_connection_counted(connection_id)
2953        } else {
2954            // A module connection's teardown needs the count of abandoned
2955            // route.bind relays for its route.closed notice below. Any other
2956            // connection, such as a client's, sends no such notice and needs
2957            // only its routes released, so it uses the route-only wrapper and
2958            // reports zero.
2959            self.forwarding
2960                .cleanup_connection(connection_id)
2961                .map(|released| crate::forwarding::ConnectionCleanup {
2962                    released,
2963                    abandoned_relays: 0,
2964                })
2965        };
2966        // The route.closed push waits for forwarding teardown because only
2967        // teardown knows how many pending route.bind relays it aborted. It still
2968        // goes out before the GOODBYEs for the released routes, and its targets
2969        // were captured above, before teardown removed those routes.
2970        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2971            let abandoned = cleanup
2972                .as_ref()
2973                .map_or(0, |cleanup| cleanup.abandoned_relays);
2974            send_route_control_pushes(
2975                &self.forwarding,
2976                routes,
2977                ClientControlPush::RouteClosed {
2978                    module_id,
2979                    channels: Vec::new(),
2980                    reason,
2981                    drained,
2982                    abandoned,
2983                    excluded_subscriptions: 0,
2984                    terminal: Some(terminal),
2985                },
2986            );
2987        }
2988        if let Ok(cleanup) = cleanup {
2989            self.emit_route_goodbyes(cleanup.released);
2990        }
2991        if let Some(record) = ended_authority {
2992            self.change_spawn_authority(&record);
2993        }
2994        // Signal the registration-release watch only now that BOTH registry and
2995        // forwarding teardown are done, so a supervisor waiting to spawn a
2996        // replacement never observes release while old routes still exist.
2997        if matches!(&registrations, Ok(r) if !r.is_empty()) {
2998            crate::supervise::notify_registration_release();
2999            self.capability_evaluator.wake_deadline_loop();
3000            self.refresh_capability_requirements();
3001        }
3002        self.supervisor.remove_spawn_subscribers(connection_id);
3003        self.activity_streams
3004            .lock()
3005            .unwrap_or_else(|p| p.into_inner())
3006            .retain(|(id, _), task| {
3007                if *id == connection_id {
3008                    task.abort();
3009                    false
3010                } else {
3011                    true
3012                }
3013            });
3014        // Sync authority dies with its connection, so the owner's next
3015        // connection can take it; the owner's scopes stay as they are.
3016        self.hello_launch_nonces
3017            .lock()
3018            .unwrap_or_else(|poisoned| poisoned.into_inner())
3019            .forget(connection_id);
3020        self.scopes
3021            .write()
3022            .unwrap_or_else(|poisoned| poisoned.into_inner())
3023            .release_connection(connection_id);
3024        registrations
3025    }
3026
3027    pub(crate) fn handle_route_goodbye(
3028        &self,
3029        connection_id: ConnectionId,
3030        route_channel: u16,
3031        route_epoch: u32,
3032    ) -> Result<bool, RouterError> {
3033        debug!(
3034            connection_id = connection_id.get(),
3035            route_channel, route_epoch, "handling route GOODBYE"
3036        );
3037        let RouteRelease::Removed(released_route) = self
3038            .forwarding
3039            .release_client_route(connection_id, route_channel, route_epoch)
3040            .map_err(RouterError::Forwarding)?
3041        else {
3042            return Ok(false);
3043        };
3044        self.emit_route_goodbyes(vec![released_route]);
3045        Ok(true)
3046    }
3047
3048    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
3049        scope_expiry::ScopeExpiryContext::new(self).emit_route_goodbyes(released_routes);
3050    }
3051
3052    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
3053    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
3054    /// own commit failed after the module had already accepted). Without this, a
3055    /// module that accepts late keeps a binding subc has torn down, so a later
3056    /// frame on that module channel could misdeliver if the channel is reused.
3057    ///
3058    /// Never closes the shared module connection on failure: a dropped notification
3059    /// only wastes a bounded amount of warm module-side state, which the module's
3060    /// own idle reaper reclaims. Only call this once the route.bind relay was
3061    /// actually enqueued to the module — if the relay send itself failed, the
3062    /// module never created a binding and there is nothing to tear down.
3063    fn send_abandoned_route_bind_goodbye(
3064        &self,
3065        module_sink: &crate::FrameSink,
3066        negotiated_ver: u8,
3067        module_channel: u16,
3068        module_epoch: u32,
3069    ) {
3070        let frame = match Frame::build_with_version(
3071            negotiated_ver,
3072            FrameType::Goodbye,
3073            control_flags(),
3074            module_channel,
3075            module_epoch,
3076            0,
3077            Vec::new(),
3078        ) {
3079            Ok(frame) => frame,
3080            Err(err) => {
3081                warn!(
3082                    route_channel = module_channel,
3083                    error = %err,
3084                    "failed to build GOODBYE for abandoned route.bind"
3085                );
3086                return;
3087            }
3088        };
3089        crate::forwarding::send_module_route_goodbye(
3090            &self.counters,
3091            module_sink,
3092            frame,
3093            None,
3094            "abandoned route.bind",
3095        );
3096    }
3097
3098    #[cfg(test)]
3099    fn handle_hello(
3100        &self,
3101        connection_id: ConnectionId,
3102        egress: Option<crate::router::FrameSink>,
3103        frame: Frame,
3104    ) -> Result<Vec<Frame>, RouterError> {
3105        let binding = self.claims.binding(connection_id);
3106        let channel_spawn = self.supervisor.registration_spawn(connection_id);
3107        self.handle_hello_checked(connection_id, egress, frame, false, binding, channel_spawn)
3108    }
3109
3110    fn handle_hello_checked(
3111        &self,
3112        connection_id: ConnectionId,
3113        sink: Option<crate::FrameSink>,
3114        frame: Frame,
3115        wait_for_gate: bool,
3116        prefix_binding: Option<claims::ClaimBinding>,
3117        channel_spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3118    ) -> Result<Vec<Frame>, RouterError> {
3119        let primary_sink = channel_spawn.as_ref().and_then(|_| sink.clone());
3120        #[cfg(test)]
3121        if prefix_binding.is_some() {
3122            self.claims.pause_prefix_hello_for_test(5);
3123            match self.claims.take_prefix_hello_failure_for_test() {
3124                Some("error") => {
3125                    return Err(RouterError::backend(
3126                        0,
3127                        frame.header.corr,
3128                        "injected HELLO failure",
3129                    ))
3130                }
3131                Some("panic") => panic!("injected HELLO task panic"),
3132                _ => {}
3133            }
3134        }
3135        if let Some(binding) = &prefix_binding {
3136            let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3137            let module = value
3138                .as_ref()
3139                .and_then(|value| value["manifest"]["module_id"].as_str());
3140            if !binding.admit_prefix_hello(module)
3141                || !self.prefix_binding_live(binding, connection_id)
3142                || !module.is_some_and(|module| {
3143                    self.supervisor
3144                        .prefix_claim_authorized(&binding.registration.module_id, module)
3145                })
3146                || binding.registration.check().is_err()
3147            {
3148                return Ok(vec![control_error_frame(
3149                    &frame,
3150                    "claim_prefix_refused",
3151                    "HELLO must name the prefix claim's exact module id once",
3152                )?]);
3153            }
3154        }
3155        struct RefuseUnregisteredChannel {
3156            spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3157            sink: Option<crate::FrameSink>,
3158        }
3159        impl Drop for RefuseUnregisteredChannel {
3160            fn drop(&mut self) {
3161                if let Some(spawn) = &self.spawn {
3162                    if !spawn.registered() {
3163                        if let Some(sink) = &self.sink {
3164                            sink.terminate_ingress();
3165                        }
3166                        spawn.retire_registration_channel();
3167                    }
3168                }
3169            }
3170        }
3171        let _channel_attempt = RefuseUnregisteredChannel {
3172            spawn: channel_spawn.clone(),
3173            sink: sink.clone(),
3174        };
3175        if let Some(spawn) = &channel_spawn {
3176            let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3177            if let Err(code) = spawn.admit_hello(
3178                value
3179                    .as_ref()
3180                    .and_then(|value| value["manifest"]["module_id"].as_str()),
3181            ) {
3182                return Ok(vec![control_error_frame(
3183                    &frame,
3184                    code,
3185                    "registration channel HELLO refused",
3186                )?]);
3187            }
3188        }
3189        debug!(
3190            connection_id = connection_id.get(),
3191            corr = frame.header.corr,
3192            "handling HELLO"
3193        );
3194        // A module connection has one identity for its entire lifetime. A second
3195        // registration would leave the old registry owner behind while replacing
3196        // its forwarding endpoint and launch nonce.
3197        if self
3198            .registry
3199            .get_module_by_connection(connection_id)
3200            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3201            .is_some()
3202        {
3203            return Ok(vec![control_error_frame(
3204                &frame,
3205                "invalid_hello",
3206                "connection is already registered as a module",
3207            )?]);
3208        }
3209        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
3210            Ok(value) => value,
3211            Err(err) => {
3212                return Ok(vec![control_error_frame(
3213                    &frame,
3214                    "invalid_hello",
3215                    format!("malformed HELLO body: {err}"),
3216                )?])
3217            }
3218        };
3219        if let Err(detail) = catalog::validate_hello_declarations(&hello_value) {
3220            return Ok(vec![control_error_frame(
3221                &frame,
3222                error_codes::INVALID_PROVIDER_DECLARATIONS,
3223                detail,
3224            )?]);
3225        }
3226        if let Some(module) = hello_value
3227            .get("manifest")
3228            .and_then(|value| value.get("module_id"))
3229            .and_then(|value| value.as_str())
3230        {
3231            let recovering = self
3232                .config_runtime
3233                .as_ref()
3234                .is_some_and(|owner| owner.check_recovery().unwrap_or(true));
3235            // Defence in depth: RuntimeOwner::module_is_gated in
3236            // config_gate/admission.rs already gates every id during recovery.
3237            // A registration channel still never uses the nonce fallback.
3238            if recovering
3239                && channel_spawn.as_ref().is_none_or(|spawn| {
3240                    !self.supervisor.has_live_verified_channel_binding(spawn)
3241                        || spawn.check().is_err()
3242                })
3243                && !(channel_spawn.is_none()
3244                    && hello_value
3245                        .get("launch_nonce")
3246                        .and_then(|value| value.as_str())
3247                        .is_some_and(|nonce| {
3248                            self.supervisor.spawned_consumer_authorized(module, nonce)
3249                                && self
3250                                    .supervisor
3251                                    .has_live_verified_program_binding(module, nonce)
3252                        }))
3253            {
3254                return Ok(vec![control_error_frame(
3255                    &frame,
3256                    "config_gated",
3257                    "recovery admits only previously bound supervised launches",
3258                )?]);
3259            }
3260            if self
3261                .config_runtime
3262                .as_ref()
3263                .is_some_and(|owner| owner.module_is_gated(module))
3264                && (channel_spawn
3265                    .as_ref()
3266                    .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3267                    || self.config_runtime.as_ref().is_some_and(|owner| {
3268                        !recovering && owner.check_module_hold(module).is_err()
3269                    }))
3270            {
3271                return Ok(vec![control_error_frame(
3272                    &frame,
3273                    "reserved_module",
3274                    "a gated identity belongs only to its supervised child",
3275                )?]);
3276            }
3277        }
3278        if channel_spawn.is_none() {
3279            if let (Some(module), Some(nonce)) = (
3280                hello_value
3281                    .get("manifest")
3282                    .and_then(|value| value.get("module_id"))
3283                    .and_then(|value| value.as_str()),
3284                hello_value
3285                    .get("launch_nonce")
3286                    .and_then(|value| value.as_str()),
3287            ) {
3288                let binding = self.supervisor.has_program_binding_record(module, nonce);
3289                let required = self
3290                    .config_runtime
3291                    .as_ref()
3292                    .is_some_and(|owner| owner.module_is_gated(module));
3293                if let Some(error) = (binding || required)
3294                    .then(|| {
3295                        self.supervisor
3296                            .check_current_program_binding(module, nonce)
3297                            .err()
3298                    })
3299                    .flatten()
3300                {
3301                    let code = if error.code == "program_refused" {
3302                        "program_refused"
3303                    } else {
3304                        "config_delivery_deferred"
3305                    };
3306                    self.request_refusal_close(
3307                        connection_id,
3308                        sink.as_ref(),
3309                        CloseReason::new(code, error.detail.clone()),
3310                    );
3311                    return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3312                }
3313            }
3314        }
3315        if let Err(err) = validate_hello_display_name(&hello_value) {
3316            return Ok(vec![control_error_frame(
3317                &frame,
3318                "invalid_manifest",
3319                err.to_string(),
3320            )?]);
3321        }
3322        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
3323            return Ok(vec![control_error_frame(
3324                &frame,
3325                "invalid_capability_grammar",
3326                err.to_string(),
3327            )?]);
3328        }
3329        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
3330            return Ok(vec![control_error_frame(
3331                &frame,
3332                "invalid_manifest",
3333                err.to_string(),
3334            )?]);
3335        }
3336        if let Err(err) = validate_hello_event_declarations(&hello_value) {
3337            return Ok(vec![control_error_frame(
3338                &frame,
3339                "invalid_event_declaration",
3340                err.to_string(),
3341            )?]);
3342        }
3343        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
3344            return Ok(vec![control_error_frame(
3345                &frame,
3346                "invalid_manifest",
3347                err.to_string(),
3348            )?]);
3349        }
3350        if let Some(provenance) = hello_value
3351            .get("manifest")
3352            .and_then(|manifest| manifest.get("provenance"))
3353        {
3354            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
3355                return Ok(vec![control_error_frame(
3356                    &frame,
3357                    "invalid_manifest",
3358                    format!("malformed manifest provenance: {err}"),
3359                )?]);
3360            }
3361        }
3362        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
3363            let module_id = hello_value["manifest"]["module_id"]
3364                .as_str()
3365                .unwrap_or("<unknown>");
3366            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
3367        }
3368        if let Err(reason) = validate_hello_operation_deferral(&hello_value) {
3369            let module_id = hello_value
3370                .get("manifest")
3371                .and_then(|m| m.get("module_id"))
3372                .and_then(serde_json::Value::as_str)
3373                .unwrap_or("unknown");
3374            warn!(module_id, %reason, "dropping operation_deferral from HELLO");
3375        }
3376        if let Err(reason) = validate_hello_flow_operations(&hello_value) {
3377            let module_id = hello_value["manifest"]["module_id"]
3378                .as_str()
3379                .unwrap_or("<unknown>");
3380            warn!(module_id, %reason, "dropping flow_operations from HELLO");
3381        }
3382        if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
3383            let module_id = hello_value["manifest"]["module_id"]
3384                .as_str()
3385                .unwrap_or("<unknown>");
3386            warn!(module_id, %reason, "dropping confirm_operations from HELLO");
3387        }
3388        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
3389            Ok(hello) => hello,
3390            Err(err) => {
3391                return Ok(vec![control_error_frame(
3392                    &frame,
3393                    "invalid_hello",
3394                    format!("malformed HELLO body: {err}"),
3395                )?])
3396            }
3397        };
3398
3399        if hello.protocol_ver != hello.manifest.protocol_ver {
3400            return Ok(vec![control_error_frame(
3401                &frame,
3402                "invalid_manifest",
3403                format!(
3404                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
3405                    hello.protocol_ver, hello.manifest.protocol_ver
3406                ),
3407            )?]);
3408        }
3409
3410        if hello.manifest.module_id.trim().is_empty() {
3411            return Ok(vec![control_error_frame(
3412                &frame,
3413                "invalid_manifest",
3414                "manifest module_id must not be empty",
3415            )?]);
3416        }
3417
3418        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
3419            Ok(negotiated_ver) => negotiated_ver,
3420            Err(message) => {
3421                return Ok(vec![control_error_frame(
3422                    &frame,
3423                    "version_unsupported",
3424                    message,
3425                )?])
3426            }
3427        };
3428
3429        // Registration and forwarding publication use the same module gate as
3430        // stamp, unstamp and spawning. Recheck after taking it; parsing may
3431        // precede those transitions. A program-refusal hold blocks future
3432        // spawns, but does not take this gate or revoke a HELLO whose supervised
3433        // child's image has already passed the check.
3434        let _publication_gate = if let Some(owner) = &self.config_runtime {
3435            let gate = owner
3436                .module_gate(&hello.manifest.module_id)
3437                .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3438            Some(if wait_for_gate {
3439                gate.blocking_lock_owned()
3440            } else {
3441                match gate.try_lock_owned() {
3442                    Ok(guard) => guard,
3443                    Err(_) => {
3444                        return Ok(vec![control_error_frame(
3445                            &frame,
3446                            "config_delivery_deferred",
3447                            "module admission is busy",
3448                        )?])
3449                    }
3450                }
3451            })
3452        } else {
3453            None
3454        };
3455        #[cfg(test)]
3456        if let Some(hold) = &self.hello_before_publication_hold {
3457            hold.wait();
3458            hold.wait();
3459        }
3460        let module = &hello.manifest.module_id;
3461        let required = self
3462            .config_runtime
3463            .as_ref()
3464            .is_some_and(|owner| owner.module_is_gated(module));
3465        let recovering = self
3466            .config_runtime
3467            .as_ref()
3468            .is_some_and(|owner| owner.is_recovering());
3469        // Both entry paths (handle_control -> handle_hello and
3470        // handle_control_frame_timed -> spawn_blocking) enter this same
3471        // handle_hello_checked function. Its fresh, fail-closed store check
3472        // precedes parsing and this publication-gate check; this cached read
3473        // also observes any recovery transition published during that work.
3474        // The nonce-connection-only fallback is defence in depth alongside
3475        // RuntimeOwner::module_is_gated's all-id recovery gate.
3476        if recovering
3477            && channel_spawn.as_ref().is_none_or(|spawn| {
3478                !self.supervisor.has_live_verified_channel_binding(spawn) || spawn.check().is_err()
3479            })
3480            && !(channel_spawn.is_none()
3481                && hello.launch_nonce.as_deref().is_some_and(|nonce| {
3482                    self.supervisor.spawned_consumer_authorized(module, nonce)
3483                        && self
3484                            .supervisor
3485                            .has_live_verified_program_binding(module, nonce)
3486                }))
3487        {
3488            return Ok(vec![control_error_frame(
3489                &frame,
3490                "config_gated",
3491                "recovery admits only previously bound supervised launches",
3492            )?]);
3493        }
3494        if required
3495            && (channel_spawn
3496                .as_ref()
3497                .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3498                || self
3499                    .config_runtime
3500                    .as_ref()
3501                    .is_some_and(|owner| !recovering && owner.check_module_hold(module).is_err()))
3502        {
3503            return Ok(vec![control_error_frame(
3504                &frame,
3505                "reserved_module",
3506                "gated admission requires the current supervised child's registration channel and verified image binding",
3507            )?]);
3508        }
3509        if let Some(spawn) = &channel_spawn {
3510            if let Err(code) = spawn.check() {
3511                return Ok(vec![control_error_frame(
3512                    &frame,
3513                    code,
3514                    "registration launch is no longer live",
3515                )?]);
3516            }
3517        } else if let Some(nonce) = hello.launch_nonce.as_deref() {
3518            if required || self.supervisor.has_program_binding_record(module, nonce) {
3519                if let Err(error) = self.supervisor.check_current_program_binding(module, nonce) {
3520                    let code = if error.code == "program_refused" {
3521                        "program_refused"
3522                    } else {
3523                        "config_delivery_deferred"
3524                    };
3525                    self.request_refusal_close(
3526                        connection_id,
3527                        sink.as_ref(),
3528                        CloseReason::new(code, error.detail.clone()),
3529                    );
3530                    return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3531                }
3532            }
3533        }
3534
3535        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
3536        // swap is open for this id, the only HELLO admitted as a second process
3537        // is the one carrying the candidate's launch nonce (the swap token), and
3538        // it registers into the candidate slot rather than being refused as a
3539        // duplicate. Run after the reserved gate, a reserved module's candidate
3540        // would be refused `reserved_module` for presenting a nonce that gate
3541        // does not know. See `SupervisorHandle::swap_hello_admission`.
3542        let swap_admission = match &channel_spawn {
3543            Some(spawn) if spawn.is_candidate() => SwapHelloAdmission::Candidate,
3544            Some(_) => SwapHelloAdmission::NotSwapping,
3545            None if prefix_binding.is_some() => SwapHelloAdmission::NotSwapping,
3546            None => self
3547                .supervisor
3548                .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref()),
3549        };
3550        if swap_admission == SwapHelloAdmission::ProtocolNone {
3551            warn!(
3552                module_id = %hello.manifest.module_id,
3553                connection_id = connection_id.get(),
3554                "HELLO refused: the module_id belongs to a protocol:none module, which never registers"
3555            );
3556            // The code stays `swap_token_invalid` for existing callers; only the
3557            // explanation names the actual reason.
3558            return Ok(vec![control_error_frame(
3559                &frame,
3560                "swap_token_invalid",
3561                format!(
3562                    "module_id '{}' is a protocol:none module; a protocol:none module may not register",
3563                    hello.manifest.module_id
3564                ),
3565            )?]);
3566        }
3567        if swap_admission == SwapHelloAdmission::ChannelRequired {
3568            warn!(
3569                module_id = %hello.manifest.module_id,
3570                connection_id = connection_id.get(),
3571                "HELLO refused: the module_id is gated and the HELLO did not arrive over its registration channel"
3572            );
3573            // The code stays `swap_token_invalid` for existing callers; only the
3574            // explanation names the actual reason.
3575            return Ok(vec![control_error_frame(
3576                &frame,
3577                "swap_token_invalid",
3578                format!(
3579                    "module_id '{}' is gated; it must register over its supervised registration channel",
3580                    hello.manifest.module_id
3581                ),
3582            )?]);
3583        }
3584        if swap_admission == SwapHelloAdmission::Refused {
3585            warn!(
3586                module_id = %hello.manifest.module_id,
3587                connection_id = connection_id.get(),
3588                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
3589            );
3590            return Ok(vec![control_error_frame(
3591                &frame,
3592                "swap_token_invalid",
3593                format!(
3594                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
3595                    hello.manifest.module_id
3596                ),
3597            )?]);
3598        }
3599        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
3600
3601        // A swap candidate proves its identity with the candidate nonce above.
3602        // Other registrations receive this catalog fact only when reserved-module
3603        // policy required and accepted a nonce; unreserved registrations require
3604        // no nonce check.
3605        let verification = if swap_candidate && channel_spawn.is_none() {
3606            ReservedHelloVerification {
3607                nonce_verified: true,
3608                prefix_owner: None,
3609            }
3610        } else {
3611            let verification = if let Some(binding) = &prefix_binding {
3612                if !self.supervisor.prefix_claim_authorized(
3613                    &binding.registration.module_id,
3614                    &hello.manifest.module_id,
3615                ) {
3616                    return Ok(vec![control_error_frame(
3617                        &frame,
3618                        "claim_prefix_refused",
3619                        "prefix ownership changed",
3620                    )?]);
3621                }
3622                Ok(ReservedHelloVerification {
3623                    nonce_verified: true,
3624                    prefix_owner: Some(binding.registration.module_id.clone()),
3625                })
3626            } else if channel_spawn.is_some() {
3627                self.supervisor
3628                    .channel_hello_verification(&hello.manifest.module_id)
3629            } else {
3630                self.supervisor.reserved_hello_verification(
3631                    &hello.manifest.module_id,
3632                    hello.launch_nonce.as_deref(),
3633                )
3634            };
3635            match verification {
3636                Ok(verified) => verified,
3637                Err(rejection) => {
3638                    let message = match rejection {
3639                        ReservedHelloRejection::Exact { module_id } => format!(
3640                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
3641                        ),
3642                        ReservedHelloRejection::Prefix {
3643                            prefix,
3644                            owner_module_id,
3645                        } => format!(
3646                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
3647                            hello.manifest.module_id
3648                        ),
3649                    };
3650                    return Ok(vec![control_error_frame(
3651                        &frame,
3652                        "reserved_module",
3653                        message,
3654                    )?]);
3655                }
3656            }
3657        };
3658
3659        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
3660            &hello.manifest.module_id,
3661            hello.manifest.capabilities.as_ref(),
3662        );
3663        if let Some(refusal) = reserved_capability_refusals.first() {
3664            let capability = refusal.capability.clone();
3665            let bound_module = refusal.claimants[0].clone();
3666            log_duplicate_claim_events(reserved_capability_refusals);
3667            return Ok(vec![control_error_frame(
3668                &frame,
3669                "reserved_capability",
3670                format!(
3671                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3672                    capability, bound_module, hello.manifest.module_id
3673                ),
3674            )?]);
3675        }
3676
3677        // A connection that already opened client routes must not also register as
3678        // a module: cleanup would then release only one side and leak the other.
3679        if self
3680            .forwarding
3681            .connection_has_client_routes(connection_id)
3682            .map_err(RouterError::Forwarding)?
3683        {
3684            return Ok(vec![control_error_frame(
3685                &frame,
3686                "invalid_hello",
3687                "connection has open client routes and cannot also register as a module",
3688            )?]);
3689        }
3690
3691        // Kept for scope sync authority, which goes only to the connection that
3692        // presented the module's current launch nonce. Recorded before the
3693        // registration is attempted: a connection whose registration then fails
3694        // has no registration, so it cannot sync anyway, and cleanup forgets it.
3695        // A registration channel's authority is the channel itself, never a
3696        // nonce it also presented, and a prefix registration's is its claim:
3697        // neither records one, so neither can fall back to nonce authority.
3698        self.hello_launch_nonces
3699            .lock()
3700            .unwrap_or_else(|poisoned| poisoned.into_inner())
3701            .record(
3702                connection_id,
3703                if prefix_binding.is_some() || channel_spawn.is_some() {
3704                    None
3705                } else {
3706                    hello.launch_nonce.as_deref()
3707                },
3708            );
3709        let control_ops = effective_module_control_ops(hello.control_ops);
3710        let prefix_submodule = verification.prefix_owner.is_some();
3711        // Built before anything is registered so an encoding failure leaves no
3712        // registry or forwarding state behind.
3713        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
3714        #[cfg(test)]
3715        struct PublicationFinished(Arc<std::sync::Barrier>);
3716        #[cfg(test)]
3717        impl Drop for PublicationFinished {
3718            fn drop(&mut self) {
3719                self.0.wait();
3720            }
3721        }
3722        #[cfg(test)]
3723        let _publication_finished = self.hello_after_launch_check_hold.as_ref().map(|hold| {
3724            hold.wait();
3725            hold.wait();
3726            PublicationFinished(hold.clone())
3727        });
3728        let publication_spawn = channel_spawn
3729            .as_ref()
3730            .or_else(|| prefix_binding.as_ref().map(|binding| &binding.registration));
3731        #[cfg(test)]
3732        if prefix_binding.is_some() {
3733            self.claims.pause_prefix_hello_for_test(7);
3734        }
3735        let _registration_publication = match publication_spawn {
3736            Some(spawn) => Some(match spawn.begin_publication() {
3737                Ok(guard) => guard,
3738                Err(code) => {
3739                    return Ok(vec![control_error_frame(
3740                        &frame,
3741                        code,
3742                        "registration ended before HELLO publication",
3743                    )?])
3744                }
3745            }),
3746            None => None,
3747        };
3748        let (_prefix_live, _prefix_ownership) = if let Some(binding) = &prefix_binding {
3749            let live = self.claims.lock_prefix_binding(binding, connection_id);
3750            let ownership = self.supervisor.prefix_claim_publication(
3751                &binding.registration.module_id,
3752                &hello.manifest.module_id,
3753            );
3754            if live.is_none()
3755                || ownership.is_none()
3756                || !self.claim_primary_registered(binding.spawn, &binding.registration)
3757                || binding.prefix_module_id() != Some(hello.manifest.module_id.as_str())
3758            {
3759                return Ok(vec![control_error_frame(
3760                    &frame,
3761                    "claim_prefix_refused",
3762                    "prefix authority ended before publication",
3763                )?]);
3764            }
3765            (live, ownership)
3766        } else {
3767            (None, None)
3768        };
3769        if let Some(spawn) = publication_spawn {
3770            self.forwarding
3771                .bind_spawn(connection_id, spawn)
3772                .map_err(RouterError::Forwarding)?;
3773        }
3774        // The spawn whose own registration this HELLO is, for `target` on
3775        // the route.open reply: its channel, or the launch its nonce was
3776        // minted for. A prefix child's HELLO names no spawn of its own. It is
3777        // only looked up here. The connection is bound to it and its
3778        // requested declarations are frozen once the registry has accepted
3779        // this HELLO (`admit_hello_authority`), so a HELLO refused as a
3780        // duplicate, or for any other reason, leaves the spawn untouched: its
3781        // closing connection is then no registration of the spawn and ends
3782        // none of the spawn's routes.
3783        let hello_authority = match (&channel_spawn, &prefix_binding) {
3784            (Some(spawn), _) => Some(spawn.authority.clone()),
3785            (None, None) => hello.launch_nonce.as_deref().and_then(|nonce| {
3786                self.supervisor
3787                    .spawned_consumer_record(&hello.manifest.module_id, nonce)
3788            }),
3789            (None, Some(_)) => None,
3790        }
3791        .map(|record| HelloAuthority {
3792            record,
3793            requested: hello
3794                .manifest
3795                .provider_declarations
3796                .clone()
3797                .unwrap_or_default(),
3798            froze: Default::default(),
3799        });
3800        #[cfg(test)]
3801        if prefix_binding.is_some() {
3802            self.claims.pause_prefix_hello_for_test(8);
3803        }
3804        if swap_candidate {
3805            let module_id = hello.manifest.module_id.clone();
3806            let mut differing_declarations = Default::default();
3807            let reply = self.register_swap_candidate(
3808                connection_id,
3809                sink,
3810                &frame,
3811                hello.manifest,
3812                negotiated_ver,
3813                control_ops,
3814                hello_ack,
3815                hello_authority.as_ref(),
3816                &mut differing_declarations,
3817                #[cfg(windows)]
3818                hello.launch_nonce.as_deref(),
3819            )?;
3820            if let Some(spawn) = &channel_spawn {
3821                if self
3822                    .registry
3823                    .get_module_by_connection(connection_id)
3824                    .ok()
3825                    .flatten()
3826                    .is_some()
3827                {
3828                    if let Some(sink) = primary_sink {
3829                        spawn.bind_primary_sink(sink);
3830                    }
3831                    spawn.hello_accepted();
3832                }
3833            } else if self
3834                .registry
3835                .get_module_by_connection(connection_id)
3836                .ok()
3837                .flatten()
3838                .is_some()
3839            {
3840                self.supervisor
3841                    .retire_unused_registration_channel(&module_id, true);
3842            }
3843            if self
3844                .registry
3845                .get_module_by_connection(connection_id)
3846                .ok()
3847                .flatten()
3848                .is_some()
3849            {
3850                self.file_development_widening(
3851                    hello_authority.map(|authority| authority.record),
3852                    differing_declarations,
3853                );
3854            }
3855            return Ok(reply);
3856        }
3857        // Registered unpublished: the module id is claimed, so a duplicate HELLO
3858        // is refused, but `catalog.list` does not show the module and the
3859        // catalog generation does not move until it is published below, once
3860        // its forwarding endpoint exists. Listing it now would advertise a
3861        // module that `route.open` still refuses for having no forwarding
3862        // connection.
3863        let registration = match self
3864            .registry
3865            .register_unpublished_verified_with_control_ops(
3866                hello.manifest,
3867                negotiated_ver,
3868                connection_id,
3869                control_ops,
3870                verification.nonce_verified,
3871            ) {
3872            Ok(registration) => registration,
3873            Err(RegistryError::DuplicateModuleId { module_id }) => {
3874                return Ok(vec![control_error_frame(
3875                    &frame,
3876                    "duplicate_module_id",
3877                    format!(
3878                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
3879                    ),
3880                )?])
3881            }
3882            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3883                return Ok(vec![control_error_frame(
3884                    &frame,
3885                    "invalid_module_id",
3886                    err.to_string(),
3887                )?])
3888            }
3889            Err(err) => {
3890                return Ok(vec![control_error_frame(
3891                    &frame,
3892                    "registry_error",
3893                    err.to_string(),
3894                )?])
3895            }
3896        };
3897        let differing_declarations =
3898            match self.admit_hello_authority(connection_id, hello_authority.as_ref()) {
3899                Ok(differing) => differing,
3900                Err(err) => {
3901                    if matches!(
3902                        self.rollback_unpublished_hello(connection_id),
3903                        Ok(r) if !r.is_empty()
3904                    ) {
3905                        crate::supervise::notify_registration_release();
3906                    }
3907                    return Err(RouterError::Forwarding(err));
3908                }
3909            };
3910        #[cfg(test)]
3911        self.hold_between_registry_and_forwarding();
3912
3913        let mut reply = Vec::new();
3914        // Registry readers must wait from before the ACK can be read until
3915        // publication. Nest forwarding inside the registry lock, never reverse.
3916        let publication = self
3917            .registry
3918            .publish_registration_after(connection_id, || {
3919                if let Some(sink) = sink {
3920                    // The forwarding table's module store is also the daemon-to-module
3921                    // control-RPC lane, so every HELLO gets a live endpoint even when the
3922                    // manifest has no routable provider role. Non-routable modules still
3923                    // cannot receive route.bind in production: `handle_route_open` checks
3924                    // the registry manifest with `target_has_required_role` before the
3925                    // only production call to `begin_route_bind_relay_for` below that
3926                    // route.open path. The remaining direct relay callers are unit tests
3927                    // and benchmark harnesses that construct forwarding state explicitly.
3928                    //
3929                    // The HELLO_ACK is queued by the forwarding table itself, before the
3930                    // endpoint becomes visible, and is NOT returned as a reply. A module
3931                    // reads HELLO_ACK first and exits on anything else; a reply is only
3932                    // written after this handler returns, by which time a route.open on
3933                    // another connection could already have queued a route.bind request
3934                    // for this module ahead of it.
3935                    let concurrency = manifest_concurrency(&registration.manifest);
3936                    self.forwarding.register_module_connection_acked(
3937                        connection_id,
3938                        registration.manifest.module_id.clone(),
3939                        negotiated_ver,
3940                        concurrency,
3941                        sink,
3942                        crate::forwarding::ModuleHelloAck {
3943                            frame: hello_ack,
3944                            verified_prefix_owner: verification.prefix_owner,
3945                        },
3946                    )?;
3947                } else {
3948                    // No sink means no forwarding endpoint, so nothing can be routed
3949                    // ahead of the ack; it goes out as the reply.
3950                    reply.push(hello_ack);
3951                }
3952                #[cfg(test)]
3953                self.hold_between_ack_and_publication();
3954                Ok(())
3955            })
3956            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3957        let published = match publication {
3958            Ok(published) => published,
3959            Err(err) => {
3960                // The registry lock has been released, so rollback can take it
3961                // again. No forwarding endpoint or route was published. Keep
3962                // forwarding's weak connection-to-spawn association until the
3963                // connection's read loop ends and removes its remaining routes.
3964                // Removing this unpublished entry does not advance the catalog
3965                // generation.
3966                if matches!(
3967                    self.rollback_unpublished_hello(connection_id),
3968                    Ok(r) if !r.is_empty()
3969                ) {
3970                    crate::supervise::notify_registration_release();
3971                }
3972                self.refuse_hello_authority(connection_id, hello_authority.as_ref());
3973                return Ok(vec![control_error_frame(
3974                    &frame,
3975                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
3976                        "invalid_hello"
3977                    } else {
3978                        forwarding_error_code(&err)
3979                    },
3980                    err.to_string(),
3981                )?]);
3982            }
3983        };
3984        if !published {
3985            debug!(
3986                module_id = %registration.manifest.module_id,
3987                connection_id = connection_id.get(),
3988                "registration was removed before it could be published"
3989            );
3990            self.refuse_hello_authority(connection_id, hello_authority.as_ref());
3991            return Ok(reply);
3992        }
3993
3994        drop(_prefix_ownership);
3995        drop(_prefix_live);
3996
3997        // Exposure over assumption: Concurrency's serde default is pinned to the
3998        // pre-field behavior (ModuleManaged), so a management surface that is
3999        // genuinely Serial and just never declared it inherits concurrent
4000        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
4001        // turns "no module has been bitten yet" into the checkable claim "no
4002        // module is exposed" -- one read of the boot log instead of a fleet
4003        // audit. Detected from the raw HELLO bytes because the serde default
4004        // deliberately erases the absent/declared distinction from the type.
4005        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
4006            info!(
4007                module_id = %registration.manifest.module_id,
4008                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
4009            );
4010        }
4011        if registration.manifest.registration_channel == Some(false)
4012            && self
4013                .supervisor
4014                .module_is_gated(&registration.manifest.module_id)
4015            && self
4016                .config_runtime
4017                .as_ref()
4018                .and_then(|owner| owner.registration_declaration(&registration.manifest.module_id))
4019                .as_deref()
4020                == Some("declared")
4021        {
4022            warn!(module_id = %registration.manifest.module_id,
4023                "HELLO registration_channel is false despite the approved channel declaration; registration remains admitted");
4024        }
4025
4026        self.apply_registration_capabilities(&registration);
4027
4028        self.supervisor
4029            .record_activity_registration(&registration.manifest.module_id);
4030
4031        info!(
4032            module_id = %registration.manifest.module_id,
4033            module_version = %registration.manifest.module_version,
4034            negotiated_ver,
4035            routable_provider = manifest_provides_routable_role(&registration.manifest),
4036            connection_id = connection_id.get(),
4037            "module registered"
4038        );
4039
4040        if let Some(spawn) = &channel_spawn {
4041            if let Some(sink) = primary_sink {
4042                spawn.bind_primary_sink(sink);
4043            }
4044            spawn.hello_accepted();
4045        } else if !prefix_submodule {
4046            self.supervisor
4047                .retire_unused_registration_channel(&registration.manifest.module_id, false);
4048        }
4049        self.file_development_widening(
4050            hello_authority.map(|authority| authority.record),
4051            differing_declarations,
4052        );
4053        #[cfg(windows)]
4054        if let Some(nonce) = hello.launch_nonce.as_deref() {
4055            self.supervisor
4056                .program_hello_accepted(&registration.manifest.module_id, nonce);
4057        }
4058
4059        #[cfg(test)]
4060        if let Some(hold) = &self.hello_inside_publication_hold {
4061            hold.wait();
4062            hold.wait();
4063        }
4064
4065        Ok(reply)
4066    }
4067
4068    /// Register a HELLO the swap gate admitted into the candidate slot of the
4069    /// registry and of forwarding, where it is reachable over its own
4070    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
4071    /// nothing routes to it until the supervisor cuts over.
4072    ///
4073    /// Registry first (unpublished), then forwarding, then publication, the
4074    /// same order as an ordinary HELLO; a forwarding failure removes the
4075    /// registry entry again. The swap supervisor treats a published, ready
4076    /// candidate as registered and promotes it, and promotion needs the
4077    /// candidate's forwarding endpoint, so the candidate is published only
4078    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
4079    /// recompute capability verdicts or run the capability deny census
4080    /// (`enforce_capability_denies`): both are about the module in the active
4081    /// slot, which `route.open` can target. A candidate sits in the candidate
4082    /// slot, so it neither provides capabilities to other modules nor can be
4083    /// routed to until promotion moves it into the active slot.
4084    #[allow(clippy::too_many_arguments)]
4085    fn register_swap_candidate(
4086        &self,
4087        connection_id: ConnectionId,
4088        sink: Option<crate::FrameSink>,
4089        frame: &Frame,
4090        manifest: ModuleManifest,
4091        negotiated_ver: u8,
4092        control_ops: Vec<String>,
4093        hello_ack: Frame,
4094        hello_authority: Option<&HelloAuthority>,
4095        differing_declarations: &mut ProviderDeclarations,
4096        #[cfg(windows)] launch_nonce: Option<&str>,
4097    ) -> Result<Vec<Frame>, RouterError> {
4098        let module_id = manifest.module_id.clone();
4099        // Closing a swap and publishing its candidate must be ordered. The
4100        // spawn role proves which slot is intended, not that the slot is open.
4101        self.supervisor
4102            .with_swap_candidate_admission(&module_id, || {
4103                let reply = self.publish_swap_candidate(
4104                    connection_id,
4105                    sink,
4106                    frame,
4107                    manifest,
4108                    negotiated_ver,
4109                    control_ops,
4110                    hello_ack,
4111                    hello_authority,
4112                    differing_declarations,
4113                    #[cfg(windows)]
4114                    launch_nonce,
4115                );
4116                let admitted = self
4117                    .registry
4118                    .get_candidate(&module_id)
4119                    .ok()
4120                    .flatten()
4121                    .is_some_and(|candidate| candidate.connection_id == connection_id);
4122                (reply, admitted)
4123            })
4124            .unwrap_or_else(|| {
4125                Ok(vec![control_error_frame(
4126                    frame,
4127                    "swap_token_invalid",
4128                    "swap ended before candidate HELLO publication",
4129                )?])
4130            })
4131    }
4132
4133    #[allow(clippy::too_many_arguments)]
4134    fn publish_swap_candidate(
4135        &self,
4136        connection_id: ConnectionId,
4137        sink: Option<crate::FrameSink>,
4138        frame: &Frame,
4139        manifest: ModuleManifest,
4140        negotiated_ver: u8,
4141        control_ops: Vec<String>,
4142        hello_ack: Frame,
4143        hello_authority: Option<&HelloAuthority>,
4144        differing_declarations: &mut ProviderDeclarations,
4145        #[cfg(windows)] launch_nonce: Option<&str>,
4146    ) -> Result<Vec<Frame>, RouterError> {
4147        let module_id = manifest.module_id.clone();
4148        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
4149            manifest,
4150            negotiated_ver,
4151            connection_id,
4152            control_ops,
4153        ) {
4154            Ok(registration) => registration,
4155            Err(RegistryError::DuplicateModuleId { module_id }) => {
4156                return Ok(vec![control_error_frame(
4157                    frame,
4158                    "duplicate_module_id",
4159                    format!(
4160                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
4161                    ),
4162                )?])
4163            }
4164            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
4165                return Ok(vec![control_error_frame(
4166                    frame,
4167                    "invalid_module_id",
4168                    err.to_string(),
4169                )?])
4170            }
4171            Err(err) => {
4172                return Ok(vec![control_error_frame(
4173                    frame,
4174                    "registry_error",
4175                    err.to_string(),
4176                )?])
4177            }
4178        };
4179        *differing_declarations = match self.admit_hello_authority(connection_id, hello_authority) {
4180            Ok(differing) => differing,
4181            Err(err) => {
4182                if matches!(
4183                    self.rollback_unpublished_hello(connection_id),
4184                    Ok(r) if !r.is_empty()
4185                ) {
4186                    crate::supervise::notify_registration_release();
4187                }
4188                return Err(RouterError::Forwarding(err));
4189            }
4190        };
4191        #[cfg(test)]
4192        self.hold_between_registry_and_forwarding();
4193        let mut reply = Vec::new();
4194        let publication = self
4195            .registry
4196            .publish_registration_after(connection_id, || {
4197                if let Some(sink) = sink {
4198                    // Same ordering as an ordinary HELLO: queue the ACK before
4199                    // inserting the endpoint, and keep registry readers excluded
4200                    // until the candidate is published.
4201                    let concurrency = manifest_concurrency(&registration.manifest);
4202                    self.forwarding.register_candidate_module_connection_acked(
4203                        connection_id,
4204                        module_id.clone(),
4205                        negotiated_ver,
4206                        concurrency,
4207                        sink,
4208                        hello_ack,
4209                    )?;
4210                } else {
4211                    reply.push(hello_ack);
4212                }
4213                #[cfg(test)]
4214                self.hold_between_ack_and_publication();
4215                Ok(())
4216            })
4217            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4218        let published = match publication {
4219            Ok(published) => published,
4220            Err(err) => {
4221                if matches!(
4222                    self.rollback_unpublished_hello(connection_id),
4223                    Ok(r) if !r.is_empty()
4224                ) {
4225                    crate::supervise::notify_registration_release();
4226                }
4227                self.refuse_hello_authority(connection_id, hello_authority);
4228                return Ok(vec![control_error_frame(
4229                    frame,
4230                    forwarding_error_code(&err),
4231                    err.to_string(),
4232                )?]);
4233            }
4234        };
4235        if !published {
4236            debug!(
4237                module_id = %module_id,
4238                connection_id = connection_id.get(),
4239                "swap candidate registration was removed before it could be published"
4240            );
4241            self.refuse_hello_authority(connection_id, hello_authority);
4242            return Ok(reply);
4243        }
4244        info!(
4245            module_id = %module_id,
4246            module_version = %registration.manifest.module_version,
4247            negotiated_ver,
4248            ready = registration.ready,
4249            connection_id = connection_id.get(),
4250            "swap candidate registered; not routable until cutover"
4251        );
4252        #[cfg(windows)]
4253        if let Some(nonce) = launch_nonce {
4254            self.supervisor.program_hello_accepted(&module_id, nonce);
4255        }
4256        Ok(reply)
4257    }
4258
4259    /// Bind `connection` to the spawn its HELLO registers as and freeze the
4260    /// spawn's requested provider declarations. Runs only once the registry
4261    /// has accepted the HELLO, before its endpoint is published, so a route
4262    /// reserved on the endpoint always finds the spawn. Returns the requested
4263    /// entries that differ from the saved approvals, for a widening request.
4264    fn admit_hello_authority(
4265        &self,
4266        connection: ConnectionId,
4267        authority: Option<&HelloAuthority>,
4268    ) -> Result<ProviderDeclarations, ForwardingError> {
4269        let Some(authority) = authority else {
4270            return Ok(ProviderDeclarations::new());
4271        };
4272        let froze = authority
4273            .record
4274            .freeze_declarations(authority.requested.clone());
4275        authority
4276            .froze
4277            .store(froze.is_some(), std::sync::atomic::Ordering::Relaxed);
4278        self.forwarding
4279            .bind_target_authority(connection, authority.record.clone())
4280            .inspect_err(|_| self.refuse_hello_authority(connection, Some(authority)))?;
4281        Ok(froze.unwrap_or_default())
4282    }
4283
4284    /// Undo [`Self::admit_hello_authority`] for a HELLO refused after the
4285    /// registry accepted it, so the refused connection is no registration of
4286    /// the spawn and the spawn's declarations are left for the HELLO that
4287    /// does register it to freeze.
4288    fn refuse_hello_authority(&self, connection: ConnectionId, authority: Option<&HelloAuthority>) {
4289        let Some(authority) = authority else {
4290            return;
4291        };
4292        if let Err(err) = self.forwarding.unbind_target_authority(connection) {
4293            warn!(error = %err, "failed to unbind a refused HELLO from its spawn");
4294        }
4295        if authority
4296            .froze
4297            .swap(false, std::sync::atomic::Ordering::Relaxed)
4298        {
4299            authority.record.unfreeze_refused_declarations();
4300        }
4301    }
4302
4303    fn build_hello_ack(
4304        &self,
4305        frame: &Frame,
4306        negotiated_ver: u8,
4307        module_id: &str,
4308    ) -> Result<Frame, RouterError> {
4309        let ack = ModuleHelloAckBody {
4310            negotiated_ver,
4311            subc_ops: module_subc_ops(),
4312            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
4313            storage: self
4314                .storage_config
4315                .as_ref()
4316                .map(|cfg| cfg.descriptor_for(module_id)),
4317            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4318        };
4319        let body = serde_json::to_vec(&ack).map_err(|err| {
4320            RouterError::backend(
4321                0,
4322                frame.header.corr,
4323                format!("failed to encode HELLO_ACK: {err}"),
4324            )
4325        })?;
4326
4327        Frame::build_with_version(
4328            negotiated_ver,
4329            FrameType::HelloAck,
4330            control_flags(),
4331            0,
4332            0,
4333            frame.header.corr,
4334            body,
4335        )
4336        .map_err(RouterError::FrameBuild)
4337    }
4338
4339    async fn handle_client_control_request(
4340        &self,
4341        ctx: &RouteCtx,
4342        frame: Frame,
4343        request: ClientControlRequest,
4344        route_open_timings: &mut Option<RouteOpenTimings>,
4345    ) -> Result<Vec<Frame>, RouterError> {
4346        if let Some(owner) = &self.config_runtime {
4347            let module = match &request {
4348                ClientControlRequest::SupervisorRestart { module_id, .. }
4349                | ClientControlRequest::SupervisorReload { module_id }
4350                | ClientControlRequest::SupervisorSwap { module_id, .. }
4351                | ClientControlRequest::SupervisorSetEnabled {
4352                    module_id,
4353                    enabled: true,
4354                } => Some(module_id.as_str()),
4355                _ => None,
4356            };
4357            let rescan = matches!(
4358                &request,
4359                ClientControlRequest::SupervisorRescan { preview: false }
4360                    | ClientControlRequest::SupervisorRescanChecked { preview: false, .. }
4361            );
4362            let held = if module.is_some() || rescan {
4363                let module = module.map(str::to_owned);
4364                owner
4365                    .on_blocking(move |owner| owner.checked_admission_hold(module.as_deref()))
4366                    .await
4367            } else {
4368                Ok(())
4369            };
4370            if let Err(error) = held {
4371                if error.code != "config_delivery_deferred" {
4372                    let code = if error.code == "store_tampered" {
4373                        "store_tampered"
4374                    } else {
4375                        "config_gated"
4376                    };
4377                    return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
4378                }
4379            }
4380        }
4381        if let Some(operation) = privileged_operation(&request) {
4382            let caller = self.resolve_caller(ctx.connection_id).await;
4383            let op = client_control_request_op(&request);
4384            let text = prompt_text(operation, &caller);
4385            match self.privileged_operator_prompt {
4386                PrivilegedOperatorPrompt::Log => {
4387                    info!(target: "subc_daemon::privileged_prompt", operation = op,
4388                        target = ?operation, caller = %caller.description(), summary = %text,
4389                        "privileged request would prompt the operator");
4390                }
4391                PrivilegedOperatorPrompt::Enforce => {
4392                    let key =
4393                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
4394                    let stamp = self
4395                        .operator_apps
4396                        .lock()
4397                        .unwrap_or_else(|p| p.into_inner())
4398                        .get(&ctx.connection_id)
4399                        .cloned();
4400                    let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
4401                        &self.operator_app_policy,
4402                        &self.launch_table,
4403                        stamp.as_ref(),
4404                    )
4405                    .then(|| self.shortcut_context(stamp, None, None));
4406                    let outcome = self
4407                        .forwarding
4408                        .operator_confirms()
4409                        .approve_privileged_with_shortcut(
4410                            ctx,
4411                            &frame,
4412                            op,
4413                            text,
4414                            caller.description(),
4415                            key,
4416                            shortcut,
4417                        )
4418                        .await;
4419                    if outcome.code != "confirmed" {
4420                        return Ok(vec![outcome.frame(&frame)]);
4421                    }
4422                }
4423            }
4424        }
4425        match request {
4426            ClientControlRequest::DaemonShutdownIntent { intent } => {
4427                if intent.as_str() != Some("restart") {
4428                    return Ok(vec![control_error_frame(
4429                        &frame,
4430                        "invalid_params",
4431                        "intent must be 'restart'",
4432                    )?]);
4433                }
4434                // A stamped daemon boots a newer approved generation only from a
4435                // durable restart intent, written here before the daemon exits.
4436                // Failing to write it never blocks the restart: the next start
4437                // then boots the last active generation and the newer one waits.
4438                if let Some(owner) = &self.config_runtime {
4439                    match owner
4440                        .on_blocking(|owner| owner.assign_daemon_restart())
4441                        .await
4442                    {
4443                        Ok(Some(generation)) => {
4444                            info!(
4445                                generation,
4446                                "daemon restart will warm up approved generation"
4447                            )
4448                        }
4449                        Ok(None) => {}
4450                        Err(error) => warn!(
4451                            %error,
4452                            "daemon restart intent not recorded; the next start boots the last active generation"
4453                        ),
4454                    }
4455                }
4456                let response = ClientControlResponse::DaemonShutdownIntent {
4457                    intent: "restart".to_owned(),
4458                    expires_at_ms: self.supervisor.announce_daemon_restart(),
4459                };
4460                Ok(vec![control_response_body_frame(
4461                    &frame,
4462                    &response,
4463                    "ClientControlResponse::DaemonShutdownIntent",
4464                )?])
4465            }
4466            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
4467            ClientControlRequest::CatalogList { module_id } => {
4468                self.handle_catalog_list(frame, module_id)
4469            }
4470            ClientControlRequest::RouteOpen {
4471                target,
4472                identity,
4473                consumer_identity,
4474                consumer_capabilities,
4475                role_versions,
4476                busy_refusals,
4477                admission_facts,
4478                scope,
4479                ..
4480            } => {
4481                let extensions = match serde_json::from_slice::<subc_control::RouteOpenRequestExt>(
4482                    &frame.body,
4483                ) {
4484                    Ok(extensions) => extensions,
4485                    Err(err) => {
4486                        return Ok(vec![control_error_frame(
4487                            &frame,
4488                            "invalid_control_body",
4489                            format!("malformed route.open extensions: {err}"),
4490                        )?]);
4491                    }
4492                };
4493                self.handle_route_open(
4494                    ctx,
4495                    frame,
4496                    RouteOpenRequest {
4497                        target,
4498                        identity,
4499                        consumer_identity,
4500                        consumer_capabilities,
4501                        role_versions,
4502                        busy_refusals,
4503                        admission_facts,
4504                        scope,
4505                        accepts_close_reasons: extensions.accepts_close_reasons,
4506                    },
4507                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
4508                )
4509                .await
4510            }
4511            ClientControlRequest::RoutePoll {
4512                route_channel,
4513                route_epoch,
4514                kind,
4515            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
4516            ClientControlRequest::SupervisorList {} => {
4517                let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
4518                    self.agent_declarations.refuse(
4519                        self,
4520                        ctx,
4521                        control_error_frame(
4522                            &frame,
4523                            "busy",
4524                            "supervisor list work is at capacity; retry",
4525                        )?,
4526                    );
4527                    return Ok(Vec::new());
4528                };
4529                // Filesystem-backed diagnostics must not hold up this connection's reader.
4530                let handler = self.clone();
4531                let sink = ctx.egress.clone();
4532                tokio::spawn(
4533                    async move {
4534                        let _permit = permit;
4535                        let result = handler.handle_supervisor_list(frame.clone()).await;
4536                        let replies = match result {
4537                            Ok(replies) => replies,
4538                            Err(error) => {
4539                                control_error_frame(&frame, "backend_error", error.to_string())
4540                                    .into_iter()
4541                                    .collect()
4542                            }
4543                        };
4544                        for reply in replies {
4545                            if !matches!(
4546                                tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
4547                                    .await,
4548                                Ok(Ok(()))
4549                            ) {
4550                                break;
4551                            }
4552                        }
4553                    }
4554                    .with_current_subscriber(),
4555                );
4556                Ok(Vec::new())
4557            }
4558            ClientControlRequest::SupervisorSpawnSnapshot {} => {
4559                self.handle_supervisor_spawn_snapshot(frame)
4560            }
4561            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
4562                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
4563            }
4564            ClientControlRequest::SupervisorRestart {
4565                module_id,
4566                drain_timeout_ms,
4567            } => {
4568                self.log_supervisor_request_received(
4569                    ctx,
4570                    frame.header.corr,
4571                    ops::SUPERVISOR_RESTART,
4572                    Some(&module_id),
4573                    None,
4574                )?;
4575                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4576                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
4577                    .await
4578            }
4579            ClientControlRequest::SupervisorSwap {
4580                module_id,
4581                ready_timeout_ms,
4582            } => {
4583                self.log_supervisor_request_received(
4584                    ctx,
4585                    frame.header.corr,
4586                    ops::SUPERVISOR_SWAP,
4587                    Some(&module_id),
4588                    None,
4589                )?;
4590                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4591                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
4592                    .await
4593            }
4594            ClientControlRequest::SupervisorReload { module_id } => {
4595                self.log_supervisor_request_received(
4596                    ctx,
4597                    frame.header.corr,
4598                    ops::SUPERVISOR_RELOAD,
4599                    Some(&module_id),
4600                    None,
4601                )?;
4602                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4603                self.handle_supervisor_reload(frame, module_id, requester)
4604                    .await
4605            }
4606            ClientControlRequest::SupervisorRescan { preview } => {
4607                if !preview {
4608                    self.log_supervisor_request_received(
4609                        ctx,
4610                        frame.header.corr,
4611                        ops::SUPERVISOR_RESCAN,
4612                        None,
4613                        None,
4614                    )?;
4615                }
4616                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4617                let operation = self.handle_supervisor_rescan(
4618                    frame,
4619                    if preview {
4620                        RescanMode::Preview
4621                    } else {
4622                        RescanMode::Execute
4623                    },
4624                    requester,
4625                );
4626                self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
4627                    .await
4628            }
4629            ClientControlRequest::SupervisorRescanChecked {
4630                preview,
4631                candidate,
4632                expect,
4633            } => {
4634                self.observe_control_wait(
4635                    ctx,
4636                    None,
4637                    ops::SUPERVISOR_RESCAN,
4638                    self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
4639                )
4640                .await
4641            }
4642            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
4643                self.log_supervisor_request_received(
4644                    ctx,
4645                    frame.header.corr,
4646                    ops::SUPERVISOR_RELEASE_RESERVED,
4647                    Some(&module_id),
4648                    None,
4649                )?;
4650                self.handle_supervisor_release_reserved(frame, module_id)
4651                    .await
4652            }
4653            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
4654                self.log_supervisor_request_received(
4655                    ctx,
4656                    frame.header.corr,
4657                    ops::SUPERVISOR_SET_ENABLED,
4658                    Some(&module_id),
4659                    Some(enabled),
4660                )?;
4661                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4662                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
4663                    .await
4664            }
4665            ClientControlRequest::SupervisorHealthProbe { module_id } => {
4666                self.handle_supervisor_health_probe(ctx, frame, module_id)
4667                    .await
4668            }
4669            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
4670            ClientControlRequest::SupervisorRoutes { module_id } => {
4671                self.handle_supervisor_routes(frame, module_id)
4672            }
4673            ClientControlRequest::SupervisorProvenance { module_id } => {
4674                self.handle_supervisor_provenance(ctx, frame, module_id)
4675                    .await
4676            }
4677            ClientControlRequest::SupervisorStderrTail {
4678                module_id,
4679                max_lines,
4680                max_bytes,
4681            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
4682            ClientControlRequest::SupervisorTerminals { module_id } => {
4683                self.handle_supervisor_terminals(ctx, frame, module_id)
4684                    .await
4685            }
4686        }
4687    }
4688
4689    fn log_supervisor_request_received(
4690        &self,
4691        ctx: &RouteCtx,
4692        corr: u64,
4693        op: &'static str,
4694        module_id: Option<&str>,
4695        enabled: Option<bool>,
4696    ) -> Result<(), RouterError> {
4697        let caller = self.supervisor_requester(ctx, corr)?;
4698        let caller = principal_label(&caller);
4699        // `Option` fields are recorded only when present, so a request with no
4700        // module id (rescan) or no enabled flag simply omits that field.
4701        info!(
4702            target: "control",
4703            op,
4704            module_id,
4705            enabled,
4706            connection_id = ctx.connection_id.get(),
4707            caller = %caller,
4708            "supervisor request received"
4709        );
4710        Ok(())
4711    }
4712
4713    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
4714        Ok(self
4715            .registry
4716            .get_module_by_connection(ctx.connection_id)
4717            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4718            .map(|registration| Principal::Reserved {
4719                module_id: registration.manifest.module_id,
4720            })
4721            .unwrap_or(Principal::Direct))
4722    }
4723
4724    fn handle_module_control_request(
4725        &self,
4726        connection_id: ConnectionId,
4727        frame: Frame,
4728        request: ModuleControlRequestFromModule,
4729    ) -> Result<Vec<Frame>, RouterError> {
4730        match request {
4731            ModuleControlRequestFromModule::RouteAuthorize {
4732                principal,
4733                scope,
4734                target,
4735                expected,
4736            } => self.handle_route_authorize(
4737                connection_id,
4738                frame,
4739                principal,
4740                scope,
4741                target,
4742                expected,
4743            ),
4744            ModuleControlRequestFromModule::CatalogUpdate {
4745                provides,
4746                capabilities,
4747                ready,
4748            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
4749            ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { role_declarations } => {
4750                self.handle_catalog_update_role_declarations(
4751                    connection_id,
4752                    frame,
4753                    role_declarations,
4754                )
4755            }
4756            ModuleControlRequestFromModule::LiveRoots {} => {
4757                let registered = self
4758                    .registry
4759                    .get_module_by_connection(connection_id)
4760                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4761                let Some(registration) = registered else {
4762                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
4763                };
4764                let response = self
4765                    .forwarding
4766                    .live_roots(&registration.manifest.module_id)
4767                    .map_err(RouterError::Forwarding)?;
4768                Ok(vec![control_response_body_frame(
4769                    &frame,
4770                    &response,
4771                    "ModuleControlResponseToModule::LiveRoots",
4772                )?])
4773            }
4774            ModuleControlRequestFromModule::AgentCommands {
4775                started,
4776                finished,
4777                known,
4778            } => self.handle_agent_commands(connection_id, frame, started, finished, known),
4779            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
4780                self.handle_scope_sync(connection_id, frame, generation, scopes)
4781            }
4782            ModuleControlRequestFromModule::ScopeApply {
4783                generation,
4784                upsert,
4785                end,
4786            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
4787            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
4788                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
4789            }
4790            _ => Ok(vec![control_error_frame(
4791                &frame,
4792                "unsupported_control_frame",
4793                "unsupported module control operation",
4794            )?]),
4795        }
4796    }
4797
4798    fn handle_scope_served_by(
4799        &self,
4800        connection_id: ConnectionId,
4801        frame: Frame,
4802    ) -> Result<Vec<Frame>, RouterError> {
4803        let registered = self
4804            .registry
4805            .get_module_by_connection(connection_id)
4806            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4807        let Some(registered) = registered else {
4808            return Ok(vec![control_error_frame(
4809                &frame,
4810                "not_registered",
4811                "scope.served_by requires an active module registration owned by this connection",
4812            )?]);
4813        };
4814        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
4815        {
4816            Ok(request) => request,
4817            Err(err) => {
4818                return Ok(vec![control_error_frame(
4819                    &frame,
4820                    "invalid_control_body",
4821                    format!("malformed scope.served_by body: {err}"),
4822                )?])
4823            }
4824        };
4825        let answer = self
4826            .scopes
4827            .read()
4828            .unwrap_or_else(|poisoned| poisoned.into_inner())
4829            .served_by(
4830                &registered.manifest.module_id,
4831                &request.scope_ref,
4832                request.scope_epoch,
4833                &request.module_id,
4834            );
4835        let reply = subc_control::ScopeServedByReply::new(
4836            answer,
4837            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
4838        );
4839        Ok(vec![control_response_body_frame(
4840            &frame,
4841            &reply,
4842            "ScopeServedByReply",
4843        )?])
4844    }
4845
4846    fn handle_operator_confirm(
4847        &self,
4848        ctx: &RouteCtx,
4849        frame: Frame,
4850    ) -> Result<Vec<Frame>, RouterError> {
4851        use crate::operator_confirm::{audit, Outcome};
4852        let registration = self
4853            .registry
4854            .get_module_by_connection(ctx.connection_id)
4855            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4856        let Some(registration) = registration else {
4857            let outcome = Outcome::refusal("not_registered");
4858            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4859            return Ok(vec![outcome.frame(&frame)]);
4860        };
4861        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
4862            Ok(request) => request,
4863            Err(_) => {
4864                let outcome = Outcome::refusal("invalid_control_body");
4865                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4866                return Ok(vec![outcome.frame(&frame)]);
4867            }
4868        };
4869        let declared = matches!(
4870            request.kind,
4871            Some(subc_protocol::session::OperatorConfirmKind::Operation)
4872        ) && request.operation.as_ref().is_some_and(|operation| {
4873            registration
4874                .manifest
4875                .confirm_operations
4876                .as_ref()
4877                .is_some_and(|operations| operations.contains(operation))
4878        });
4879        let module_id = registration.manifest.module_id;
4880        // Read the launch nonce (under its own lock) before taking the forwarding
4881        // table's lock below: holding forwarding while waiting on another daemon
4882        // lock risks a lock-order deadlock with paths that take them the other way.
4883        let nonce = self
4884            .hello_launch_nonces
4885            .lock()
4886            .unwrap_or_else(|p| p.into_inner())
4887            .nonce(ctx.connection_id)
4888            .map(str::to_owned);
4889        // A registration channel proves its own spawn; only a connection that
4890        // registered over the launch nonce falls back to the nonce.
4891        let nonce_proven = self
4892            .confirm_channel_proven(ctx.connection_id, &module_id)
4893            .unwrap_or_else(|| {
4894                nonce.as_deref().is_some_and(|nonce| {
4895                    self.supervisor
4896                        .spawned_consumer_authorized(&module_id, nonce)
4897                })
4898            });
4899        let confirms = self.forwarding.operator_confirms();
4900        self.forwarding
4901            .with_operator_route(
4902                ctx.connection_id,
4903                request.route_channel,
4904                request.route_epoch,
4905                |binding| {
4906                    let shortcut = binding
4907                        .filter(|binding| {
4908                            nonce_proven
4909                                && crate::operator_confirm::valid_summary(&request.summary)
4910                                && declared
4911                                && crate::operator_confirm::ShortcutContext::route_eligible(
4912                                    &self.operator_app_policy,
4913                                    &self.launch_table,
4914                                    binding,
4915                                )
4916                        })
4917                        .map(|binding| {
4918                            let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
4919                                &self.operator_app_policy,
4920                                binding,
4921                            );
4922                            let stamp = if phone.is_some() {
4923                                None
4924                            } else {
4925                                binding.operator_app.clone()
4926                            };
4927                            self.shortcut_context(
4928                                stamp,
4929                                phone,
4930                                Some((request.route_channel, request.route_epoch)),
4931                            )
4932                        });
4933                    confirms.admit_with_shortcut(
4934                        ctx,
4935                        frame,
4936                        module_id,
4937                        nonce_proven,
4938                        request,
4939                        binding,
4940                        shortcut,
4941                    )
4942                },
4943            )
4944            .map_err(RouterError::Forwarding)
4945    }
4946
4947    fn shortcut_context(
4948        &self,
4949        stamp: Option<crate::operator_apps::OperatorAppStamp>,
4950        phone_fingerprint: Option<String>,
4951        route: Option<(u16, u32)>,
4952    ) -> crate::operator_confirm::ShortcutContext {
4953        crate::operator_confirm::ShortcutContext {
4954            ledger: self.activity.clone(),
4955            forwarding: self.forwarding.clone(),
4956            launches: self.launch_table.clone(),
4957            stamp,
4958            phone_fingerprint,
4959            route,
4960            connections: self.operator_apps.clone(),
4961            #[cfg(test)]
4962            hooks: Default::default(),
4963        }
4964    }
4965
4966    /// Explicit test-only phone policy; daemon configuration cannot select it.
4967    #[cfg(any(test, feature = "test-support"))]
4968    pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
4969        self.operator_app_policy.phone_enabled = enabled;
4970        self
4971    }
4972
4973    /// Resolve folders before taking the scope-table lock: a slow filesystem
4974    /// must not hold up admissions for unrelated scoped routes.
4975    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
4976        let mut scopes = ScopeRecords::from(scopes);
4977        for (index, record) in scopes.records.iter_mut().enumerate() {
4978            if let Some(refusal) = Self::canonicalize_workdir(record) {
4979                scopes.refusals.insert(index, refusal);
4980            }
4981        }
4982        scopes
4983    }
4984
4985    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
4986        let workdir = record.workdir.as_mut()?;
4987        let path = std::path::Path::new(workdir);
4988        if !path.is_absolute() {
4989            return Some((
4990                error_codes::INVALID_WORKDIR,
4991                "workdir must be absolute".to_string(),
4992            ));
4993        }
4994        match ProjectRootId::from_path_allowing_missing(path) {
4995            Ok(root) => match root.as_path().to_str() {
4996                Some(canonical) => {
4997                    *workdir = canonical.to_string();
4998                    None
4999                }
5000                None => Some((
5001                    error_codes::INVALID_WORKDIR,
5002                    "canonical workdir is not UTF-8".to_string(),
5003                )),
5004            },
5005            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
5006        }
5007    }
5008
5009    /// `scope.sync`: the owner is the module registered on this connection.
5010    /// A connection with no registration (every client connection, `direct`
5011    /// included) is refused `not_registered` before the table is consulted.
5012    fn handle_scope_sync(
5013        &self,
5014        connection_id: ConnectionId,
5015        frame: Frame,
5016        generation: u64,
5017        scopes: Vec<ScopeRecord>,
5018    ) -> Result<Vec<Frame>, RouterError> {
5019        self.handle_scope_change(connection_id, frame, generation, scopes, None)
5020    }
5021
5022    fn handle_scope_change(
5023        &self,
5024        connection_id: ConnectionId,
5025        frame: Frame,
5026        generation: u64,
5027        scopes: Vec<ScopeRecord>,
5028        end: Option<Vec<ScopeEnd>>,
5029    ) -> Result<Vec<Frame>, RouterError> {
5030        let is_apply = end.is_some();
5031        let Some(registration) = self
5032            .registry
5033            .get_module_by_connection(connection_id)
5034            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5035        else {
5036            return Ok(vec![control_error_frame(
5037                &frame,
5038                "not_registered",
5039                "scope.sync and scope.apply require an active module registration owned by this connection",
5040            )?]);
5041        };
5042        let owner = registration.manifest.module_id;
5043        let scopes = Self::prepare_scope_records(scopes);
5044        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
5045        let is_current_launch = |connection: ConnectionId| {
5046            self.connection_is_current_launch(&owner, connection, current_nonce.as_deref())
5047        };
5048        // Lock order is the scope table, then the forwarding table: the new
5049        // tags are published, and the routes the change closes are selected,
5050        // while the scope table is still write-locked, so no admission can read
5051        // a record whose tag is not yet published.
5052        let mut table = self
5053            .scopes
5054            .write()
5055            .unwrap_or_else(|poisoned| poisoned.into_inner());
5056        let outcome = match end {
5057            Some(end) => table.apply(
5058                &owner,
5059                connection_id,
5060                is_current_launch,
5061                generation,
5062                scopes,
5063                end,
5064            ),
5065            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
5066        };
5067        let drained = match &outcome {
5068            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
5069            Err(_) => Vec::new(),
5070        };
5071        let has_deadlines = outcome.is_ok() && table.has_deadlines();
5072        drop(table);
5073        if has_deadlines {
5074            self.scope_deadline_added.notify_one();
5075        }
5076        match outcome {
5077            Ok(applied) => {
5078                let counts = ScopeOutcomeCounts::of(&applied.results);
5079                info!(
5080                    owner = %owner,
5081                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5082                    generation,
5083                    records = applied.results.len(),
5084                    created = counts.created,
5085                    replaced = counts.replaced,
5086                    updated = counts.updated,
5087                    unchanged = counts.unchanged,
5088                    refused = counts.refused,
5089                    ended = applied.ended.len(),
5090                    tag_changes = applied.tag_changes.len(),
5091                    routes_closed = drained.len(),
5092                    "scope change accepted"
5093                );
5094                // An accepted scope change can still refuse individual records, and the
5095                // owner is the only party that sees the reply. Name them here so
5096                // an operator can tell a refused session from a missing one
5097                // without the owner's logs. Capped so a call that refuses
5098                // thousands cannot flood the log; the count above is complete.
5099                for refused in applied
5100                    .results
5101                    .iter()
5102                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
5103                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
5104                {
5105                    warn!(
5106                        owner = %owner,
5107                        generation,
5108                        scope_ref = %refused.scope_ref,
5109                        scope_epoch = refused.scope_epoch,
5110                        code = refused.code.as_deref().unwrap_or(""),
5111                        "scope record refused"
5112                    );
5113                }
5114                self.close_scope_drained_routes(drained);
5115                let response = if is_apply {
5116                    ModuleControlResponseToModule::ScopeApply {
5117                        generation,
5118                        results: applied.results,
5119                        end_results: applied.end_results,
5120                        ended: applied.ended,
5121                    }
5122                } else {
5123                    ModuleControlResponseToModule::ScopeSync {
5124                        generation,
5125                        results: applied.results,
5126                        ended: applied.ended,
5127                    }
5128                };
5129                Ok(vec![control_response_body_frame(
5130                    &frame,
5131                    &response,
5132                    "scope change response",
5133                )?])
5134            }
5135            Err(refusal) => {
5136                info!(
5137                    owner = %owner,
5138                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5139                    generation,
5140                    code = refusal.code,
5141                    "scope change refused"
5142                );
5143                Ok(vec![control_error_frame(
5144                    &frame,
5145                    refusal.code,
5146                    refusal.message,
5147                )?])
5148            }
5149        }
5150    }
5151
5152    /// Notify both endpoints of routes drained by a scope change.
5153    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
5154        scope_expiry::ScopeExpiryContext::new(self).close_scope_drained_routes(drained);
5155    }
5156
5157    /// Re-authorize a stored source on a principal the module supplies.
5158    ///
5159    /// That principal carries no spawn or authority epoch, so this reply never
5160    /// conveys a caller attestation or a first-party mark, and a provider must
5161    /// not apply automatic grants on anything it authorizes.
5162    fn handle_route_authorize(
5163        &self,
5164        connection_id: ConnectionId,
5165        frame: Frame,
5166        principal: Principal,
5167        scope: subc_protocol::scope::RouteAuthorizeScope,
5168        target: String,
5169        expected: subc_protocol::scope::RouteAuthorizeExpected,
5170    ) -> Result<Vec<Frame>, RouterError> {
5171        use crate::scopes::admission::{admit_route, AdmissionMode};
5172        use subc_protocol::scope::{
5173            RouteAuthorizeOutcome, RouteAuthorizeResponse, ROUTE_AUTHORIZE_CAPABILITY,
5174        };
5175        let registry = self
5176            .registry
5177            .route_admission_snapshot()
5178            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5179        let owner_configured = match &scope.owner {
5180            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5181            _ => false,
5182        };
5183        let selector = ScopeSelector::new(scope.owner, scope.scope_ref, Some(scope.epoch));
5184        let answer = {
5185            let scopes = self.scopes.read().unwrap_or_else(|p| p.into_inner());
5186            let permitted = registry
5187                .iter()
5188                .find(|r| r.connection_id == connection_id)
5189                .is_some_and(|r| {
5190                    scopes.is_federated_owner(&Principal::Reserved {
5191                        module_id: r.manifest.module_id.clone(),
5192                    }) && r.manifest.capabilities.as_ref().is_some_and(|caps| {
5193                        caps.provides
5194                            .iter()
5195                            .any(|cap| cap == ROUTE_AUTHORIZE_CAPABILITY)
5196                    })
5197                });
5198            if !permitted {
5199                drop(scopes);
5200                return Ok(vec![control_error_frame(&frame, error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED,
5201                    "route.authorize requires the configured federated owner's route-authorize/v1 registration")?]);
5202            }
5203            admit_route(
5204                &scopes,
5205                &registry,
5206                &principal,
5207                &target,
5208                Some(&selector),
5209                owner_configured,
5210                AdmissionMode::Authorize(&expected),
5211            )
5212        };
5213        let response = match answer {
5214            Ok(admission) => {
5215                let mut response = RouteAuthorizeResponse::new(
5216                    RouteAuthorizeOutcome::Authorized,
5217                    "route admission authorized",
5218                );
5219                response.content_version = admission.content_version;
5220                response
5221            }
5222            Err(refusal) => refusal.response(),
5223        };
5224        Ok(vec![control_response_body_frame(
5225            &frame,
5226            &response,
5227            "RouteAuthorizeResponse",
5228        )?])
5229    }
5230
5231    /// `scope.describe`: any registered module may read any scope, because a
5232    /// provider must read the scope a route it serves is stamped with.
5233    fn handle_scope_describe(
5234        &self,
5235        connection_id: ConnectionId,
5236        frame: Frame,
5237        owner: Principal,
5238        scope_ref: String,
5239    ) -> Result<Vec<Frame>, RouterError> {
5240        let registered = self
5241            .registry
5242            .get_module_by_connection(connection_id)
5243            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5244        let Some(registered) = registered else {
5245            return Ok(vec![control_error_frame(
5246                &frame,
5247                "not_registered",
5248                "scope.describe requires an active module registration owned by this connection",
5249            )?]);
5250        };
5251        let scopes = self
5252            .scopes
5253            .read()
5254            .unwrap_or_else(|poisoned| poisoned.into_inner());
5255        if scopes.is_federated_owner(&owner)
5256            && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
5257        {
5258            return Ok(vec![control_error_frame(
5259                &frame,
5260                error_codes::SCOPE_FEDERATED_UNSUPPORTED,
5261                "scope.describe of the federated owner requires scope-federated/v1",
5262            )?]);
5263        }
5264        let mut description = scopes.describe(&owner, &scope_ref);
5265        drop(scopes);
5266        if owner
5267            != (Principal::Reserved {
5268                module_id: registered.manifest.module_id.clone(),
5269            })
5270        {
5271            if let Some(facts) = description
5272                .stamp
5273                .as_mut()
5274                .and_then(|stamp| stamp.attributes.federated.as_mut())
5275            {
5276                facts.stamp_required = false;
5277            }
5278        }
5279        // Describe readers also decode strict ScopeAttributes. Attribution is
5280        // safe to omit unless the requesting module declares it understands it.
5281        if !registered
5282            .manifest
5283            .capabilities
5284            .as_ref()
5285            .is_some_and(|caps| {
5286                caps.provides
5287                    .iter()
5288                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5289            })
5290        {
5291            if let Some(stamp) = description.stamp.as_mut() {
5292                stamp.attributes.on_behalf_of_agent = None;
5293            }
5294        }
5295        if !registered
5296            .manifest
5297            .capabilities
5298            .as_ref()
5299            .is_some_and(|caps| {
5300                caps.provides
5301                    .iter()
5302                    .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5303            })
5304        {
5305            if let Some(stamp) = description.stamp.as_mut() {
5306                stamp.attributes.flow_version = None;
5307                stamp.attributes.flow_code_hash = None;
5308                stamp.attributes.flow_install_id = None;
5309            }
5310        }
5311        let owner_configured = match &owner {
5312            // Ask whether the owner is configured (`is_configured`), not
5313            // whether it is on the roster (`get(..).is_some()`): a supervised
5314            // module's process can register and describe a scope before the
5315            // supervisor has put it on the roster.
5316            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5317            _ => false,
5318        };
5319        let response = ModuleControlResponseToModule::ScopeDescribe {
5320            status: description.status,
5321            scope_epoch: description.scope_epoch,
5322            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
5323            owner_synced: description.owner_synced,
5324            owner_configured,
5325            scope: description.stamp,
5326        };
5327        Ok(vec![control_response_body_frame(
5328            &frame,
5329            &response,
5330            "ModuleControlResponseToModule::ScopeDescribe",
5331        )?])
5332    }
5333
5334    fn handle_catalog_update_role_declarations(
5335        &self,
5336        connection_id: ConnectionId,
5337        frame: Frame,
5338        role_declarations: serde_json::Value,
5339    ) -> Result<Vec<Frame>, RouterError> {
5340        if let Err(err) = validate_role_declarations_value(Some(&role_declarations)) {
5341            return Ok(vec![control_error_frame(
5342                &frame,
5343                "invalid_role_declarations",
5344                err.to_string(),
5345            )?]);
5346        }
5347        let declarations = serde_json::from_value(role_declarations)
5348            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5349        if self
5350            .registry
5351            .replace_role_declarations_for_connection(connection_id, declarations)
5352            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5353            .is_none()
5354        {
5355            return Ok(vec![control_error_frame(
5356                &frame,
5357                "not_registered",
5358                "catalog.update_role_declarations requires an active module registration owned by this connection",
5359            )?]);
5360        }
5361        control_response_body_frame(
5362            &frame,
5363            &ModuleControlResponseToModule::CatalogUpdateRoleDeclarations {},
5364            "ModuleControlResponseToModule::CatalogUpdateRoleDeclarations",
5365        )
5366        .map(|frame| vec![frame])
5367    }
5368
5369    fn handle_catalog_update(
5370        &self,
5371        connection_id: ConnectionId,
5372        frame: Frame,
5373        provides: Vec<ProviderRole>,
5374        capabilities: Option<CapabilityDeclarations>,
5375        ready: Option<bool>,
5376    ) -> Result<Vec<Frame>, RouterError> {
5377        self.refresh_capability_requirements();
5378        let Some(registration) = self
5379            .registry
5380            .get_module_by_connection(connection_id)
5381            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5382        else {
5383            return Ok(vec![control_error_frame(
5384                &frame,
5385                "not_registered",
5386                "catalog.update requires an active module registration owned by this connection",
5387            )?]);
5388        };
5389
5390        if let Some(message) =
5391            catalog_update_frozen_field_message(&registration.manifest, &provides)
5392        {
5393            return Ok(vec![control_error_frame(
5394                &frame,
5395                "catalog_update_frozen_field",
5396                message,
5397            )?]);
5398        }
5399
5400        let mut candidate = registration.manifest.clone();
5401        candidate.provides = provides.clone();
5402        candidate.capabilities = capabilities
5403            .clone()
5404            .or_else(|| registration.manifest.capabilities.clone());
5405        if let Some(rich_ui) = &candidate.rich_ui {
5406            if let Err(err) = rich_ui.validate() {
5407                return Ok(vec![control_error_frame(
5408                    &frame,
5409                    "invalid_manifest",
5410                    err.to_string(),
5411                )?]);
5412            }
5413        }
5414        if let Err(err) = candidate.validate_capability_grammar() {
5415            return Ok(vec![control_error_frame(
5416                &frame,
5417                "invalid_capability_grammar",
5418                err.to_string(),
5419            )?]);
5420        }
5421
5422        // Updates must honor the same reserved owner as initial registration;
5423        // otherwise an empty HELLO could acquire the claim after admission.
5424        let mut conflicts = self
5425            .capability_evaluator
5426            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
5427        if let Some(conflict) = conflicts.first() {
5428            let message = format!(
5429                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
5430                conflict.capability, conflict.claimants[0], candidate.module_id
5431            );
5432            for conflict in &mut conflicts {
5433                conflict.source = DuplicateClaimSource::CatalogUpdate;
5434            }
5435            log_duplicate_claim_events(conflicts);
5436            return Ok(vec![control_error_frame(
5437                &frame,
5438                "reserved_capability",
5439                message,
5440            )?]);
5441        }
5442
5443        let updated = match self.registry.replace_catalog_for_connection(
5444            connection_id,
5445            provides,
5446            capabilities,
5447            ready,
5448        ) {
5449            Err(crate::registry::RegistryError::FederatedCapabilityChanged) => {
5450                return Ok(vec![control_error_frame(
5451                    &frame,
5452                    error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5453                    "scope-federated/v1 is fixed at HELLO; restart the module to change it",
5454                )?])
5455            }
5456            Err(crate::registry::RegistryError::CallerAttestationCapabilityChanged) => {
5457                return Ok(vec![control_error_frame(
5458                    &frame,
5459                    error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5460                    "caller-attestation/v1 is fixed at HELLO; restart the module to change it",
5461                )?])
5462            }
5463            result => {
5464                result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5465            }
5466        };
5467        if updated.is_none() {
5468            return Ok(vec![control_error_frame(
5469                &frame,
5470                "not_registered",
5471                "catalog.update requires an active module registration owned by this connection",
5472            )?]);
5473        }
5474        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
5475            log_duplicate_claim_events(
5476                self.capability_evaluator
5477                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
5478            );
5479        }
5480        if capability_census_trigger(
5481            registration.manifest.capabilities.as_ref(),
5482            updated
5483                .as_ref()
5484                .and_then(|entry| entry.manifest.capabilities.as_ref()),
5485        ) {
5486            self.enforce_capability_denies();
5487        }
5488        self.refresh_capability_requirements();
5489
5490        let response = ModuleControlResponseToModule::CatalogUpdate {};
5491        control_response_body_frame(
5492            &frame,
5493            &response,
5494            "ModuleControlResponseToModule::CatalogUpdate",
5495        )
5496        .map(|frame| vec![frame])
5497    }
5498
5499    pub(crate) fn with_activity_ledger_config(
5500        mut self,
5501        run_dir: &Path,
5502        incarnation: String,
5503        config: crate::activity::ActivityConfig,
5504    ) -> Self {
5505        self.activity = Some(crate::activity::ActivityLedger::open_config(
5506            run_dir.join("activity.jsonl"),
5507            incarnation,
5508            config,
5509        ));
5510        self.supervisor.set_activity(self.activity.clone());
5511        self
5512    }
5513
5514    pub(crate) async fn configure_activity(
5515        &self,
5516        config: crate::activity::ActivityConfig,
5517    ) -> Result<(), &'static str> {
5518        match &self.activity {
5519            Some(ledger) => ledger.configure(config).await,
5520            None => Ok(()),
5521        }
5522    }
5523
5524    fn cancel_activity_subscription(
5525        &self,
5526        ctx: &RouteCtx,
5527        frame: &Frame,
5528    ) -> Result<bool, RouterError> {
5529        let task = self
5530            .activity_streams
5531            .lock()
5532            .unwrap_or_else(|p| p.into_inner())
5533            .remove(&(ctx.connection_id, frame.header.corr));
5534        let Some(task) = task else {
5535            return Ok(false);
5536        };
5537        task.abort();
5538        let end = Frame::build_with_version(
5539            response_version(frame),
5540            FrameType::StreamEnd,
5541            control_flags(),
5542            0,
5543            0,
5544            frame.header.corr,
5545            Vec::new(),
5546        )
5547        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
5548        let sink = ctx.egress.clone();
5549        tokio::spawn(async move {
5550            let _ = sink.send(end).await;
5551        });
5552        Ok(true)
5553    }
5554
5555    async fn handle_activity_read(
5556        &self,
5557        ctx: &RouteCtx,
5558        frame: Frame,
5559    ) -> Result<Vec<Frame>, RouterError> {
5560        #[cfg(test)]
5561        {
5562            let pause = self
5563                .activity_request_pause
5564                .lock()
5565                .unwrap()
5566                .clone()
5567                .filter(|pause| pause.connection == ctx.connection_id);
5568            if let Some(pause) = pause {
5569                let _cancelled = ActivityRequestCancellation(pause.clone());
5570                pause.started.notify_one();
5571                std::future::pending::<()>().await;
5572            }
5573        }
5574        use subc_control::{
5575            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
5576            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
5577        };
5578        #[derive(Deserialize)]
5579        struct Identity {
5580            #[serde(default)]
5581            consumer_identity: Option<ConsumerIdentity>,
5582        }
5583        // A registration channel or claim-bound connection carries its module in
5584        // its binding, so no request field can name another principal, and its
5585        // subscription ends when the bound spawn's authority does.
5586        let mut bound_spawn = None;
5587        let legacy = match self.activity_reader(ctx.connection_id) {
5588            channel_operations::ActivityReader::Module { module_id, spawn } => {
5589                bound_spawn = Some(spawn);
5590                Some(Ok(Principal::Reserved { module_id }))
5591            }
5592            // Refused outright: falling back to `direct` would admit it.
5593            channel_operations::ActivityReader::Refused => {
5594                return Ok(vec![control_error_frame(
5595                    &frame,
5596                    ACTIVITY_READ_FORBIDDEN,
5597                    "helper, relay and retired connections cannot read activity",
5598                )?]);
5599            }
5600            channel_operations::ActivityReader::Legacy => None,
5601        };
5602        // A registered module connection already identifies its caller. Ignore
5603        // consumer_identity there so a module cannot borrow a mirror's identity.
5604        // Only separate consumer connections use the Direct/nonce trust boundary.
5605        let principal = match legacy {
5606            Some(principal) => principal,
5607            None => match self
5608                .forwarding
5609                .module_id_for_connection(ctx.connection_id)
5610                .map_err(RouterError::Forwarding)?
5611            {
5612                Some(module_id) => Ok(Principal::Reserved { module_id }),
5613                None => {
5614                    let identity: Identity = match serde_json::from_slice(&frame.body) {
5615                        Ok(identity) => identity,
5616                        Err(_) => {
5617                            return Ok(vec![control_error_frame(
5618                                &frame,
5619                                ACTIVITY_MALFORMED,
5620                                "malformed activity read",
5621                            )?])
5622                        }
5623                    };
5624                    self.route_open_principal(
5625                        ctx.connection_id,
5626                        &frame,
5627                        identity.consumer_identity,
5628                    )?
5629                }
5630            },
5631        };
5632        let admitted = matches!(principal, Ok(Principal::Direct))
5633            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
5634        if !admitted {
5635            return Ok(vec![control_error_frame(
5636                &frame,
5637                ACTIVITY_READ_FORBIDDEN,
5638                "activity reads require an operator or trusted mirror principal",
5639            )?]);
5640        }
5641        let Some(ledger) = &self.activity else {
5642            return Ok(vec![control_error_frame(
5643                &frame,
5644                ACTIVITY_UNAVAILABLE,
5645                "activity ledger unavailable",
5646            )?]);
5647        };
5648        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
5649            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
5650        {
5651            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
5652                Ok(request) => ledger.query(request).await,
5653                Err(_) => Err(ACTIVITY_MALFORMED),
5654            };
5655            return match result {
5656                Ok(reply) => Ok(vec![control_response_body_frame(
5657                    &frame,
5658                    &reply,
5659                    ops::ACTIVITY_QUERY,
5660                )?]),
5661                Err(code) => Ok(vec![control_error_frame(
5662                    &frame,
5663                    code,
5664                    "activity query refused",
5665                )?]),
5666            };
5667        }
5668        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
5669            Ok(request) => request,
5670            Err(_) => {
5671                return Ok(vec![control_error_frame(
5672                    &frame,
5673                    ACTIVITY_MALFORMED,
5674                    "malformed activity subscription",
5675                )?])
5676            }
5677        };
5678        let mut subscription = match ledger.subscribe(request.since).await {
5679            Ok(subscription) => subscription,
5680            Err(code) => {
5681                return Ok(vec![control_error_frame(
5682                    &frame,
5683                    code,
5684                    "activity subscription refused",
5685                )?])
5686            }
5687        };
5688        let key = (ctx.connection_id, frame.header.corr);
5689        let sink = ctx.egress.clone();
5690        let mut streams = self
5691            .activity_streams
5692            .lock()
5693            .unwrap_or_else(|p| p.into_inner());
5694        streams.retain(|_, task| !task.is_finished());
5695        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
5696            return Ok(vec![control_error_frame(
5697                &frame,
5698                "duplicate_subscription",
5699                "activity subscription correlation id already in use",
5700            )?]);
5701        }
5702        let stream = async move {
5703            loop {
5704                let entry = if let Some(entry) = subscription.replay.pop_front() {
5705                    Some(entry)
5706                } else {
5707                    subscription.receiver.recv().await
5708                };
5709                let Some(entry) = entry else {
5710                    break;
5711                };
5712                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
5713                    return;
5714                };
5715                let Ok(next) = Frame::build_with_version(
5716                    response_version(&frame),
5717                    FrameType::StreamData,
5718                    control_flags(),
5719                    0,
5720                    0,
5721                    frame.header.corr,
5722                    body,
5723                ) else {
5724                    return;
5725                };
5726                if sink.send(next).await.is_err() {
5727                    return;
5728                }
5729            }
5730            // Dropping a full subscriber's sender drains its accepted entries
5731            // before the terminal error, preserving their order on the wire.
5732            if let Some(code) = subscription.terminal_code() {
5733                if let Ok(end) =
5734                    control_error_frame(&frame, code, "activity subscriber fell behind")
5735                {
5736                    let _ = sink.send(end).await;
5737                }
5738            }
5739        };
5740        // The spawn's retirement revokes this connection's authority. End the
5741        // subscription then, rather than when the connection's close is
5742        // eventually processed or the next entry fails to send.
5743        let task = tokio::spawn(async move {
5744            match bound_spawn {
5745                Some(spawn) => {
5746                    tokio::select! {
5747                        () = stream => {}
5748                        () = spawn.wait_closed() => {}
5749                    }
5750                }
5751                None => stream.await,
5752            }
5753        });
5754        streams.insert(key, task.abort_handle());
5755        Ok(Vec::new())
5756    }
5757
5758    async fn handle_activity_record(
5759        &self,
5760        ctx: &RouteCtx,
5761        frame: Frame,
5762    ) -> Result<Vec<Frame>, RouterError> {
5763        use subc_control::{
5764            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
5765            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
5766        };
5767        let result = async {
5768            let module = self
5769                .forwarding
5770                .module_id_for_connection(ctx.connection_id)
5771                .ok()
5772                .flatten()
5773                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
5774            let request: ActivityRecordRequest =
5775                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
5776            crate::activity::validate(&module, &request)?;
5777            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
5778            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
5779            let at_ms = std::time::SystemTime::now()
5780                .duration_since(std::time::UNIX_EPOCH)
5781                .unwrap_or_default()
5782                .as_millis() as u64;
5783            ledger.record(module, request, actor, at_ms).await
5784        }
5785        .await;
5786        match result {
5787            Ok(entry) => Ok(vec![control_response_body_frame(
5788                &frame,
5789                &ActivityRecordResponse { seq: entry.seq },
5790                "activity.record",
5791            )?]),
5792            Err(code) => Ok(vec![control_error_frame(
5793                &frame,
5794                code,
5795                "activity recording refused",
5796            )?]),
5797        }
5798    }
5799
5800    fn activity_actor(
5801        &self,
5802        connection: ConnectionId,
5803        module: &str,
5804        on: Option<&subc_control::ActivityOn>,
5805    ) -> Result<subc_control::ActivityActor, &'static str> {
5806        use crate::forwarding::{DataRoute, DataRouteState};
5807        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
5808        let principal = |id: &str| Principal::Reserved {
5809            module_id: id.to_owned(),
5810        };
5811        let mut actor = ActivityActor {
5812            principal: principal(module),
5813            scope_owner: None,
5814            agent_id: None,
5815            run_id: None,
5816            flow_id: None,
5817            project_root: None,
5818            workdir: None,
5819            project_id: None,
5820        };
5821        let scope = match on {
5822            None => return Ok(actor),
5823            Some(ActivityOn::Scope(scope)) => {
5824                let selector = subc_protocol::scope::ScopeSelector::new(
5825                    principal(module),
5826                    scope.scope_ref.clone(),
5827                    Some(scope.epoch),
5828                );
5829                self.scopes
5830                    .read()
5831                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5832                    .attribute(&principal(module), module, &selector, true)
5833                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5834            }
5835            Some(ActivityOn::Route(route)) => {
5836                let DataRoute::Module(DataRouteState::Bound(binding)) = self
5837                    .forwarding
5838                    .lookup_data_route(connection, route.channel, route.epoch)
5839                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5840                else {
5841                    return Err(ACTIVITY_ACTOR_UNKNOWN);
5842                };
5843                if binding.module_id != module {
5844                    return Err(ACTIVITY_ACTOR_UNKNOWN);
5845                }
5846                actor.principal = binding.principal.clone();
5847                actor.project_root = binding
5848                    .project_root
5849                    .as_ref()
5850                    .map(|root| root.as_path().to_path_buf());
5851                let Some(bound) = &binding.scope else {
5852                    return Ok(actor);
5853                };
5854                let owner = principal(&bound.owner);
5855                let selector = subc_protocol::scope::ScopeSelector::new(
5856                    owner.clone(),
5857                    bound.scope_ref.clone(),
5858                    Some(bound.tag.scope_epoch),
5859                );
5860                let admitted = self
5861                    .scopes
5862                    .read()
5863                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5864                    .attribute(&owner, module, &selector, true)
5865                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
5866                if admitted.tag != bound.tag {
5867                    return Err(ACTIVITY_ACTOR_UNKNOWN);
5868                }
5869                admitted
5870            }
5871        };
5872        actor.scope_owner = Some(scope.stamp.owner);
5873        actor.agent_id = scope.stamp.attributes.agent_id;
5874        actor.run_id = scope.stamp.attributes.run_id;
5875        actor.flow_id = scope.stamp.attributes.flow_id;
5876        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
5877        actor.project_id = scope.stamp.project_id;
5878        Ok(actor)
5879    }
5880
5881    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5882        self.refresh_capability_requirements();
5883        // A bare connection count is ambiguous between many clients holding a
5884        // route each and one client accumulating hundreds, so publish the
5885        // concentration alongside it. Route state is best-effort here: a
5886        // diagnostic endpoint must still answer if the forwarding lock is
5887        // contended.
5888        let mut counters = self.counters.snapshot();
5889        let losses = self.resource_tracker.loss_diagnostics();
5890        if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
5891            if let Some(obj) = counters.as_object_mut() {
5892                obj.insert("agent_command_declaration_loss_by_module".into(), losses);
5893            }
5894        }
5895        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
5896            obj.insert(
5897                "activity_unparseable_lines".into(),
5898                ledger.read_errors().into(),
5899            );
5900            obj.insert(
5901                "activity_lifecycle_append_failures".into(),
5902                ledger.lifecycle_failures().into(),
5903            );
5904            obj.insert(
5905                "activity_started_digest_skips".into(),
5906                ledger.lifecycle_digest_skips().into(),
5907            );
5908        }
5909        if let (Ok((connections_with_routes, max)), Some(obj)) = (
5910            self.forwarding.client_route_concentration(),
5911            counters.as_object_mut(),
5912        ) {
5913            obj.insert(
5914                "client_connections_with_routes".into(),
5915                connections_with_routes.into(),
5916            );
5917            obj.insert("max_routes_on_one_connection".into(), max.into());
5918        }
5919        // A module that is being fast-refused and a module that is fine look
5920        // identical from a client that retries and succeeds, so name the open
5921        // breakers here. This rides the existing free-form counters object
5922        // rather than a new wire field, so no sibling that deserializes
5923        // `ServerDescribe` has to be rebuilt to keep reading it.
5924        if let (Some(open_breakers), Some(obj)) = (
5925            self.route_bind_breakers.open_snapshot(),
5926            counters.as_object_mut(),
5927        ) {
5928            obj.insert("route_bind_breakers_open".into(), open_breakers);
5929        }
5930        // A daemon whose local endpoint could not be bound keeps serving TCP,
5931        // and the connection file names no endpoint. Name the reason here, in
5932        // the free-form counters object, so an operator reading daemon facts
5933        // sees it without searching the log, and no wire field changes.
5934        if let (Some(reason), Some(obj)) = (
5935            self.local_endpoint_unavailable.clone(),
5936            counters.as_object_mut(),
5937        ) {
5938            obj.insert("local_endpoint_unavailable".into(), reason.into());
5939        }
5940        let response = ClientControlResponse::ServerDescribe {
5941            protocol_ver: PROTOCOL_VERSION,
5942            subc_ops: subc_ops(),
5943            capabilities: self.describe_capabilities(),
5944            connected_clients: self.connected_clients.count(),
5945            counters: Some(counters),
5946            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
5947            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
5948            capability_requirements: self.capability_requirement_statuses(),
5949            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
5950            config_path: self
5951                .rescan
5952                .as_ref()
5953                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
5954        };
5955        Ok(vec![control_response_body_frame(
5956            &frame,
5957            &response,
5958            "ClientControlResponse::ServerDescribe",
5959        )?])
5960    }
5961
5962    fn handle_catalog_list(
5963        &self,
5964        frame: Frame,
5965        module_id: Option<String>,
5966    ) -> Result<Vec<Frame>, RouterError> {
5967        let (generation, modules) = self
5968            .registry
5969            .list_modules_with_nonce_verification()
5970            .map_err(|err| {
5971                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
5972            })?;
5973        let entries = modules
5974            .into_iter()
5975            .filter(|(registration, _)| {
5976                module_id
5977                    .as_deref()
5978                    .map(|wanted| registration.manifest.module_id == wanted)
5979                    .unwrap_or(true)
5980            })
5981            .map(|(registration, nonce_verified)| {
5982                let entry = self
5983                    .catalog_discovery_entry(&registration, module_id.is_some())
5984                    .map_err(|err| RouterError::backend(0, frame.header.corr, err))?;
5985                let not_ready = self.not_ready_reason(&registration);
5986                let mut entry = entry;
5987                entry.roles = registration.manifest.provides;
5988                entry.control_ops = registration.control_ops;
5989                Ok(entry
5990                    .with_ready(not_ready.is_none())
5991                    .with_display_name(registration.manifest.display_name)
5992                    .with_not_ready(not_ready)
5993                    .with_module_version(Some(registration.manifest.module_version))
5994                    .with_provenance(registration.manifest.provenance)
5995                    .with_capabilities(registration.manifest.capabilities)
5996                    .with_self_signals(registration.manifest.self_signals)
5997                    .with_events(registration.manifest.events)
5998                    .with_operation_delivery(registration.manifest.operation_delivery)
5999                    .with_operation_payload(registration.manifest.operation_payload)
6000                    .with_flow_operations(registration.manifest.flow_operations)
6001                    .with_operation_deferral(registration.manifest.operation_deferral)
6002                    .with_confirm_operations(registration.manifest.confirm_operations)
6003                    .with_rich_ui(registration.manifest.rich_ui)
6004                    .with_role_declarations(registration.manifest.role_declarations)
6005                    .with_nonce_verified(nonce_verified))
6006            })
6007            .collect::<Result<Vec<_>, RouterError>>()?;
6008        let response = ClientControlResponse::CatalogList {
6009            generation,
6010            modules: entries,
6011            subc_ops: subc_ops(),
6012        };
6013        Ok(vec![control_response_body_frame(
6014            &frame,
6015            &response,
6016            "ClientControlResponse::CatalogList",
6017        )?])
6018    }
6019
6020    /// The caller fields for a refusal that only has the raw request frame,
6021    /// such as the connection-level admission refusal sent before the request
6022    /// reaches [`Self::handle_route_open`].
6023    fn route_open_caller_from_frame(&self, ctx: &RouteCtx, frame: &Frame) -> RouteOpenCaller {
6024        let Ok(ClientControlRequest::RouteOpen {
6025            identity,
6026            consumer_identity,
6027            scope,
6028            ..
6029        }) = parse_client_control_request(&frame.body)
6030        else {
6031            return RouteOpenCaller::unknown();
6032        };
6033        let principal = self
6034            .attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref())
6035            .map(|attested| attested.principal);
6036        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
6037    }
6038
6039    fn route_open_principal(
6040        &self,
6041        connection_id: ConnectionId,
6042        frame: &Frame,
6043        consumer_identity: Option<ConsumerIdentity>,
6044    ) -> Result<Result<Principal, Frame>, RouterError> {
6045        let attested = self
6046            .attest_route_open_consumer(connection_id, consumer_identity.as_ref())
6047            .map(|attested| attested.principal);
6048        self.route_open_principal_attested(frame, consumer_identity, attested)
6049    }
6050
6051    /// [`Self::route_open_principal`] with `attested` already taken by
6052    /// [`Self::attest_route_open_consumer`], so one open resolves its connection
6053    /// claim or launch nonce once.
6054    fn route_open_principal_attested(
6055        &self,
6056        frame: &Frame,
6057        consumer_identity: Option<ConsumerIdentity>,
6058        attested: Option<Principal>,
6059    ) -> Result<Result<Principal, Frame>, RouterError> {
6060        if let Some(principal) = attested {
6061            return Ok(Ok(principal));
6062        }
6063        Ok(Err(control_error_frame(
6064            frame,
6065            "bad_consumer_identity",
6066            consumer_identity.map_or_else(
6067                || "connection claim is no longer authorized for routes".to_string(),
6068                |identity| format!(
6069                    "consumer_identity for module_id '{}' conflicts with its connection claim or did not match a supervised launch nonce",
6070                    identity.module_id
6071                ),
6072            ),
6073        )?))
6074    }
6075
6076    /// Ordinary `route.open` refusals go through here; admission and breaker
6077    /// refusals log separately with their capacity or breaker state. The daemon can
6078    /// attest which code it sent: without the event, a client's "the daemon
6079    /// refused me" and the daemon's own view could only be reconciled by
6080    /// argument. Malformed input (`invalid_project_root`) does not come here;
6081    /// rejecting a request that was never a valid open is not a refusal of one.
6082    #[allow(clippy::too_many_arguments)]
6083    fn route_open_refusal_frame(
6084        &self,
6085        ctx: &RouteCtx,
6086        caller: &RouteOpenCaller,
6087        frame: &Frame,
6088        module_id: &str,
6089        reason: &'static str,
6090        code: &'static str,
6091        message: impl Into<String>,
6092    ) -> Result<Frame, RouterError> {
6093        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
6094        let detail = if matches!(
6095            reason,
6096            "registration_not_active"
6097                | "supervisor_not_live"
6098                | "no_forwarding_connection"
6099                | "reloading"
6100        ) {
6101            // A planned restart, reload or swap is already replacing the
6102            // process; it does not need a slot in the crash-restart budget.
6103            let expected_back = (reason == "reloading").then_some(true);
6104            self.route_open_recovery_detail(module_id, frame.header.corr, None, expected_back)?
6105        } else {
6106            None
6107        };
6108        control_error_body_frame(
6109            frame,
6110            ErrorBody {
6111                code: code.to_string(),
6112                message: message.into(),
6113                detail,
6114            },
6115        )
6116    }
6117
6118    fn route_open_recovery_detail(
6119        &self,
6120        module_id: &str,
6121        corr: u64,
6122        detail: Option<serde_json::Value>,
6123        expected_back: Option<bool>,
6124    ) -> Result<Option<serde_json::Value>, RouterError> {
6125        let Some((status, _)) = self.supervisor_status(module_id, corr)? else {
6126            return Ok(detail);
6127        };
6128        let mut detail = detail.unwrap_or_else(|| serde_json::json!({}));
6129        detail["state"] = serde_json::json!(status.state.to_string());
6130        if let Some(expected_back) = expected_back {
6131            detail["expected_back"] = serde_json::json!(expected_back);
6132        } else if let Some(module) = self.supervisor.get(module_id) {
6133            match module.will_recover_after_connection_loss() {
6134                Ok(expected_back) => detail["expected_back"] = serde_json::json!(expected_back),
6135                Err(err) => warn!(
6136                    %module_id,
6137                    error = %err,
6138                    "failed to read route.open recovery verdict; omitting expected_back"
6139                ),
6140            }
6141        }
6142        Ok(Some(detail))
6143    }
6144
6145    /// Answer a waiting `route.open` whose relayed bind was refused.
6146    ///
6147    /// `daemon_authored` is provenance, not a code match: the daemon
6148    /// synthesized the refusal itself (a drain, a late reply, a scope that
6149    /// ended mid-bind) or the module refused the bind with its own body. A
6150    /// daemon refusal carries the daemon-origin bit; a module's own
6151    /// rejection is relayed with the bit clear, so a module that refuses
6152    /// with `module_reloading` still cannot mint a daemon refusal.
6153    #[allow(clippy::too_many_arguments)]
6154    fn route_bind_rejection(
6155        &self,
6156        ctx: &RouteCtx,
6157        caller: &RouteOpenCaller,
6158        frame: &Frame,
6159        target_module_id: &str,
6160        body: ErrorBody,
6161        daemon_authored: bool,
6162    ) -> Result<Vec<Frame>, RouterError> {
6163        // The daemon's own commit re-check refused the bind because the
6164        // scope ended or changed after admission. The module accepted;
6165        // counting it as a module rejection would blame the module. This
6166        // branch only runs for daemon-synthesized bodies: a module that
6167        // refuses with a scope code is answered as a module rejection below.
6168        if daemon_authored {
6169            let scope_code = match body.code.as_str() {
6170                error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
6171                error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
6172                _ => None,
6173            };
6174            if let Some(code) = scope_code {
6175                self.observe_route_open_refusal(
6176                    ctx,
6177                    caller,
6178                    target_module_id,
6179                    "scope_changed_before_commit",
6180                    code,
6181                );
6182                return Ok(vec![control_error_body_frame(frame, body)?]);
6183            }
6184        }
6185        self.counters
6186            .increment_route_open_refused("module_rejected");
6187        info!(
6188            target: "control",
6189            code = "module_rejected",
6190            module_code = ?body.code,
6191            module_id = ?target_module_id,
6192            connection_id = ctx.connection_id.get(),
6193            principal = %caller.principal,
6194            scope = %caller.scope,
6195            harness = %caller.harness,
6196            "route.open refused"
6197        );
6198        let refusal = if daemon_authored {
6199            control_error_body_frame(frame, body)?
6200        } else {
6201            control_relayed_error_body_frame(frame, body)?
6202        };
6203        Ok(vec![refusal])
6204    }
6205
6206    /// Refuse a `route.open` because the target module's bind-relay breaker is
6207    /// open, without attempting the relay.
6208    ///
6209    /// The wire code is `module_timeout`, which is the truth (the module has
6210    /// not been answering binds) and which both SDKs already classify as
6211    /// retryable with capped backoff. Reusing it is what keeps this change out
6212    /// of both SDKs; the daemon-side distinction lives in the counter key
6213    /// instead.
6214    ///
6215    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
6216    /// While a breaker is open this fires on every open to that module, and the
6217    /// stall written up in `docs/designs/route-open-head-of-line.md` already
6218    /// produced 261 lines about a single module inside 3000 lines of daemon
6219    /// log. The rare transitions are logged at warn/info instead and the volume
6220    /// is carried by the counter, so the evidence survives without the flood.
6221    /// The debug line keeps a per-refusal record reachable for whoever turns
6222    /// the level up.
6223    #[allow(clippy::too_many_arguments)]
6224    fn route_open_breaker_refusal_frame(
6225        &self,
6226        ctx: &RouteCtx,
6227        caller: &RouteOpenCaller,
6228        frame: &Frame,
6229        module_id: &str,
6230        consecutive_timeouts: u32,
6231        retry_in: Duration,
6232        probe_in_flight: bool,
6233    ) -> Result<Frame, RouterError> {
6234        self.counters
6235            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
6236        debug!(
6237            target: "control",
6238            code = "module_timeout",
6239            module_id = ?module_id,
6240            connection_id = ctx.connection_id.get(),
6241            principal = %caller.principal,
6242            scope = %caller.scope,
6243            harness = %caller.harness,
6244            consecutive_timeouts,
6245            retry_in_ms = retry_in.as_millis() as u64,
6246            probe_in_flight,
6247            "route.open refused by open bind-relay breaker"
6248        );
6249        // Say what a caller can act on. An open bind-relay breaker means the
6250        // module timed out accepting several new routes in a row. The module
6251        // is still running and its established routes keep working; only new
6252        // route.open requests are refused until the cooldown ends and one
6253        // test route (the probe) gets through. A message that only counts
6254        // failed relays reads as "the module is down" to a worker that sees it.
6255        let detail = if probe_in_flight {
6256            "one test route is already being tried; retry once it settles".to_string()
6257        } else {
6258            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
6259        };
6260        control_error_frame(
6261            frame,
6262            "module_timeout",
6263            format!(
6264                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
6265                 timed out in a row); {detail}; its established routes are unaffected"
6266            ),
6267        )
6268    }
6269
6270    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
6271    /// requester's bytes (an unknown target is whatever the client sent) and
6272    /// is Debug-formatted so control characters land in the log escaped
6273    /// rather than as terminal sequences for whoever tails it.
6274    ///
6275    /// `reason` names the check that refused, because one wire code has
6276    /// several senders: after a module registers, `target_unavailable` can
6277    /// come from a missing role, an inactive registration, a supervisor that
6278    /// has not marked the process live, a missing forwarding connection, or a
6279    /// failed relay, and a log that records only the code cannot say which of
6280    /// them fired. It is a static, daemon-chosen label per branch, so it is
6281    /// safe to print plainly and stays a closed set.
6282    fn observe_route_open_refusal(
6283        &self,
6284        ctx: &RouteCtx,
6285        caller: &RouteOpenCaller,
6286        module_id: &str,
6287        reason: &'static str,
6288        code: &'static str,
6289    ) {
6290        self.counters.increment_route_open_refused(code);
6291        info!(
6292            target: "control",
6293            code,
6294            reason,
6295            module_id = ?module_id,
6296            connection_id = ctx.connection_id.get(),
6297            principal = %caller.principal,
6298            scope = %caller.scope,
6299            harness = %caller.harness,
6300            "route.open refused"
6301        );
6302        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
6303            self.route_outages.record_not_serving(module_id, reason);
6304        }
6305    }
6306
6307    /// Record an ACCEPTED route.open.
6308    ///
6309    /// Refusals have been logged and counted since the attestation work; accepts
6310    /// were invisible, so the daemon knew every principal it stamped and wrote
6311    /// none of them down. The party that attests the identity was the only party
6312    /// not recording it, which left a credential vault unable to name the sender
6313    /// of a call that reached it (claustrum #43) and left the launch-nonce
6314    /// concurrency question unanswerable from the outside.
6315    ///
6316    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
6317    /// `code`/`module_id`/`connection_id` returns both directions of the same
6318    /// decision rather than two shapes a reader has to join by hand.
6319    ///
6320    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
6321    /// and the difference carries information rather than being an
6322    /// inconsistency. This line is only reachable after a successful bind to a
6323    /// REGISTERED module, so the value has already passed HELLO validation
6324    /// including the path-hazard refusal and cannot contain control bytes. A
6325    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
6326    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
6327    ///
6328    /// Bare is also what every other daemon line already emits (`module
6329    /// registered`, `configured module supervised`). Shipping `?module_id` here
6330    /// made this instrument the only one in the file whose ids did not answer
6331    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
6332    /// line whose whole purpose is being grepped beside its sibling.
6333    ///
6334    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
6335    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
6336    /// forwards every field type through it and a bare `&str` and a `?`-escaped
6337    /// one are recorded identically. A test written against that harness passes
6338    /// either way -- I wrote one, measured it, and deleted it rather than ship a
6339    /// green assertion that cannot fail. The same limit applies to the escaping
6340    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
6341    /// it reads as a guard on the Debug escaping and cannot detect its removal.
6342    /// Fencing either needs the real formatter, not the capture layer.
6343    ///
6344    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
6345    /// unix-socket options and subc is loopback TCP, so there is no peer identity
6346    /// to record. The ephemeral port would decay within minutes and answer only a
6347    /// live question. The identity question is instead answered by counting
6348    /// distinct live connections presenting one module's `consumer_identity` --
6349    /// "is anyone else holding this secret" rather than "is this the right
6350    /// process".
6351    fn observe_route_open_accept(
6352        &self,
6353        ctx: &RouteCtx,
6354        module_id: &str,
6355        principal: &str,
6356        scope: &str,
6357    ) {
6358        self.route_outages.record_accepted(module_id);
6359        self.counters.increment_route_open_accepted(principal);
6360        info!(
6361            target: "control",
6362            principal,
6363            module_id,
6364            connection_id = ctx.connection_id.get(),
6365            scope = %scope,
6366            "route.open accepted"
6367        );
6368    }
6369
6370    fn supervised_absent_route_open_refusal_frame(
6371        &self,
6372        ctx: &RouteCtx,
6373        caller: &RouteOpenCaller,
6374        frame: &Frame,
6375        module_id: &str,
6376        code: &'static str,
6377        status: &crate::supervise::ModuleStatus,
6378    ) -> Result<Frame, RouterError> {
6379        self.counters.increment_route_open_refused(code);
6380        info!(
6381            target: "control",
6382            code,
6383            reason = "supervised_not_registered",
6384            module_id = ?module_id,
6385            connection_id = ctx.connection_id.get(),
6386            principal = %caller.principal,
6387            scope = %caller.scope,
6388            harness = %caller.harness,
6389            state = %status.state,
6390            enabled = status.enabled,
6391            live = status.live,
6392            "route.open refused"
6393        );
6394        // A supervised module whose process has not registered is not
6395        // serving, whatever the reason; the supervisor knows this id, so it is
6396        // safe to track.
6397        self.route_outages
6398            .record_not_serving(module_id, "supervised_not_registered");
6399        let message = if let Some(pid) = status.waiting_for_previous_pid {
6400            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
6401                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
6402            })
6403        } else {
6404            format!(
6405                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
6406                status.state, status.enabled, status.live
6407            )
6408        };
6409        // `module_warming` here means the process is starting, running or
6410        // being restarted and has not registered yet. Nothing has to be
6411        // restarted for it to serve, so the crash budget does not decide it.
6412        let expected_back = (code == error_codes::MODULE_WARMING).then_some(true);
6413        control_error_body_frame(
6414            frame,
6415            ErrorBody {
6416                code: code.to_string(),
6417                message,
6418                detail: self.route_open_recovery_detail(
6419                    module_id,
6420                    frame.header.corr,
6421                    None,
6422                    expected_back,
6423                )?,
6424            },
6425        )
6426    }
6427
6428    async fn handle_route_open(
6429        &self,
6430        ctx: &RouteCtx,
6431        frame: Frame,
6432        request: RouteOpenRequest,
6433        timings: &mut RouteOpenTimings,
6434    ) -> Result<Vec<Frame>, RouterError> {
6435        let RouteOpenRequest {
6436            target,
6437            mut identity,
6438            consumer_identity,
6439            consumer_capabilities,
6440            role_versions,
6441            busy_refusals,
6442            admission_facts,
6443            scope,
6444            accepts_close_reasons,
6445        } = request;
6446        let target_module_id = target_module_id(&target).to_string();
6447        timings.module_id.clone_from(&target_module_id);
6448        if self
6449            .registry
6450            .get_module_by_connection(ctx.connection_id)
6451            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6452            .is_some()
6453        {
6454            return Ok(vec![control_error_frame(
6455                &frame,
6456                "invalid_request",
6457                "module connections cannot open client routes",
6458            )?]);
6459        }
6460        debug!(
6461            connection_id = ctx.connection_id.get(),
6462            corr = frame.header.corr,
6463            module_id = %target_module_id,
6464            "handling route.open"
6465        );
6466
6467        // The caller's attested principal, requested scope and harness, for
6468        // every `route.open refused` line below. The connection's claim
6469        // binding or launch nonce is checked here, once, and the answer is
6470        // reused where the principal is enforced further down, so the order
6471        // of refusals is unchanged.
6472        let attested =
6473            self.attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref());
6474        let mut caller = RouteOpenCaller::new(
6475            attested.as_ref().map(|attested| &attested.principal),
6476            scope.as_ref(),
6477            &identity,
6478        );
6479
6480        // A malformed declaration is refused first, before anything about the
6481        // target is looked up: the same body would be refused against any
6482        // module, so the caller learns nothing by retrying or waiting. An empty
6483        // map declares nothing and travels as no field at all, so a provider
6484        // only ever sees a missing field or a non-empty one.
6485        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
6486        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
6487            self.observe_route_open_refusal(
6488                ctx,
6489                &caller,
6490                &target_module_id,
6491                "invalid_role_versions",
6492                error_codes::INVALID_REQUEST,
6493            );
6494            return Ok(vec![control_error_body_frame(
6495                &frame,
6496                ErrorBody {
6497                    code: error_codes::INVALID_REQUEST.to_string(),
6498                    message: error.to_string(),
6499                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
6500                },
6501            )?]);
6502        }
6503
6504        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
6505            if status.waiting_for_previous_pid.is_some() {
6506                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6507                    ctx,
6508                    &caller,
6509                    &frame,
6510                    &target_module_id,
6511                    "module_warming",
6512                    &status,
6513                )?]);
6514            }
6515        }
6516
6517        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
6518        // opposite. Below, a caller learns whether a module is unregistered,
6519        // supervised-but-down (with state/enabled/live), or registered without the
6520        // requested role. Elsewhere that is an enumeration leak: a probe learning
6521        // the shape of a fleet it cannot otherwise see.
6522        //
6523        // It is not one here, and the reason is the ACCESS MODEL rather than
6524        // anything about these errors. Reaching route.open requires the
6525        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
6526        // connection file, so any caller who completes it already runs as this
6527        // user -- and can read subc.jsonc for the module list and `ck module
6528        // status` for live state. The reply discloses nothing the caller cannot
6529        // read more easily from disk, while the precision is load-bearing:
6530        // `unknown_module` is retryable and a missing role is not.
6531        //
6532        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
6533        // remote transport, a sandboxed caller, a shared-host mode -- THAT
6534        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
6535        let admission_registry = self
6536            .registry
6537            .route_admission_snapshot()
6538            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6539        let Some(registration) =
6540            crate::scopes::admission::registered_target(&admission_registry, &target_module_id)
6541        else {
6542            if let Some((status, warming)) =
6543                self.supervisor_status(&target_module_id, frame.header.corr)?
6544            {
6545                // BEFORE the two availability codes below, because for a module
6546                // that speaks no subc wire both of them are false comfort: they
6547                // say "not right now" and are retried, and this module will
6548                // never register no matter how long the caller waits. The
6549                // absence here is the declaration being honoured, not a module
6550                // that is late.
6551                if status.protocol == ModuleProtocol::None {
6552                    return Ok(vec![self.route_open_refusal_frame(
6553                        ctx,
6554                        &caller,
6555                        &frame,
6556                        &target_module_id,
6557                        "protocol_none",
6558                        error_codes::MODULE_NO_PROTOCOL,
6559                        format!(
6560                            "module_id '{target_module_id}' is declared protocol: none; \
6561                             it speaks no subc wire and serves no routes"
6562                        ),
6563                    )?]);
6564                }
6565                let code = if warming {
6566                    "module_warming"
6567                } else {
6568                    "target_unavailable"
6569                };
6570                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6571                    ctx,
6572                    &caller,
6573                    &frame,
6574                    &target_module_id,
6575                    code,
6576                    &status,
6577                )?]);
6578            }
6579            if let Some(removed_ago_ms) =
6580                self.supervisor.removal_tombstone_age_ms(&target_module_id)
6581            {
6582                return Ok(vec![self.route_open_refusal_frame(
6583                    ctx,
6584                    &caller,
6585                    &frame,
6586                    &target_module_id,
6587                    "removed",
6588                    error_codes::MODULE_REMOVED,
6589                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
6590                )?]);
6591            }
6592            return Ok(vec![self.route_open_refusal_frame(
6593                ctx,
6594                &caller,
6595                &frame,
6596                &target_module_id,
6597                "not_registered",
6598                error_codes::UNKNOWN_MODULE,
6599                format!("module_id '{target_module_id}' is not registered"),
6600            )?]);
6601        };
6602
6603        // Best-effort only: registry readiness and forwarding reservation use
6604        // different locks, so a module can flip readiness between this read and
6605        // the relay. Modules must still tolerate an `on_bind` while not ready.
6606        if crate::scopes::admission::target_registration(&admission_registry, &target_module_id)
6607            .is_err()
6608        {
6609            self.counters
6610                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
6611            info!(
6612                target: "control",
6613                code = error_codes::MODULE_WARMING,
6614                module_id = ?target_module_id,
6615                connection_id = ctx.connection_id.get(),
6616                principal = %caller.principal,
6617                scope = %caller.scope,
6618                harness = %caller.harness,
6619                reason = "declared_not_ready",
6620                "route.open refused"
6621            );
6622            // The module is registered but says it cannot take work, which is
6623            // an outage from the caller's side even though its process is up.
6624            self.route_outages
6625                .record_not_serving(&target_module_id, "declared_not_ready");
6626            // The registered process can declare itself ready without a
6627            // restart, so an exhausted crash budget does not prevent serving.
6628            return Ok(vec![control_error_body_frame(
6629                &frame,
6630                ErrorBody {
6631                    code: error_codes::MODULE_WARMING.to_string(),
6632                    message: format!(
6633                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
6634                    ),
6635                    detail: self.route_open_recovery_detail(
6636                        &target_module_id,
6637                        frame.header.corr,
6638                        Some(serde_json::json!({ "reason": "declared_not_ready" })),
6639                        Some(true),
6640                    )?,
6641                },
6642            )?]);
6643        }
6644
6645        // Effective readiness, second half: a module that declares a capability
6646        // `need: required` is not routable while that capability has no
6647        // registered provider. It is enforced HERE, as a retryable routing
6648        // refusal, and deliberately not as spawn ordering or a boot block. The
6649        // module is still started and registered and can make its own calls;
6650        // spawn ordering is a promise that cannot be kept once a provider
6651        // crashes at runtime, and refusing to boot would stop the whole
6652        // machine, including the tools needed to fix its configuration.
6653        //
6654        // "Provided" is the evaluator's verdict, which counts a provider as
6655        // soon as it has REGISTERED, not once it is ready. Two modules that
6656        // require each other's capabilities are therefore both routable once
6657        // both register; counting readiness instead would deadlock them.
6658        //
6659        // Only new opens are refused. Routes already bound when a provider
6660        // goes away stay bound: nothing here tears them down, and the module
6661        // answers them as it can. Like the readiness read above this is
6662        // best-effort against a provider registering or leaving concurrently.
6663        if let Some(capability) = self.first_unprovided_required_capability(registration) {
6664            self.counters
6665                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
6666            info!(
6667                target: "control",
6668                code = error_codes::MODULE_WARMING,
6669                module_id = ?target_module_id,
6670                connection_id = ctx.connection_id.get(),
6671                principal = %caller.principal,
6672                scope = %caller.scope,
6673                harness = %caller.harness,
6674                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6675                capability = %capability,
6676                "route.open refused"
6677            );
6678            return Ok(vec![control_error_body_frame(
6679                &frame,
6680                ErrorBody {
6681                    code: error_codes::MODULE_WARMING.to_string(),
6682                    message: format!(
6683                        "module_id '{target_module_id}' requires capability '{capability}', \
6684                         which no registered module provides; retry"
6685                    ),
6686                    detail: Some(serde_json::json!({
6687                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6688                        "capability": capability,
6689                    })),
6690                },
6691            )?]);
6692        }
6693
6694        if !target_has_required_role(&target, &registration.manifest.provides) {
6695            return Ok(vec![self.route_open_refusal_frame(
6696                ctx,
6697                &caller,
6698                &frame,
6699                &target_module_id,
6700                "role_not_provided",
6701                "target_unavailable",
6702                format!("module_id '{target_module_id}' does not provide the requested target"),
6703            )?]);
6704        }
6705
6706        if registration.state != ChannelState::Active {
6707            return Ok(vec![self.route_open_refusal_frame(
6708                ctx,
6709                &caller,
6710                &frame,
6711                &target_module_id,
6712                "registration_not_active",
6713                "target_unavailable",
6714                format!("module_id '{target_module_id}' is not active"),
6715            )?]);
6716        }
6717
6718        if self
6719            .forwarding
6720            .module_is_draining(&target_module_id)
6721            .map_err(RouterError::Forwarding)?
6722        {
6723            return Ok(vec![self.route_open_refusal_frame(
6724                ctx,
6725                &caller,
6726                &frame,
6727                &target_module_id,
6728                "reloading",
6729                "module_reloading",
6730                format!("module_id '{target_module_id}' is reloading"),
6731            )?]);
6732        }
6733
6734        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
6735            process_liveness.process_live(&target_module_id) == Some(false)
6736        }) {
6737            // A module the supervisor is restarting or reloading can still hold
6738            // a registration: the old process before its connection closes, or
6739            // a new one that registered while the supervisor was draining. The
6740            // forwarding table does not see that as draining, but the consumer
6741            // should still be told to retry soon, exactly as for the drain
6742            // above, rather than that the target is unavailable.
6743            if process_liveness.process_replacing(&target_module_id) {
6744                return Ok(vec![self.route_open_refusal_frame(
6745                    ctx,
6746                    &caller,
6747                    &frame,
6748                    &target_module_id,
6749                    "reloading",
6750                    "module_reloading",
6751                    format!("module_id '{target_module_id}' is reloading"),
6752                )?]);
6753            }
6754            return Ok(vec![self.route_open_refusal_frame(
6755                ctx,
6756                &caller,
6757                &frame,
6758                &target_module_id,
6759                "supervisor_not_live",
6760                "target_unavailable",
6761                format!("module_id '{target_module_id}' is not live"),
6762            )?]);
6763        }
6764
6765        if !self
6766            .forwarding
6767            .has_live_module_connection(&target_module_id)
6768            .map_err(RouterError::Forwarding)?
6769        {
6770            return Ok(vec![self.route_open_refusal_frame(
6771                ctx,
6772                &caller,
6773                &frame,
6774                &target_module_id,
6775                "no_forwarding_connection",
6776                "target_unavailable",
6777                format!("module_id '{target_module_id}' has no live forwarding connection"),
6778            )?]);
6779        }
6780
6781        if let Some(error) =
6782            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
6783        {
6784            self.observe_route_open_refusal(
6785                ctx,
6786                &caller,
6787                &target_module_id,
6788                "op_not_allowed",
6789                "op_not_allowed",
6790            );
6791            return Ok(vec![error]);
6792        }
6793
6794        let principal = match (&attested, consumer_identity.is_some()) {
6795            // A claim-bound connection whose launch ended attests nothing,
6796            // and it must never fall back to opening as `direct`.
6797            (None, false) => {
6798                self.observe_route_open_refusal(
6799                    ctx,
6800                    &caller,
6801                    &target_module_id,
6802                    "registration_channel_closed",
6803                    crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6804                );
6805                return Ok(vec![control_error_frame(
6806                    &frame,
6807                    crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6808                    "the claim binding's launch has ended",
6809                )?]);
6810            }
6811            _ => match self.route_open_principal_attested(
6812                &frame,
6813                consumer_identity,
6814                attested.as_ref().map(|attested| attested.principal.clone()),
6815            )? {
6816                Ok(principal) => principal,
6817                Err(error) => {
6818                    self.observe_route_open_refusal(
6819                        ctx,
6820                        &caller,
6821                        &target_module_id,
6822                        "bad_consumer_identity",
6823                        "bad_consumer_identity",
6824                    );
6825                    return Ok(vec![error]);
6826                }
6827            },
6828        };
6829        let attested = attested.expect("an attested principal was resolved above");
6830        timings.principal = principal_label(&principal);
6831
6832        // The opener's spawn authority, read once here and carried with the
6833        // relay; nothing below reads its record again.
6834        let Ok(mut caller_capture) = attested.capture() else {
6835            self.observe_route_open_refusal(
6836                ctx,
6837                &caller,
6838                &target_module_id,
6839                "registration_channel_closed",
6840                crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6841            );
6842            return Ok(vec![control_error_frame(
6843                &frame,
6844                crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6845                "the opener's launch has ended",
6846            )?]);
6847        };
6848        // The caller-side checks below read the opener's own spawn: its
6849        // module from its record and its manifest from its own registration,
6850        // never another launch's active registration under the same id.
6851        let caller_module_id = match &attested.spawn {
6852            Some((record, _)) => Some(record.module_id().to_owned()),
6853            None => match &principal {
6854                Principal::Reserved { module_id } => Some(module_id.clone()),
6855                _ => None,
6856            },
6857        };
6858
6859        // Only Callosum can attest a production phone build. Consume the claim
6860        // here so it can be kept on the route without reaching any module.
6861        let build_class = identity.build_class.take().filter(|build_class| {
6862            if caller_module_id.as_deref() == Some("callosum") {
6863                build_class == subc_protocol::BUILD_CLASS_PRODUCTION
6864            } else {
6865                if self
6866                    .build_class_warned
6867                    .lock()
6868                    .unwrap_or_else(|p| p.into_inner())
6869                    .insert(ctx.connection_id)
6870                {
6871                    warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
6872                        "ignoring build_class from a non-Callosum opener");
6873                }
6874                false
6875            }
6876        });
6877
6878        // This is attested, control-plane policy for supervised module origins.
6879        // Keep it before route reservation and out of the opaque forwarding hot
6880        // path: data frames must never acquire a per-frame capability check.
6881        if let Some(opening_registration) = self.caller_registration(&attested, &admission_registry)
6882        {
6883            let opening_module_id = opening_registration.manifest.module_id.as_str();
6884            if let Some(capability) =
6885                denied_capability(&opening_registration.manifest, &registration.manifest)
6886            {
6887                warn!(
6888                    opening_module_id,
6889                    target_module_id,
6890                    capability,
6891                    "refusing route.open because an attested capability deny edge matches"
6892                );
6893                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6894                    &frame,
6895                    &target_module_id,
6896                    "capability_deny_edge",
6897                    "capability_forbidden",
6898                    format!(
6899                        "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
6900                    ),
6901                )?]);
6902            }
6903        }
6904
6905        if admission_facts.is_some() {
6906            let carrier_matches = caller_module_id.is_some()
6907                && self.admission_facts_carrier_module_id.as_deref() == caller_module_id.as_deref();
6908            if !carrier_matches {
6909                return Ok(vec![self.route_open_refusal_frame(
6910                    ctx,
6911                    &caller,
6912                    &frame,
6913                    &target_module_id,
6914                    "admission_facts_carrier_not_permitted",
6915                    "admission_facts_not_permitted",
6916                    "admission facts may only be carried by the configured reserved module",
6917                )?]);
6918            }
6919
6920            let target_allowed = self
6921                .admission_facts_targets
6922                .as_ref()
6923                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
6924            if !target_allowed {
6925                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6926                    &frame,
6927                    &target_module_id,
6928                    "admission_facts_target_not_listed",
6929                    "admission_facts_target_not_allowed",
6930                    format!(
6931                        "admission facts are not permitted for target module_id '{target_module_id}'"
6932                    ),
6933                )?]);
6934            }
6935
6936            // Keep the value opaque to subc. The downstream admission validator owns
6937            // schema and semantic checks; this daemon only enforces carrier authority
6938            // and the configured destination allowlist.
6939        }
6940
6941        // Scope admission, on the attested principal above and never on the
6942        // request body. The tag read here travels with the pending bind and is
6943        // compared with the published one at commit, so a sync between here
6944        // and the module's ack refuses the open instead of binding a stamp
6945        // that is no longer true.
6946        let (bound_scope, mut scope_stamp, stamp_required) = match &scope {
6947            None => (None, None, false),
6948            Some(selector) => {
6949                let owner_configured = match &selector.owner {
6950                    // A reserved owner counts as configured from before its
6951                    // process is spawned (see `SupervisorHandle::is_configured`).
6952                    // So an owner that has not synced its scopes yet is refused
6953                    // as retryable (`scope_not_synced`), not as one that will
6954                    // never sync.
6955                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
6956                    _ => false,
6957                };
6958                let mut scopes = self
6959                    .scopes
6960                    .write()
6961                    .unwrap_or_else(|poisoned| poisoned.into_inner());
6962                let admitted = crate::scopes::admission::admit_route(
6963                    &scopes,
6964                    &admission_registry,
6965                    &principal,
6966                    &target_module_id,
6967                    Some(selector),
6968                    owner_configured,
6969                    crate::scopes::admission::AdmissionMode::Open,
6970                )
6971                .map(|route| {
6972                    let admission = route.scope.expect("scoped admission");
6973                    let bound = scopes.reserve_bind(
6974                        &admission,
6975                        &principal,
6976                        ctx.connection_id,
6977                        selector.hold_presence,
6978                    );
6979                    (admission, bound)
6980                });
6981                match admitted {
6982                    Ok((admission, bound)) => {
6983                        (Some(bound), Some(admission.stamp), admission.stamp_required)
6984                    }
6985                    Err(refusal) => {
6986                        return Ok(vec![self.route_open_refusal_frame(
6987                            ctx,
6988                            &caller,
6989                            &frame,
6990                            &target_module_id,
6991                            refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
6992                            refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
6993                            refusal.message,
6994                        )?]);
6995                    }
6996                }
6997            }
6998        };
6999        // From here on a refusal names the scope by the stamp admission
7000        // issued, exactly as the `route.open accepted` line does.
7001        if let Some(stamp) = &scope_stamp {
7002            caller.scope = route_open_scope_label(stamp);
7003        }
7004        // Decided from the admitted stamp, before any field of it is stripped
7005        // for a target that lacks the capability to read it.
7006        if scope_stamp.as_ref().is_some_and(scope_acts_for_another) {
7007            caller_capture = caller_capture.map(crate::spawn_authority::CallerCapture::relayed);
7008        }
7009
7010        // Allow resolvable missing-tail paths so a vanished directory does not
7011        // prevent reopening an existing session for cleanup. Binding does not
7012        // authorize creation of durable provider state.
7013        //
7014        // Resolve real paths rather than lexical cleanup: canonicalize the
7015        // longest existing ancestor and append the missing tail, retaining the
7016        // path's real existing ancestry even when a leaf is missing.
7017        timings.enter(RouteOpenPhase::ProjectRoot);
7018        let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
7019            Ok(project) => project,
7020            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
7021        };
7022        timings.enter(RouteOpenPhase::Admission);
7023        let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
7024            Ok(root) => root,
7025            Err(err) => {
7026                return Ok(vec![control_error_frame(
7027                    &frame,
7028                    "invalid_project_root",
7029                    err.to_string(),
7030                )?])
7031            }
7032        };
7033        // Compute compatibility project fields before stripping optional stamp
7034        // members. Preserve the canonical opener root and its project ID,
7035        // including absence. For rootless openers, use stamp workdir or `/`, and
7036        // opener ID, else stamp ID, else no ID. Emit these identity fields only
7037        // to targets without bind-no-project/v1.
7038        let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
7039
7040        // Last gate before any relay work, and deliberately after the cheap
7041        // registry and availability checks above: those name a more precise
7042        // condition (unknown, removed, reloading) and a caller is better served
7043        // by the precise code than by this one.
7044        //
7045        // Everything below this point costs an egress permit, a reserved handle
7046        // pair and, if the module does not answer, the whole relay budget. The
7047        // reader no longer waits for that budget, so cap each target explicitly;
7048        // serial dispatch used to provide the accidental cap of one relay per
7049        // connection. Admission is a mutex-protected count and never waits.
7050        let _concurrency_guard = match self
7051            .route_bind_concurrency
7052            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
7053        {
7054            Ok(guard) => guard,
7055            Err(in_flight) => {
7056                return Ok(vec![self.route_open_target_capacity_refusal(
7057                    ctx,
7058                    &caller,
7059                    &frame,
7060                    &target_module_id,
7061                    in_flight,
7062                )?]);
7063            }
7064        };
7065
7066        // A module that has already burned the whole budget `threshold` times
7067        // in a row does not get to charge it again until a probe says it recovered.
7068        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
7069            RouteBindAdmission::Admitted { guard, probe } => {
7070                if probe {
7071                    info!(
7072                        module_id = %target_module_id,
7073                        connection_id = ctx.connection_id.get(),
7074                        "route.bind breaker half-open: admitting one probe"
7075                    );
7076                }
7077                guard
7078            }
7079            RouteBindAdmission::Refused {
7080                consecutive_timeouts,
7081                retry_in,
7082                probe_in_flight,
7083            } => {
7084                return Ok(vec![self.route_open_breaker_refusal_frame(
7085                    ctx,
7086                    &caller,
7087                    &frame,
7088                    &target_module_id,
7089                    consecutive_timeouts,
7090                    retry_in,
7091                    probe_in_flight,
7092                )?]);
7093            }
7094        };
7095
7096        // Resolve the per-module budget here so the wait matches the operator's
7097        // intent for this specific target. A per-module override in
7098        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
7099        // daemons) wins over the daemon-wide default.
7100        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
7101        let relay_deadline = Instant::now() + route_bind_relay_timeout;
7102        let operator_app = self
7103            .operator_apps
7104            .lock()
7105            .unwrap_or_else(|p| p.into_inner())
7106            .get(&ctx.connection_id)
7107            .cloned();
7108        timings.enter(RouteOpenPhase::EgressReservation);
7109        let pending = match self
7110            .forwarding
7111            .begin_route_bind_relay_for(
7112                ctx.connection_id,
7113                ctx.egress.clone(),
7114                response_version(&frame),
7115                frame.header.corr,
7116                &target_module_id,
7117                principal.clone(),
7118                bound_scope,
7119                project_root,
7120                identity.harness.clone(),
7121                build_class,
7122                operator_app.clone(),
7123                crate::forwarding::RouteAuthority {
7124                    incarnation: self.supervisor.daemon_incarnation(),
7125                    caller: caller_capture
7126                        .as_ref()
7127                        .map(|caller| caller.authority.pair()),
7128                },
7129                relay_deadline,
7130                busy_refusals == Some(true),
7131                accepts_close_reasons,
7132            )
7133            .await
7134        {
7135            Ok(pending) => pending,
7136            Err(err) => {
7137                return Ok(vec![self.route_open_refusal_frame(
7138                    ctx,
7139                    &caller,
7140                    &frame,
7141                    &target_module_id,
7142                    "relay_reservation_failed",
7143                    forwarding_error_code(&err),
7144                    err.to_string(),
7145                )?])
7146            }
7147        };
7148        timings.enter(RouteOpenPhase::Admission);
7149        let crate::forwarding::PendingRouteBindRelay {
7150            endpoint,
7151            module_sink,
7152            negotiated_ver,
7153            client_channel,
7154            client_epoch,
7155            module_channel,
7156            module_epoch,
7157            corr: relay_corr,
7158            receiver,
7159        } = pending;
7160        let mut reservation =
7161            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
7162
7163        // Reserving egress can wait while a module reconnects or a swap cuts
7164        // over. Check the relay's captured connection, not the earlier by-id
7165        // lookup: the original module's flow/run capabilities cannot authorize
7166        // its replacement. The captured sink stays bound to that connection.
7167        let relay_registration = self
7168            .registry
7169            .get_module_by_connection(endpoint.connection_id)
7170            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7171        let relay_capabilities = relay_registration
7172            .as_ref()
7173            .and_then(|registration| registration.manifest.capabilities.as_ref());
7174        if let Some(selector) = &scope {
7175            // Egress reservation may wait across a scope update. Recheck the
7176            // proof against the current record before sending the captured bind.
7177            if let Err(refusal) = self
7178                .scopes
7179                .read()
7180                .unwrap_or_else(|poisoned| poisoned.into_inner())
7181                .check_bind_proof(&principal, selector)
7182            {
7183                reservation.release_and_disarm();
7184                return Ok(vec![self.route_open_refusal_frame(
7185                    ctx,
7186                    &caller,
7187                    &frame,
7188                    &target_module_id,
7189                    refusal.code,
7190                    refusal.code,
7191                    refusal.message,
7192                )?]);
7193            }
7194        }
7195        if let Some(stamp) = scope_stamp.as_ref() {
7196            if let Err(refusal) = crate::scopes::check_target_required_stamp(
7197                stamp_required,
7198                &target_module_id,
7199                relay_capabilities,
7200            )
7201            .and_then(|()| {
7202                crate::scopes::check_target_flow_support(
7203                    stamp,
7204                    &target_module_id,
7205                    relay_capabilities,
7206                )
7207            })
7208            .and_then(|()| {
7209                crate::scopes::check_target_agent_run_support(
7210                    stamp,
7211                    &target_module_id,
7212                    relay_capabilities,
7213                )
7214            }) {
7215                reservation.release_and_disarm();
7216                return Ok(vec![self.route_open_refusal_frame(
7217                    ctx,
7218                    &caller,
7219                    &frame,
7220                    &target_module_id,
7221                    refusal.code,
7222                    refusal.code,
7223                    refusal.message,
7224                )?]);
7225            }
7226        }
7227
7228        // Older readers may reject unknown fields. Select optional fields from
7229        // the captured endpoint's declaration, including on unscoped binds.
7230        if scope_stamp
7231            .as_ref()
7232            .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
7233            && !crate::scopes::target_supports_federated(relay_capabilities)
7234        {
7235            // Keep bound_scope: even an unstamped legacy route belongs to this epoch.
7236            scope_stamp = None;
7237        }
7238        if !relay_capabilities.is_some_and(|caps| {
7239            caps.provides
7240                .iter()
7241                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
7242        }) {
7243            if let Some(stamp) = scope_stamp.as_mut() {
7244                stamp.workdir = None;
7245                stamp.project_id = None;
7246            }
7247        }
7248        // A worker run has no agent_id; on_behalf_of_agent is its only agent
7249        // attribution. The run gate above has already required the target to
7250        // declare `worker-runs/v1`, which promises to read both
7251        // `on_behalf_of_agent` and `call_mode`.
7252        let worker_run = scope_stamp
7253            .as_ref()
7254            .is_some_and(crate::scopes::is_worker_run)
7255            && relay_capabilities.is_some_and(|caps| {
7256                caps.provides
7257                    .iter()
7258                    .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
7259            });
7260        if !worker_run
7261            && !relay_capabilities.is_some_and(|caps| {
7262                caps.provides
7263                    .iter()
7264                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
7265            })
7266        {
7267            if let Some(stamp) = scope_stamp.as_mut() {
7268                stamp.attributes.on_behalf_of_agent = None;
7269            }
7270        }
7271        if !relay_capabilities.is_some_and(|caps| {
7272            caps.provides
7273                .iter()
7274                .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
7275        }) {
7276            if let Some(stamp) = scope_stamp.as_mut() {
7277                stamp.attributes.flow_version = None;
7278                stamp.attributes.flow_code_hash = None;
7279                stamp.attributes.flow_install_id = None;
7280            }
7281        }
7282        let call_mode = if worker_run
7283            || relay_capabilities
7284                .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
7285        {
7286            Some(derive_call_mode(scope_stamp.as_ref()))
7287        } else {
7288            if let Some(stamp) = scope_stamp.as_mut() {
7289                stamp.expires_at_ms = None;
7290            }
7291            None
7292        };
7293
7294        debug!(
7295            connection_id = ctx.connection_id.get(),
7296            client_channel,
7297            client_epoch,
7298            module_channel,
7299            module_epoch,
7300            "reserved route handle pair"
7301        );
7302        // Rendered BEFORE the move into the relay, because the accept arm below
7303        // is where it is logged and the principal is gone by then.
7304        let route_principal_label = principal_label(&principal);
7305        let relay = ModuleControlRequest::RouteBind {
7306            route_channel: module_channel,
7307            epoch: module_epoch,
7308            target,
7309            identity,
7310            principal: Some(principal),
7311            consumer_capabilities,
7312            role_versions,
7313            admission_facts,
7314            scope: scope_stamp,
7315            call_mode,
7316        };
7317        let peer = RouteBindPeer::new()
7318            .with_observed_peer(
7319                relay_capabilities
7320                    .is_some_and(|caps| {
7321                        caps.provides
7322                            .iter()
7323                            .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
7324                    })
7325                    .then(|| {
7326                        self.observed_peers
7327                            .lock()
7328                            .unwrap_or_else(|p| p.into_inner())
7329                            .get(&ctx.connection_id)
7330                            .cloned()
7331                            .unwrap_or_else(ObservedPeer::unobserved)
7332                    }),
7333            )
7334            .with_operator_app(
7335                self.connection_operator_app(operator_app.as_ref())
7336                    .filter(|_| {
7337                        relay_capabilities.is_some_and(|caps| {
7338                            caps.provides
7339                                .iter()
7340                                .any(|cap| cap == OPERATOR_APP_CAPABILITY)
7341                        })
7342                    }),
7343            )
7344            .with_caller(caller_for_target(
7345                caller_capture.as_ref(),
7346                relay_capabilities,
7347                &target_module_id,
7348            ));
7349        let relay_body = crate::bind_compat::encode_bind(
7350            &relay,
7351            &peer,
7352            &legacy_bind_project,
7353            relay_capabilities,
7354        )
7355        .map_err(|err| {
7356            RouterError::backend(
7357                0,
7358                frame.header.corr,
7359                format!("failed to encode route.bind request: {err}"),
7360            )
7361        })?;
7362        let relay_frame = Frame::build_with_version(
7363            negotiated_ver,
7364            FrameType::Request,
7365            control_flags(),
7366            0,
7367            0,
7368            relay_corr,
7369            relay_body,
7370        )
7371        .map_err(RouterError::FrameBuild)?;
7372
7373        timings.enter(RouteOpenPhase::BindSend);
7374        let sent = module_sink.send(relay_frame).await;
7375        timings.enter(RouteOpenPhase::Admission);
7376        if let Err(err) = sent {
7377            reservation.release_and_disarm();
7378            return Ok(vec![self.route_open_refusal_frame(
7379                ctx,
7380                &caller,
7381                &frame,
7382                &target_module_id,
7383                "relay_send_failed",
7384                "target_unavailable",
7385                err.to_string(),
7386            )?]);
7387        }
7388
7389        #[cfg(test)]
7390        if let Some(hold) = &self.route_bind_relay_sent_hold {
7391            // First wait: tell the test the bind is queued. Second wait: stay
7392            // here until the test has driven whatever it wants to happen
7393            // between queueing the bind and marking it enqueued.
7394            hold.wait().await;
7395            hold.wait().await;
7396        }
7397
7398        // A missing relay is not by itself an abandoned bind: the module may
7399        // already have acked it and the route may be committed. Only a relay
7400        // dropped without any module reply needs this GOODBYE; one a reply
7401        // settled has already had its GOODBYE decided by the reply path.
7402        if self
7403            .forwarding
7404            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
7405            .map_err(RouterError::Forwarding)?
7406            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
7407        {
7408            self.send_abandoned_route_bind_goodbye(
7409                &module_sink,
7410                negotiated_ver,
7411                module_channel,
7412                module_epoch,
7413            );
7414        }
7415
7416        timings.enter(RouteOpenPhase::BindAckWait);
7417        let outcome = timeout_at(relay_deadline, receiver).await;
7418        timings.enter(RouteOpenPhase::Admission);
7419        match outcome {
7420            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
7421                reservation.disarm();
7422                if breaker.record_accepted() {
7423                    info!(
7424                        module_id = %target_module_id,
7425                        "route.bind breaker closed: the probe was accepted"
7426                    );
7427                }
7428                self.observe_route_open_accept(
7429                    ctx,
7430                    &target_module_id,
7431                    &route_principal_label,
7432                    &caller.scope,
7433                );
7434                Ok(Vec::new())
7435            }
7436            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
7437                reservation.release_and_disarm();
7438                // A module that says no in microseconds is healthy. Rejection
7439                // is a different condition with its own refusal and must not
7440                // move the breaker.
7441                breaker.record_inconclusive();
7442                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
7443            }
7444            Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
7445                reservation.release_and_disarm();
7446                breaker.record_inconclusive();
7447                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
7448            }
7449            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
7450                reservation.release_and_disarm();
7451                breaker.record_inconclusive();
7452                // Fires when the module's connection closes while a relayed
7453                // bind is pending -- typically a caller racing a module restart
7454                // whose bind was relayed BEFORE the drain mark went up. Logged
7455                // because the caller sees only its own error and the fleet has
7456                // already spent one diagnosis round unable to tell this arm
7457                // from a relay timeout without daemon-side evidence.
7458                tracing::warn!(
7459                    module_id = %target_module_id,
7460                    "route.bind relay abandoned: {message}"
7461                );
7462                Ok(vec![self.route_open_refusal_frame(
7463                    ctx,
7464                    &caller,
7465                    &frame,
7466                    &target_module_id,
7467                    "relay_abandoned",
7468                    "target_unavailable",
7469                    message,
7470                )?])
7471            }
7472            Ok(Err(_)) => {
7473                reservation.release_and_disarm();
7474                breaker.record_inconclusive();
7475                Ok(vec![self.route_open_refusal_frame(
7476                    ctx,
7477                    &caller,
7478                    &frame,
7479                    &target_module_id,
7480                    "relay_waiter_canceled",
7481                    "target_unavailable",
7482                    "route.bind relay waiter was canceled before the module responded",
7483                )?])
7484            }
7485            Err(_) => {
7486                reservation.release_and_disarm();
7487                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
7488                // answer at all is the one condition a fast refusal can
7489                // usefully stand in for; every other arm already answered.
7490                if let Some(opened) = breaker.record_timeout(
7491                    self.route_bind_breaker_threshold,
7492                    self.route_bind_breaker_cooldown,
7493                ) {
7494                    warn!(
7495                        module_id = %target_module_id,
7496                        consecutive_timeouts = opened.consecutive_timeouts,
7497                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
7498                        reopened_after_probe = opened.reopened_after_probe,
7499                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
7500                    );
7501                }
7502                // The generous budget just burned to no answer: the module is
7503                // registered and its connection is up, but its bind handler sat
7504                // on the ack for the full budget (warm-on-bind, cold configure,
7505                // or a wedged handler). Every earlier unavailability shape
7506                // fast-refuses BEFORE the relay, so this arm firing means the
7507                // slowness is module-side -- log it so the per-module timeline
7508                // is reconstructable without client audit rows.
7509                tracing::warn!(
7510                    module_id = %target_module_id,
7511                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
7512                    "route.bind relay timed out: module did not ack within budget"
7513                );
7514                Ok(vec![self.route_open_refusal_frame(
7515                    ctx,
7516                    &caller,
7517                    &frame,
7518                    &target_module_id,
7519                    "relay_timed_out",
7520                    "module_timeout",
7521                    format!(
7522                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
7523                        route_bind_relay_timeout
7524                    ),
7525                )?])
7526            }
7527        }
7528    }
7529
7530    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7531        let response = ClientControlResponse::SupervisorSpawnSnapshot {
7532            snapshot: self.supervisor.spawn_snapshot(),
7533        };
7534        Ok(vec![control_response_body_frame(
7535            &frame,
7536            &response,
7537            "ClientControlResponse::SupervisorSpawnSnapshot",
7538        )?])
7539    }
7540
7541    fn handle_supervisor_spawn_subscribe(
7542        &self,
7543        ctx: &RouteCtx,
7544        frame: Frame,
7545        since: Option<SpawnCursor>,
7546    ) -> Result<Vec<Frame>, RouterError> {
7547        match self.supervisor.subscribe_spawns(
7548            ctx.connection_id,
7549            frame.header.corr,
7550            response_version(&frame),
7551            since,
7552            ctx.egress.clone(),
7553        ) {
7554            Ok(()) => Ok(Vec::new()),
7555            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
7556                Ok(vec![control_error_body_frame(
7557                    &frame,
7558                    ErrorBody {
7559                        code: "spawn_cursor_incarnation_mismatch".to_string(),
7560                        message: "spawn cursor belongs to a different daemon incarnation"
7561                            .to_string(),
7562                        detail: Some(serde_json::json!({
7563                            "current_daemon_incarnation": current
7564                        })),
7565                    },
7566                )?])
7567            }
7568            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
7569                &frame,
7570                ErrorBody {
7571                    code: "spawn_cursor_too_old".to_string(),
7572                    message: "spawn cursor predates the retained event ring".to_string(),
7573                    detail: Some(serde_json::json!({
7574                        "oldest_retained_cursor": oldest
7575                    })),
7576                },
7577            )?]),
7578            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
7579                0,
7580                frame.header.corr,
7581                format!("failed to open supervisor spawn subscription: {error}"),
7582            )),
7583        }
7584    }
7585
7586    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7587        // Status/configuration snapshots may wait on synchronous supervisor
7588        // locks. Keep that work off the connection's async runtime thread.
7589        let handler = self.clone();
7590        let runtime = tokio::runtime::Handle::current();
7591        // The blocking pool does not inherit the control request's tracing
7592        // subscriber. Preserve it so slow snapshot-lock warnings stay visible.
7593        let subscriber = tracing::dispatcher::get_default(Clone::clone);
7594        let corr = frame.header.corr;
7595        tokio::task::spawn_blocking(move || {
7596            runtime.block_on(
7597                list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
7598                    .with_subscriber(subscriber),
7599            )
7600        })
7601        .await
7602        .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
7603    }
7604
7605    async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7606        use crate::config_gate::runtime::ApprovedDaemonView;
7607        let publication = self
7608            .config_runtime
7609            .as_ref()
7610            .and_then(|owner| owner.polled_snapshot().ok());
7611        let approved = publication
7612            .as_ref()
7613            .map(|view| view.approved_daemon.clone())
7614            .unwrap_or_else(|| {
7615                if self.config_runtime.is_some() {
7616                    ApprovedDaemonView::Unavailable
7617                } else {
7618                    ApprovedDaemonView::NotStamped
7619                }
7620            });
7621        let generation = self
7622            .registry
7623            .generation()
7624            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7625        let supervised = self.supervisor.list();
7626        let mut statuses = BTreeMap::new();
7627        let mut not_running = BTreeSet::new();
7628        for module in &supervised {
7629            let status = module.status_for_control("list").map_err(|err| {
7630                RouterError::backend(
7631                    0,
7632                    frame.header.corr,
7633                    format!("failed to read supervisor status: {err}"),
7634                )
7635            })?;
7636            if !status.enabled || !status.process_alive {
7637                not_running.insert(module.module_id().to_owned());
7638            }
7639            statuses.insert(module.module_id().to_owned(), status);
7640        }
7641        // Legacy per-process diagnostics keep their shared optional budget.
7642        // The cached tree and accounting export below does not spend it.
7643        let mut process_resources = BTreeMap::new();
7644        for module in &supervised {
7645            process_resources.insert(
7646                module.module_id().to_owned(),
7647                crate::subtree_resources::read_process(module.clone()).await,
7648            );
7649        }
7650        // Image evidence uses the same optional deadline, but an exhausted
7651        // budget must not suppress the committed resource view.
7652        let mut image_readings = BTreeMap::new();
7653        for module in &supervised {
7654            image_readings.insert(
7655                module.module_id().to_owned(),
7656                module.running_image_agreement().await,
7657            );
7658        }
7659        let readings = self.resource_tracker.list(supervised.clone());
7660        let mut modules = Vec::new();
7661        for module in supervised {
7662            let status = statuses
7663                .remove(module.module_id())
7664                .expect("each supervised module has a status snapshot");
7665            let (configured, _) = module.configuration().map_err(|err| {
7666                RouterError::backend(
7667                    0,
7668                    frame.header.corr,
7669                    format!("failed to read module configuration: {err}"),
7670                )
7671            })?;
7672            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
7673                RouterError::backend(
7674                    0,
7675                    frame.header.corr,
7676                    format!("failed to read retry time: {err}"),
7677                )
7678            })?;
7679            let image = image_readings
7680                .remove(module.module_id())
7681                .expect("each supervised module has an image snapshot");
7682            let resources = process_resources.remove(module.module_id());
7683            let mut verdict =
7684                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
7685            verdict.spec = Some(launch_spec_agreement(
7686                &configured,
7687                status.process_alive,
7688                status.launch_spec_digest.as_ref(),
7689            ));
7690            let pending_reload = Some(verdict);
7691            let configured_enabled = match &approved {
7692                ApprovedDaemonView::Approved {
7693                    config: approved, ..
7694                } => approved
7695                    .modules
7696                    .iter()
7697                    .find(|module| module.module_id == status.module_id)
7698                    .map(|module| module.enabled),
7699                ApprovedDaemonView::NotStamped => self
7700                    .capability_evaluator
7701                    .configured_enabled(&status.module_id),
7702                ApprovedDaemonView::Unavailable => None,
7703            };
7704            let admission_refused = status
7705                .admission_refused
7706                .clone()
7707                .filter(|_| status.enabled && !status.process_alive)
7708                .filter(|_| {
7709                    publication
7710                        .as_ref()
7711                        .and_then(|view| view.registration_declaration(&status.module_id))
7712                        .as_deref()
7713                        == Some("undeclared")
7714                });
7715            // Keep the retired policy field on the wire for one release so
7716            // existing status consumers still receive the platform policy.
7717            modules.push(
7718                SupervisorEntry::new(
7719                    status.module_id,
7720                    status.state.to_string(),
7721                    status.enabled,
7722                    status.live,
7723                    status.health.status,
7724                )
7725                .with_configured_enabled(configured_enabled)
7726                .with_admission_refused(admission_refused)
7727                .with_next_retry_at_ms(next_retry_at_ms)
7728                .with_launch_nonce_env(Some(!cfg!(unix)))
7729                .with_launch_nonce_source(status.launch_nonce_source)
7730                .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
7731                .with_protocol(status.protocol)
7732                .with_pending_reload(pending_reload)
7733                .with_last_probe_ms(status.health.last_probe_ms)
7734                .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
7735                .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
7736                .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
7737                .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
7738                .with_restart_count(Some(status.restart_count))
7739                .with_max_restarts(Some(status.max_restarts))
7740                .with_lifetime_restarts(Some(status.lifetime_restarts))
7741                .with_spawn_generation(Some(status.spawn_generation))
7742                .with_restart_window_secs(Some(status.restart_window.as_secs()))
7743                .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
7744                .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
7745                .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
7746                .with_resources(resources),
7747            );
7748        }
7749        for module in &mut modules {
7750            if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
7751                module.tree = tree.clone();
7752                module.cpu_last_hour = hour.clone();
7753            }
7754            module.tree_unavailable = readings
7755                .reasons
7756                .get(&module.module_id)
7757                .map(|reason| (*reason).to_owned());
7758            if module.tree.is_none() {
7759                module.tree_unavailable = Some(
7760                    if not_running.contains(&module.module_id) {
7761                        "not_running"
7762                    } else if let Some(reason) = readings.reasons.get(&module.module_id) {
7763                        reason
7764                    } else {
7765                        "not_yet_measured"
7766                    }
7767                    .to_owned(),
7768                );
7769            }
7770        }
7771        let response = ClientControlResponse::SupervisorList {
7772            generation,
7773            modules,
7774            daemon: readings.daemon,
7775            agent_commands: readings.agent_commands,
7776        };
7777        Ok(vec![control_response_body_frame(
7778            &frame,
7779            &response,
7780            "ClientControlResponse::SupervisorList",
7781        )?])
7782    }
7783
7784    fn handle_supervisor_stderr_tail(
7785        &self,
7786        frame: Frame,
7787        module_id: String,
7788        max_lines: Option<u32>,
7789        max_bytes: Option<u32>,
7790    ) -> Result<Vec<Frame>, RouterError> {
7791        let Some(module) = self.supervisor.get(&module_id) else {
7792            return Ok(vec![control_error_frame(
7793                &frame,
7794                "unknown_module",
7795                format!("module_id '{module_id}' is not supervised"),
7796            )?]);
7797        };
7798
7799        let snapshot = module.stderr_tail(
7800            max_lines.map(|value| value as usize),
7801            max_bytes.map(|value| value as usize),
7802        );
7803
7804        let response = ClientControlResponse::SupervisorStderrTail {
7805            module_id,
7806            tail: StderrTail {
7807                capture: match snapshot.capture {
7808                    CaptureState::Captured => StderrCaptureState::Captured,
7809                    CaptureState::Incomplete { reason } => {
7810                        StderrCaptureState::Incomplete { reason }
7811                    }
7812                    CaptureState::NotCaptured { reason } => {
7813                        StderrCaptureState::NotCaptured { reason }
7814                    }
7815                },
7816                entries: snapshot
7817                    .entries
7818                    .into_iter()
7819                    .map(|entry| match entry {
7820                        TailEntry::Line {
7821                            text,
7822                            truncated,
7823                            at_ms,
7824                        } => StderrTailEntry::Line {
7825                            text,
7826                            truncated,
7827                            at_ms,
7828                        },
7829                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
7830                    })
7831                    .collect(),
7832                dropped_lines: snapshot.dropped_lines,
7833            },
7834        };
7835        Ok(vec![control_response_body_frame(
7836            &frame,
7837            &response,
7838            "ClientControlResponse::SupervisorStderrTail",
7839        )?])
7840    }
7841
7842    async fn observe_control_wait<F: std::future::Future>(
7843        &self,
7844        ctx: &RouteCtx,
7845        target: Option<&str>,
7846        kind: &'static str,
7847        future: F,
7848    ) -> F::Output {
7849        self.counters
7850            .observe_reader_wait(
7851                || {
7852                    let module = self
7853                        .forwarding
7854                        .module_id_for_connection(ctx.connection_id)
7855                        .ok()
7856                        .flatten();
7857                    let principal = if module.is_some() { "module" } else { "direct" };
7858                    (
7859                        self.resolve_reader_wait_target(kind, target)
7860                            .or(module)
7861                            .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
7862                        principal.into(),
7863                    )
7864                },
7865                kind,
7866                future,
7867            )
7868            .await
7869    }
7870
7871    /// Resolve the reader-wait module ID only from supervisor or registry
7872    /// records; never attribute a wait from the request selector alone.
7873    pub(crate) fn resolve_reader_wait_target(
7874        &self,
7875        op: &str,
7876        selector: Option<&str>,
7877    ) -> Option<String> {
7878        let selector = selector?;
7879        match op {
7880            ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
7881                .supervisor
7882                .get(selector)
7883                .map(|module| module.module_id().to_owned()),
7884            ops::SUPERVISOR_HEALTH_PROBE => self
7885                .registry
7886                .get_module(selector)
7887                .ok()
7888                .flatten()
7889                .map(|registration| registration.manifest.module_id),
7890            _ => None,
7891        }
7892    }
7893
7894    async fn handle_supervisor_terminals(
7895        &self,
7896        ctx: &RouteCtx,
7897        frame: Frame,
7898        module_id: String,
7899    ) -> Result<Vec<Frame>, RouterError> {
7900        let Some(module) = self.supervisor.get(&module_id) else {
7901            return Ok(vec![control_error_frame(
7902                &frame,
7903                "unknown_module",
7904                format!("module_id '{module_id}' is not supervised"),
7905            )?]);
7906        };
7907
7908        // The journal read runs on a blocking thread: it can be megabytes of
7909        // file I/O and must not occupy a runtime worker.
7910        let terminals = self
7911            .observe_control_wait(
7912                ctx,
7913                Some(module.module_id()),
7914                ops::SUPERVISOR_TERMINALS,
7915                module.read_durable_terminal_history(),
7916            )
7917            .await
7918            .map_err(|error| {
7919                RouterError::backend(
7920                    0,
7921                    frame.header.corr,
7922                    format!("failed to read terminal history: {error}"),
7923                )
7924            })?;
7925        let response = ClientControlResponse::SupervisorTerminals {
7926            module_id,
7927            terminals,
7928        };
7929        Ok(vec![control_response_body_frame(
7930            &frame,
7931            &response,
7932            "ClientControlResponse::SupervisorTerminals",
7933        )?])
7934    }
7935
7936    fn handle_supervisor_routes(
7937        &self,
7938        frame: Frame,
7939        module_id: Option<String>,
7940    ) -> Result<Vec<Frame>, RouterError> {
7941        let modules = self
7942            .forwarding
7943            .route_census(module_id.as_deref())
7944            .map_err(RouterError::Forwarding)?
7945            .into_iter()
7946            .map(|(module_id, routes)| SupervisorRouteModule {
7947                module_id,
7948                routes: routes
7949                    .into_iter()
7950                    .map(|route| SupervisorRoute {
7951                        consumer: match route.principal {
7952                            Principal::Reserved { module_id } => {
7953                                SupervisorRouteConsumer::Reserved { module_id }
7954                            }
7955                            Principal::Direct | Principal::Unverified => {
7956                                SupervisorRouteConsumer::Direct {
7957                                    connection_id: route.goodbye_target.connection_id.get(),
7958                                }
7959                            }
7960                        },
7961                        age_ms: Instant::now()
7962                            .saturating_duration_since(route.bound_at)
7963                            .as_millis()
7964                            .try_into()
7965                            .unwrap_or(u64::MAX),
7966                        draining: route.draining,
7967                        drain_reason: route.drain_reason,
7968                    })
7969                    .collect(),
7970            })
7971            .collect();
7972        let response = ClientControlResponse::SupervisorRoutes { modules };
7973        Ok(vec![control_response_body_frame(
7974            &frame,
7975            &response,
7976            "ClientControlResponse::SupervisorRoutes",
7977        )?])
7978    }
7979
7980    fn running_registration_method(
7981        &self,
7982        module: &str,
7983    ) -> Option<subc_control::RegistrationMethod> {
7984        let registration = self.registry.get_module(module).ok()??;
7985        let connection = registration.connection_id;
7986        let channel = self.supervisor.registration_spawn(connection).is_some();
7987        let nonce = if channel {
7988            None
7989        } else {
7990            self.hello_launch_nonces
7991                .lock()
7992                .ok()?
7993                .nonce(connection)
7994                .map(str::to_owned)
7995        };
7996        if !channel && nonce.is_none() {
7997            return None;
7998        }
7999        self.supervisor
8000            .registration_snapshot_is_current_live(
8001                &self.registry,
8002                module,
8003                connection,
8004                nonce.as_deref(),
8005            )
8006            .then_some(if channel {
8007                subc_control::RegistrationMethod::Channel
8008            } else {
8009                subc_control::RegistrationMethod::Nonce
8010            })
8011    }
8012
8013    async fn handle_supervisor_provenance(
8014        &self,
8015        ctx: &RouteCtx,
8016        frame: Frame,
8017        module_id: Option<String>,
8018    ) -> Result<Vec<Frame>, RouterError> {
8019        let targeted = module_id.is_some();
8020        let selected = if let Some(module_id) = module_id {
8021            let Some(module) = self.supervisor.get(&module_id) else {
8022                return Ok(vec![control_error_frame(
8023                    &frame,
8024                    "unknown_module",
8025                    format!("module_id '{module_id}' is not supervised"),
8026                )?]);
8027            };
8028            vec![module]
8029        } else {
8030            self.supervisor.list()
8031        };
8032
8033        let target = targeted.then(|| selected[0].module_id());
8034        let declarations = if let Some(owner) = &self.config_runtime {
8035            owner
8036                .on_blocking(|owner| owner.registration_declarations())
8037                .await
8038                .ok()
8039        } else {
8040            None
8041        };
8042        self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
8043            let mut modules = Vec::with_capacity(selected.len());
8044            let mut extensions = Vec::with_capacity(selected.len());
8045            for module in &selected {
8046                let (status, observed_image) = module
8047                    .status_and_running_image_agreement()
8048                    .await
8049                    .map_err(|err| {
8050                        RouterError::backend(
8051                            0,
8052                            frame.header.corr,
8053                            format!("failed to read supervisor status: {err}"),
8054                        )
8055                    })?;
8056                let module_declared = self
8057                    .registry
8058                    .get_module(&status.module_id)
8059                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
8060                    .and_then(|registration| registration.manifest.provenance)
8061                    .map(|build| ModuleDeclaredProvenance::Reported { build })
8062                    .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
8063                #[cfg(test)]
8064                let running_image = match &self.provenance_probe_override {
8065                    Some(result) => result.clone(),
8066                    None => observed_image,
8067                };
8068                #[cfg(not(test))]
8069                let running_image = observed_image;
8070                let registration = self.running_registration_method(&status.module_id);
8071                let mut extension = subc_control::SupervisorModuleProvenanceExt::new()
8072                    .with_registration(registration);
8073                extension.registration_declaration = declarations
8074                    .as_ref()
8075                    .and_then(|entries| entries.get(&status.module_id))
8076                    .cloned();
8077                if status.process_alive && status.registration_declaration.is_some() {
8078                    extension.registration_declaration = status.registration_declaration.clone();
8079                }
8080                extensions.push(extension);
8081                modules.push(SupervisorModuleProvenance {
8082                    module_id: status.module_id,
8083                    module_declared,
8084                    daemon_observed: SupervisorObservedProcess {
8085                        launch_nonce_source: status.launch_nonce_source,
8086                        pid: status.pid,
8087                        spawned_at_ms: status.spawned_at_ms,
8088                        spawned_from: status.spawned_from,
8089                        running_image,
8090                    },
8091                });
8092            }
8093            let daemon = SupervisorDaemonProvenance {
8094                daemon_build: self.daemon_provenance.build.clone(),
8095                daemon_observed: DaemonObservedProcess {
8096                    pid: self.daemon_provenance.pid,
8097                    started_at_ms: self
8098                        .daemon_provenance
8099                        .start_clock
8100                        .map(|clock| clock.started_at_ms())
8101                        .or(self.daemon_provenance.started_at_ms),
8102                    running_image: self
8103                        .daemon_provenance
8104                        .probe
8105                        .observe(
8106                            self.daemon_provenance.pid,
8107                            self.daemon_provenance.executable_path.as_deref(),
8108                            self.daemon_provenance.executable_identity.clone(),
8109                            self.daemon_provenance.process_start_time,
8110                        )
8111                        .await,
8112                },
8113            };
8114            let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
8115            let mut response = serde_json::to_value(response)
8116                .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
8117            for (entry, extension) in response["modules"]
8118                .as_array_mut()
8119                .expect("provenance modules are an array")
8120                .iter_mut()
8121                .zip(extensions)
8122            {
8123                extension.write_to(entry).map_err(|error| {
8124                    RouterError::backend(0, frame.header.corr, error.to_string())
8125                })?;
8126            }
8127            Ok(vec![control_response_body_frame(
8128                &frame,
8129                &response,
8130                "ClientControlResponse::SupervisorProvenance",
8131            )?])
8132        })
8133        .await
8134    }
8135
8136    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
8137        self.refresh_capability_requirements();
8138        let publication = self
8139            .config_runtime
8140            .as_ref()
8141            .and_then(|owner| owner.polled_snapshot().ok());
8142        let generation = self
8143            .registry
8144            .generation()
8145            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
8146        let modules = self
8147            .supervisor
8148            .list()
8149            .into_iter()
8150            .map(|module| {
8151                let status = module.status_for_control("health").map_err(|err| {
8152                    RouterError::backend(
8153                        0,
8154                        frame.header.corr,
8155                        format!("failed to read supervisor health: {err}"),
8156                    )
8157                })?;
8158                let module_id = status.module_id;
8159                let capability_detail = self
8160                    .capability_evaluator
8161                    .required_problem_detail(&module_id);
8162                let declaration_warning = self.registry.get_module(&module_id).ok().flatten()
8163                    .filter(|registration| registration.manifest.registration_channel == Some(false))
8164                    .filter(|_| publication.as_ref().and_then(|view| view.registration_declaration(&module_id))
8165                        .as_deref().map(|next| if status.process_alive { status.registration_declaration.as_deref().unwrap_or(next) } else { next }) == Some("declared"))
8166                    .map(|_| format!("{module_id}: HELLO registration_channel is false despite the approved channel declaration"));
8167                let capability_detail = append_capability_problem_detail(capability_detail, declaration_warning);
8168                Ok(SupervisorHealthEntry {
8169                    module_id,
8170                    status: status.health.status,
8171                    detail: append_capability_problem_detail(
8172                        status.health.detail,
8173                        capability_detail,
8174                    ),
8175                    metrics: status.health.metrics,
8176                    consecutive_failures: status.health.consecutive_failures,
8177                    late_answer_count: status.health.late_answer_count,
8178                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
8179                    last_action: status.health.last_action,
8180                    last_action_ms: status.health.last_action_ms,
8181                    last_probe_ms: status.health.last_probe_ms,
8182                })
8183            })
8184            .collect::<Result<Vec<_>, RouterError>>()?;
8185        let response = ClientControlResponse::SupervisorHealth {
8186            generation,
8187            modules,
8188        };
8189        Ok(vec![control_response_body_frame(
8190            &frame,
8191            &response,
8192            "ClientControlResponse::SupervisorHealth",
8193        )?])
8194    }
8195
8196    async fn handle_supervisor_restart(
8197        &self,
8198        frame: Frame,
8199        module_id: String,
8200        drain_timeout_ms: Option<u64>,
8201        requester: Principal,
8202    ) -> Result<Vec<Frame>, RouterError> {
8203        let operation_lock = self.supervisor.operation_lock();
8204        let _operation_guard = operation_lock.lock().await;
8205        let Some(module) = self.supervisor.get(&module_id) else {
8206            return Ok(vec![control_error_frame(
8207                &frame,
8208                "unknown_module",
8209                format!("module_id '{module_id}' is not supervised"),
8210            )?]);
8211        };
8212
8213        self.route_outages.mark_operator_action(&module_id);
8214        if let Err(err) = module
8215            .restart_requested(drain_timeout_ms, Some(requester))
8216            .await
8217        {
8218            self.route_outages
8219                .operator_action_ended_unrefused(&module_id);
8220            let (code, message) = match err {
8221                _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8222                crate::supervise::SuperviseError::Disabled { .. } => {
8223                    ("module_disabled", err.to_string())
8224                }
8225                crate::supervise::SuperviseError::SwapInProgress { .. } => {
8226                    ("swap_in_progress", err.to_string())
8227                }
8228                _ => (
8229                    "target_unavailable",
8230                    format!("failed to restart module_id '{module_id}': {err}"),
8231                ),
8232            };
8233            return Ok(vec![control_error_frame(&frame, code, message)?]);
8234        }
8235
8236        let response = ClientControlResponse::SupervisorAck {
8237            module_id,
8238            applied: true,
8239        };
8240        Ok(vec![control_response_body_frame(
8241            &frame,
8242            &response,
8243            "ClientControlResponse::SupervisorAck",
8244        )?])
8245    }
8246
8247    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
8248    /// when the old process has finished draining: a caller whose own lane
8249    /// rides the old process must get its reply before that drain waits on it.
8250    async fn handle_supervisor_swap(
8251        &self,
8252        frame: Frame,
8253        module_id: String,
8254        ready_timeout_ms: Option<u64>,
8255        requester: Principal,
8256    ) -> Result<Vec<Frame>, RouterError> {
8257        // The daemon-wide operation lock is held only to resolve the handle,
8258        // not across the swap. The swap can take its whole readiness budget,
8259        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
8260        // holding it here would park an operator's stop behind the swap it is
8261        // meant to abort. A rescan or stop that reaches the module during the
8262        // swap is served by the swap itself (see `supervise_swap`).
8263        let module = {
8264            let operation_lock = self.supervisor.operation_lock();
8265            let _operation_guard = operation_lock.lock().await;
8266            self.supervisor.get(&module_id)
8267        };
8268        let Some(module) = module else {
8269            return Ok(vec![control_error_frame(
8270                &frame,
8271                "unknown_module",
8272                format!("module_id '{module_id}' is not supervised"),
8273            )?]);
8274        };
8275
8276        self.route_outages.mark_operator_action(&module_id);
8277        if let Err(err) = module
8278            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
8279            .await
8280        {
8281            self.route_outages
8282                .operator_action_ended_unrefused(&module_id);
8283            use crate::supervise::SuperviseError;
8284            let message = err.to_string();
8285            let error = match err {
8286                SuperviseError::ConfigHeld { .. } => ErrorBody::new("config_gated", message),
8287                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
8288                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
8289                    code: "swap_refused".to_string(),
8290                    message,
8291                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
8292                },
8293                SuperviseError::SwapFailed {
8294                    arm,
8295                    candidate_exit,
8296                    ..
8297                } => ErrorBody {
8298                    code: "swap_failed".to_string(),
8299                    message,
8300                    detail: Some(serde_json::json!({
8301                        "arm": arm.as_str(),
8302                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
8303                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
8304                    })),
8305                },
8306                _ => ErrorBody::new(
8307                    "target_unavailable",
8308                    format!("failed to swap module_id '{module_id}': {message}"),
8309                ),
8310            };
8311            return Ok(vec![control_error_body_frame(&frame, error)?]);
8312        }
8313        // A completed swap kept the incumbent serving until cutover, so it
8314        // usually opened no outage; a mark left behind would make the next,
8315        // unrelated outage read as requested.
8316        self.route_outages
8317            .operator_action_ended_unrefused(&module_id);
8318
8319        let response = ClientControlResponse::SupervisorAck {
8320            module_id,
8321            applied: true,
8322        };
8323        Ok(vec![control_response_body_frame(
8324            &frame,
8325            &response,
8326            "ClientControlResponse::SupervisorAck",
8327        )?])
8328    }
8329
8330    async fn handle_supervisor_reload(
8331        &self,
8332        frame: Frame,
8333        module_id: String,
8334        requester: Principal,
8335    ) -> Result<Vec<Frame>, RouterError> {
8336        let operation_lock = self.supervisor.operation_lock();
8337        let _operation_guard = operation_lock.lock().await;
8338        let Some(module) = self.supervisor.get(&module_id) else {
8339            return Ok(vec![control_error_frame(
8340                &frame,
8341                "unknown_module",
8342                format!("module_id '{module_id}' is not supervised"),
8343            )?]);
8344        };
8345
8346        self.route_outages.mark_operator_action(&module_id);
8347        if let Err(err) = module.reload_requested(Some(requester)).await {
8348            self.route_outages
8349                .operator_action_ended_unrefused(&module_id);
8350            let (code, message) = match err {
8351                _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8352                crate::supervise::SuperviseError::Disabled { .. } => {
8353                    ("module_disabled", err.to_string())
8354                }
8355                crate::supervise::SuperviseError::SwapInProgress { .. } => {
8356                    ("swap_in_progress", err.to_string())
8357                }
8358                _ => (
8359                    "reload_failed",
8360                    format!("failed to reload module_id '{module_id}': {err}"),
8361                ),
8362            };
8363            return Ok(vec![control_error_frame(&frame, code, message)?]);
8364        }
8365
8366        let response = ClientControlResponse::SupervisorAck {
8367            module_id,
8368            applied: true,
8369        };
8370        Ok(vec![control_response_body_frame(
8371            &frame,
8372            &response,
8373            "ClientControlResponse::SupervisorAck",
8374        )?])
8375    }
8376
8377    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
8378    /// the configuration comes from and whether to apply it; every mode then
8379    /// runs the same comparison, so a preview describes the apply it previews.
8380    async fn handle_supervisor_rescan(
8381        &self,
8382        frame: Frame,
8383        mode: RescanMode,
8384        requester: Principal,
8385    ) -> Result<Vec<Frame>, RouterError> {
8386        let preview = mode.previews();
8387        let mut approved_preview = None;
8388        if matches!(mode, RescanMode::Preview) {
8389            if let Some(owner) = &self.config_runtime {
8390                match owner
8391                    .on_blocking(|owner| {
8392                        owner.observe_file_proposal()?;
8393                        owner.approved_daemon_configuration()
8394                    })
8395                    .await
8396                {
8397                    Ok(config) => approved_preview = config,
8398                    Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8399                }
8400            }
8401        }
8402        let Some(context) = self.rescan.clone() else {
8403            return Ok(vec![control_error_frame(
8404                &frame,
8405                "rescan_unavailable",
8406                "the daemon was not started with a reloadable config path".to_string(),
8407            )?]);
8408        };
8409
8410        let operation_lock = self.supervisor.operation_lock();
8411        let _operation_guard = operation_lock.lock().await;
8412        // Loaded under the operation lock, so no other rescan can change the
8413        // staged configuration between a checked apply's comparison and the
8414        // apply itself.
8415        let loaded = match self.load_rescan_config(&mode, &context) {
8416            Ok(config) => config,
8417            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
8418        };
8419        // `load` reports a missing file as Ok(None), which is correct at boot
8420        // (no config, nothing to supervise) and catastrophic here: rescan treats
8421        // "not in the config" as "remove it", so an absent file would read as an
8422        // empty module list and retire the entire running fleet. An editor
8423        // writing via write-new-then-rename, or a half-finished edit, is enough
8424        // to open that window. Refuse instead: a config that cannot be read
8425        // carries no instruction to remove anything.
8426        let Some((config, activity_config)) = loaded else {
8427            return Ok(vec![control_error_frame(
8428                &frame,
8429                "invalid_daemon_config",
8430                format!(
8431                    "daemon config not found at {}; refusing to rescan (an absent config would \
8432                     retire every supervised module)",
8433                    context.config_path.display()
8434                ),
8435            )?]);
8436        };
8437        let (
8438            configured_port,
8439            storage_config,
8440            admission_facts_carrier_module_id,
8441            admission_facts_targets,
8442            scope_authority_owners,
8443            federated_scope_owner,
8444            privileged_operator_prompt,
8445            modules,
8446            reserved_capabilities,
8447        ) = (
8448            config.port,
8449            config.storage,
8450            config.admission_facts_carrier_module_id,
8451            config.admission_facts_targets,
8452            config.scope_authority_owners,
8453            config.federated_scope_owner,
8454            config.privileged_operator_prompt,
8455            config.modules,
8456            config.reserved_capabilities,
8457        );
8458
8459        // Collect the sections rescan cannot apply, so the REPLY carries them.
8460        //
8461        // The warning below has always been correct and has always gone only to
8462        // the journal -- addressed to whoever reads logs, while the person who
8463        // just edited the config is looking at the CLI. Naming each section
8464        // individually rather than setting a flag: "something outside modules
8465        // changed" sends the operator back to diffing their own file, which is
8466        // the work this is meant to save.
8467        let mut restart_required = Vec::new();
8468        for section in RestartRequiredSection::ALL {
8469            let changed = match section {
8470                RestartRequiredSection::Port => configured_port != context.configured_port,
8471                RestartRequiredSection::Storage => storage_config != context.storage_config,
8472                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
8473                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
8474                }
8475                RestartRequiredSection::AdmissionFactsTargets => {
8476                    admission_facts_targets != context.admission_facts_targets
8477                }
8478                RestartRequiredSection::ScopeAuthorityOwners => {
8479                    scope_authority_owners != context.scope_authority_owners
8480                }
8481                RestartRequiredSection::FederatedScopeOwner => {
8482                    federated_scope_owner != context.federated_scope_owner
8483                }
8484                RestartRequiredSection::PrivilegedOperatorPrompt => {
8485                    privileged_operator_prompt != context.privileged_operator_prompt
8486                }
8487            };
8488            if changed {
8489                restart_required.push(section.label().to_string());
8490            }
8491        }
8492        if !restart_required.is_empty() {
8493            warn!(
8494                config_path = %context.config_path.display(),
8495                sections = %restart_required.join(", "),
8496                "daemon config changed outside the modules section; restart the daemon to apply those changes"
8497            );
8498        }
8499
8500        for configured in &modules {
8501            if let Err(err) = validate_spec(&configured.module_spec()) {
8502                return Ok(vec![control_error_frame(
8503                    &frame,
8504                    "invalid_daemon_config",
8505                    format!("supervisor rescan rejected daemon config: {err}"),
8506                )?]);
8507            }
8508        }
8509
8510        let configured_capabilities = modules
8511            .iter()
8512            .map(|module| (module.module_id.clone(), module.enabled))
8513            .collect::<Vec<_>>();
8514        let preview_capability_warnings = if preview {
8515            let (_, registrations) = self.runtime_capability_snapshot()?;
8516            let current_modules = self
8517                .supervisor
8518                .list()
8519                .into_iter()
8520                .map(|module| module.module_id().to_string())
8521                .collect::<BTreeSet<_>>();
8522            let resulting_modules = configured_capabilities.clone();
8523            let removed = current_modules
8524                .into_iter()
8525                .filter(|module_id| {
8526                    !resulting_modules
8527                        .iter()
8528                        .any(|(configured_id, _)| configured_id == module_id)
8529                })
8530                .collect::<Vec<_>>();
8531            // A candidate preview also names providers the candidate disables.
8532            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
8533            let disabled = match &mode {
8534                RescanMode::Candidate(_) => resulting_modules
8535                    .iter()
8536                    .filter(|(_, enabled)| !enabled)
8537                    .map(|(module_id, _)| module_id.clone())
8538                    .collect::<Vec<_>>(),
8539                _ => Vec::new(),
8540            };
8541            self.capability_evaluator.preview_warnings(
8542                resulting_modules,
8543                &removed,
8544                &disabled,
8545                &registrations,
8546            )
8547        } else {
8548            Vec::new()
8549        };
8550        let reconciled = match (&mode, approved_preview.as_ref()) {
8551            (RescanMode::Preview, Some(approved)) => Ok((
8552                config_gate::preview_approved_daemon(approved, &modules),
8553                Vec::new(),
8554            )),
8555            (RescanMode::Candidate(candidate), _) => self
8556                .preview_candidate(&context, modules, candidate)
8557                .map(|result| (result, Vec::new())),
8558            _ => {
8559                self.reconcile_supervised_modules(
8560                    &context.supervisor,
8561                    modules,
8562                    preview,
8563                    requester.clone(),
8564                )
8565                .await
8566            }
8567        };
8568        let (result, changed) = match reconciled {
8569            Ok(result) => result,
8570            Err(message) => {
8571                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
8572            }
8573        };
8574        if !preview {
8575            if let Some(ledger) = &self.activity {
8576                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
8577                ledger.config_changed(&result.added, &result.removed, &changed, requester);
8578            }
8579            if let Err(code) = self.configure_activity(activity_config).await {
8580                warn!(
8581                    code,
8582                    "activity retention could not be applied during rescan"
8583                );
8584            }
8585            self.capability_evaluator
8586                .configure(configured_capabilities, reserved_capabilities);
8587            self.capability_evaluator.wake_deadline_loop();
8588            self.refresh_capability_requirements();
8589        }
8590        let mut result = result;
8591        result.restart_required = restart_required;
8592        result.capability_warnings = preview_capability_warnings;
8593        let response = ClientControlResponse::SupervisorRescan { result };
8594        Ok(vec![control_response_body_frame(
8595            &frame,
8596            &response,
8597            "ClientControlResponse::SupervisorRescan",
8598        )?])
8599    }
8600
8601    async fn handle_supervisor_release_reserved(
8602        &self,
8603        frame: Frame,
8604        module_id: String,
8605    ) -> Result<Vec<Frame>, RouterError> {
8606        let Some(context) = self.rescan.clone() else {
8607            return Ok(vec![control_error_frame(
8608                &frame,
8609                "release_unavailable",
8610                "reserved-id release requires a daemon started with a reloadable config path",
8611            )?]);
8612        };
8613        let operation_lock = self.supervisor.operation_lock();
8614        let _operation_guard = operation_lock.lock().await;
8615        let approved = if let Some(owner) = &self.config_runtime {
8616            // Observe the file as a proposal, but decide membership solely from
8617            // approved bytes. A proposed re-add must not retain a removed id.
8618            match owner
8619                .on_blocking(|owner| {
8620                    owner.observe_file_proposal()?;
8621                    owner.approved_daemon_configuration()
8622                })
8623                .await
8624            {
8625                Ok(config) => config,
8626                Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8627            }
8628        } else {
8629            None
8630        };
8631        let loaded = if let Some(approved) = approved {
8632            Ok(Some(approved))
8633        } else {
8634            crate::daemon_config::load(&context.config_path)
8635        };
8636        let loaded = match loaded {
8637            Ok(Some(config)) => config,
8638            Ok(None) => {
8639                return Ok(vec![control_error_frame(
8640                    &frame,
8641                    "invalid_daemon_config",
8642                    format!(
8643                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
8644                        context.config_path.display()
8645                    ),
8646                )?])
8647            }
8648            Err(err) => {
8649                return Ok(vec![control_error_frame(
8650                    &frame,
8651                    "invalid_daemon_config",
8652                    format!("unable to verify reserved-id release against daemon config: {err}"),
8653                )?])
8654            }
8655        };
8656        if loaded
8657            .modules
8658            .iter()
8659            .any(|configured| configured.module_id == module_id)
8660        {
8661            return Ok(vec![control_error_frame(
8662                &frame,
8663                "reserved_module_configured",
8664                format!(
8665                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
8666                ),
8667            )?]);
8668        }
8669        if !self.supervisor.release_retained_reserved_gate(&module_id) {
8670            return Ok(vec![control_error_frame(
8671                &frame,
8672                "reserved_gate_not_retained",
8673                format!(
8674                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
8675                ),
8676            )?]);
8677        }
8678
8679        let response = ClientControlResponse::SupervisorAck {
8680            module_id,
8681            applied: true,
8682        };
8683        Ok(vec![control_response_body_frame(
8684            &frame,
8685            &response,
8686            "ClientControlResponse::SupervisorAck",
8687        )?])
8688    }
8689
8690    /// Compare the configured module set with the supervised one, changing
8691    /// nothing. The staged spec each supervised module holds is what the
8692    /// configured entry is compared with.
8693    fn plan_rescan(
8694        &self,
8695        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8696    ) -> Result<RescanPlan, String> {
8697        let mut current = BTreeMap::new();
8698        for module in self.supervisor.list() {
8699            let (spec, health) = module.configuration().map_err(|err| {
8700                format!(
8701                    "failed to read configuration for module_id '{}': {err}",
8702                    module.module_id()
8703                )
8704            })?;
8705            let enabled = module
8706                .status()
8707                .map_err(|err| {
8708                    format!(
8709                        "failed to read status for module_id '{}': {err}",
8710                        module.module_id()
8711                    )
8712                })?
8713                .enabled;
8714            current.insert(
8715                module.module_id().to_string(),
8716                (module, spec, health, enabled),
8717            );
8718        }
8719        let configured = configured_modules
8720            .into_iter()
8721            .map(|module| (module.module_id.clone(), module))
8722            .collect::<BTreeMap<_, _>>();
8723
8724        let added = configured
8725            .keys()
8726            .filter(|module_id| !current.contains_key(*module_id))
8727            .cloned()
8728            .collect::<Vec<_>>();
8729        let removed = current
8730            .keys()
8731            .filter(|module_id| !configured.contains_key(*module_id))
8732            .cloned()
8733            .collect::<Vec<_>>();
8734        let mut changed_pending_reload = Vec::new();
8735        let mut configuration_changes = BTreeSet::new();
8736        let mut enabled_changes = BTreeSet::new();
8737        let mut unchanged = 0_u32;
8738
8739        for (module_id, configured_module) in &configured {
8740            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
8741            else {
8742                continue;
8743            };
8744            // Compare the whole launch spec so a future launch field cannot
8745            // accidentally become a live-only policy change. Health is stored
8746            // separately and applies live without replacing the process.
8747            let launch_changed = *current_spec != configured_module.module_spec();
8748            let configuration_changed =
8749                launch_changed || *current_health != configured_module.health;
8750            // Applying a rescan also replaces the recorded config value, even
8751            // when an operator already put the runtime in the requested state.
8752            let enabled_changed = *current_enabled != configured_module.enabled
8753                || self
8754                    .capability_evaluator
8755                    .configured_enabled(module_id)
8756                    .is_some_and(|enabled| enabled != configured_module.enabled);
8757            if configuration_changed {
8758                configuration_changes.insert(module_id.clone());
8759            }
8760            if launch_changed {
8761                changed_pending_reload.push(module_id.clone());
8762            }
8763            if enabled_changed {
8764                enabled_changes.insert(module_id.clone());
8765            }
8766            if !configuration_changed && !enabled_changed {
8767                unchanged = unchanged.saturating_add(1);
8768            }
8769        }
8770        Ok(RescanPlan {
8771            current,
8772            configured,
8773            added,
8774            removed,
8775            changed_pending_reload,
8776            configuration_changes,
8777            enabled_changes,
8778            unchanged,
8779        })
8780    }
8781
8782    /// Reconcile the running module set against the configured one.
8783    ///
8784    /// With `preview` set, the diff is computed and returned WITHOUT applying any
8785    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
8786    /// deliberately shares this function with the executing path rather than
8787    /// computing the same diff somewhere else -- two implementations of one
8788    /// decision agree until they do not, and the whole value of a preview is that
8789    /// it describes the operation that will actually run.
8790    async fn reconcile_supervised_modules(
8791        &self,
8792        supervisor: &Supervisor,
8793        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8794        preview: bool,
8795        requester: Principal,
8796    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
8797        let plan = self.plan_rescan(configured_modules)?;
8798
8799        // Everything above this point is pure computation over two snapshots.
8800        // Everything below MUTATES. The preview returns here so the boundary is a
8801        // single early return rather than a condition repeated at each mutation
8802        // site, where one missed guard would apply part of a change the caller was
8803        // told would not happen.
8804        if preview {
8805            return Ok((plan.preview_result(), Vec::new()));
8806        }
8807        let changed = plan
8808            .configuration_changes
8809            .union(&plan.enabled_changes)
8810            .cloned()
8811            .collect::<Vec<_>>();
8812        let RescanPlan {
8813            current,
8814            configured,
8815            added,
8816            removed,
8817            changed_pending_reload,
8818            configuration_changes,
8819            enabled_changes,
8820            unchanged,
8821        } = plan;
8822        // The staged generation moves before the first change rather than
8823        // after the last, so a rescan that fails partway still invalidates
8824        // every earlier `supervisor.rescan_checked` preview: part of its diff
8825        // may have run.
8826        if !(added.is_empty()
8827            && removed.is_empty()
8828            && configuration_changes.is_empty()
8829            && enabled_changes.is_empty())
8830        {
8831            if let Some(context) = &self.rescan {
8832                context
8833                    .generation
8834                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
8835            }
8836        }
8837
8838        for module_id in &removed {
8839            let module = &current
8840                .get(module_id)
8841                .expect("removed module came from current supervisor state")
8842                .0;
8843            module
8844                .retire_requested(Some(requester.clone()))
8845                .await
8846                .map_err(|err| {
8847                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
8848                })?;
8849            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
8850            //
8851            // `handle_route_open` resolves an absent module in three steps:
8852            // registry, then supervisor status, then tombstone. Retiring first
8853            // opens a window where ALL THREE ARE ABSENT -- the registry entry
8854            // went with the teardown above, the supervisor entry went with
8855            // `retire`, and the tombstone does not exist yet -- so a route.open
8856            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
8857            // it") for a module that was deliberately removed and whose caller
8858            // should get `module_removed` (TERMINAL, carrying a removal age).
8859            //
8860            // Writing the tombstone first closes it: during the window the
8861            // supervisor entry still answers, so the caller gets
8862            // `target_unavailable` -- retryable, and TRUE, because the module
8863            // is mid-teardown. After both statements it is `module_removed`.
8864            // No instant remains where a removed module reads as one that
8865            // never existed.
8866            //
8867            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
8868            // the absence of a test beside a fix invites deletion: these are
8869            // two sync statements with no await between them, so reaching the
8870            // window needs a second worker thread to land exactly between them
8871            // and there is no hook to force it. MEASURED: the 25 daemon_config
8872            // tests pass identically with the old order and the new one, so
8873            // the existing suite cannot see this and a green run is not
8874            // evidence either way. What the suite does hold is the
8875            // post-condition -- a removed module answers `module_removed` --
8876            // which this preserves.
8877            //
8878            // Found by an Athena panel reading the shipped tree against a
8879            // design note (2026-09-19), as the one concrete instance of that
8880            // note's class that survived contact with source. Direction is
8881            // benign: retryable where terminal was intended, never the reverse.
8882            self.supervisor.record_rescan_removal(module_id);
8883            self.supervisor.retire(module_id);
8884            self.route_outages.forget(module_id);
8885        }
8886
8887        for module_id in configured.keys() {
8888            let Some((module, _, _, current_enabled)) = current.get(module_id) else {
8889                continue;
8890            };
8891            let configured_module = configured
8892                .get(module_id)
8893                .expect("configured module id came from configured map");
8894            if configuration_changes.contains(module_id) {
8895                module
8896                    .update_configuration(
8897                        configured_module.module_spec(),
8898                        configured_module.health.clone(),
8899                        configured_module.drain_timeout_ms,
8900                    )
8901                    .await
8902                    .map_err(|err| {
8903                        format!(
8904                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
8905                        )
8906                    })?;
8907            }
8908            // Config-only drift is applied by the evaluator's config update;
8909            // do not turn it into a runtime start or stop.
8910            if *current_enabled != configured_module.enabled {
8911                // A rescan that starts or stops a module applies an operator's
8912                // edit to the config, so the resulting outage was asked for.
8913                self.route_outages.mark_operator_action(module_id);
8914                module
8915                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
8916                    .await
8917                    .map_err(|err| {
8918                        self.route_outages.operator_action_ended_unrefused(module_id);
8919                        format!(
8920                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
8921                            configured_module.enabled
8922                        )
8923                    })?;
8924            }
8925        }
8926
8927        for module_id in &added {
8928            let configured_module = configured
8929                .get(module_id)
8930                .expect("added module id came from configured map");
8931            supervisor
8932                .supervise_configured_requested_async(
8933                    configured_module.module_spec(),
8934                    configured_module.enabled,
8935                    configured_module.health.clone(),
8936                    configured_module.drain_timeout_ms,
8937                    configured_module.restart,
8938                    configured_module.register_timeout,
8939                    Some(requester.clone()),
8940                )
8941                .await
8942                .map_err(|err| {
8943                    format!("failed to add module_id '{module_id}' during rescan: {err}")
8944                })?;
8945        }
8946
8947        Ok((
8948            SupervisorRescanResult {
8949                added,
8950                removed,
8951                changed_pending_reload,
8952                enabled_changes: enabled_changes.iter().cloned().collect(),
8953                unchanged,
8954                preview: false,
8955                // Filled by the caller, which is the only layer that can see the
8956                // previous config to diff against.
8957                restart_required: Vec::new(),
8958                capability_warnings: Vec::new(),
8959                module_actions: Vec::new(),
8960                earlier_pending: Vec::new(),
8961                expect: None,
8962            },
8963            changed,
8964        ))
8965    }
8966
8967    async fn handle_supervisor_set_enabled(
8968        &self,
8969        frame: Frame,
8970        module_id: String,
8971        enabled: bool,
8972        requester: Principal,
8973    ) -> Result<Vec<Frame>, RouterError> {
8974        let operation_lock = self.supervisor.operation_lock();
8975        let _operation_guard = operation_lock.lock().await;
8976        let Some(module) = self.supervisor.get(&module_id) else {
8977            return Ok(vec![control_error_frame(
8978                &frame,
8979                "unknown_module",
8980                format!("module_id '{module_id}' is not supervised"),
8981            )?]);
8982        };
8983
8984        // Enabling counts as well as disabling: a module an operator starts
8985        // is refused until it registers, and that wait was asked for.
8986        self.route_outages.mark_operator_action(&module_id);
8987        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
8988            Ok(applied) => applied,
8989            Err(err) => {
8990                self.route_outages
8991                    .operator_action_ended_unrefused(&module_id);
8992                return Ok(vec![control_error_frame(
8993                    &frame,
8994                    if err.is_config_gate_error() {
8995                        "config_gated"
8996                    } else {
8997                        "target_unavailable"
8998                    },
8999                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
9000                )?]);
9001            }
9002        };
9003        if !applied {
9004            // Already in the requested state: nothing was made unavailable,
9005            // so the mark must not outlive this request.
9006            self.route_outages
9007                .operator_action_ended_unrefused(&module_id);
9008        }
9009
9010        self.capability_evaluator.wake_deadline_loop();
9011        self.refresh_capability_requirements();
9012        let response = ClientControlResponse::SupervisorAck { module_id, applied };
9013        Ok(vec![control_response_body_frame(
9014            &frame,
9015            &response,
9016            "ClientControlResponse::SupervisorAck",
9017        )?])
9018    }
9019
9020    async fn handle_supervisor_health_probe(
9021        &self,
9022        ctx: &RouteCtx,
9023        frame: Frame,
9024        module_id: String,
9025    ) -> Result<Vec<Frame>, RouterError> {
9026        self.refresh_capability_requirements();
9027        let Some(registration) = self
9028            .registry
9029            .get_module(&module_id)
9030            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
9031        else {
9032            return Ok(vec![control_error_frame(
9033                &frame,
9034                "unknown_module",
9035                format!("module_id '{module_id}' is not registered"),
9036            )?]);
9037        };
9038
9039        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
9040        // named for it. Making `module_registration_grants_op` return false
9041        // unconditionally reddens five tests, and every one is named for something
9042        // else -- capability relay, probe/bind demultiplexing, supervision-only
9043        // probing. They exercise a successful advertisement check on the way to their
9044        // own subject.
9045        //
9046        // Real protection, fragile in a specific way: narrowing any of those tests to
9047        // focus on its stated subject would silently remove coverage nobody knows
9048        // they are carrying. Recorded here rather than as a sixth test, because the
9049        // useful fact is WHICH tests hold the guard up -- a new test would add
9050        // coverage without telling the next person what the existing ones quietly do.
9051        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
9052        {
9053            return Ok(vec![control_error_frame(
9054                &frame,
9055                "health_not_advertised",
9056                format!("module_id '{module_id}' did not advertise health.check"),
9057            )?]);
9058        }
9059
9060        self.observe_control_wait(
9061            ctx,
9062            Some(&registration.manifest.module_id),
9063            ops::SUPERVISOR_HEALTH_PROBE,
9064            async {
9065                let deadline = Instant::now() + self.health_probe_timeout;
9066                let pending = match self.forwarding.begin_module_control_rpc_for(
9067                    &module_id,
9068                    MODULE_CONTROL_OP_HEALTH_CHECK,
9069                    deadline,
9070                ) {
9071                    Ok(pending) => pending,
9072                    Err(err) => {
9073                        return Ok(vec![control_error_frame(
9074                            &frame,
9075                            forwarding_error_code(&err),
9076                            err.to_string(),
9077                        )?])
9078                    }
9079                };
9080
9081                let PendingModuleControlRpc {
9082                    endpoint,
9083                    module_sink,
9084                    negotiated_ver,
9085                    corr: probe_corr,
9086                    receiver,
9087                } = pending;
9088                let mut guard =
9089                    ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
9090                let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
9091                    .map_err(|err| {
9092                        RouterError::backend(
9093                            0,
9094                            frame.header.corr,
9095                            format!("failed to encode health.check request: {err}"),
9096                        )
9097                    })?;
9098                let probe_frame = Frame::build_with_version(
9099                    negotiated_ver,
9100                    FrameType::Request,
9101                    control_flags(),
9102                    0,
9103                    0,
9104                    probe_corr,
9105                    probe_body,
9106                )
9107                .map_err(RouterError::FrameBuild)?;
9108
9109                // Queue admission and the reply share one deadline, so a module that
9110                // stops reading cannot park the requesting connection's reader forever.
9111                match timeout_at(deadline, module_sink.send(probe_frame)).await {
9112                    Ok(Ok(())) => {}
9113                    Ok(Err(err)) => {
9114                        return Ok(vec![control_error_frame(
9115                            &frame,
9116                            "target_unavailable",
9117                            err.to_string(),
9118                        )?]);
9119                    }
9120                    Err(_) => {
9121                        return Ok(vec![control_error_frame(
9122                            &frame,
9123                            "module_timeout",
9124                            format!(
9125                                "module_id '{module_id}' did not answer health.check within {:?}",
9126                                self.health_probe_timeout
9127                            ),
9128                        )?]);
9129                    }
9130                }
9131
9132                match timeout_at(deadline, receiver).await {
9133                    Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => {
9134                        Ok(vec![control_error_frame(
9135                            &frame,
9136                            "invalid_control_body",
9137                            "health.check returned an unexpected config response",
9138                        )?])
9139                    }
9140                    Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
9141                        guard.disarm();
9142                        let Some(report) = response.health_report() else {
9143                            return Ok(vec![control_error_frame(
9144                                &frame,
9145                                "invalid_control_body",
9146                                "health.check RPC returned a non-health response",
9147                            )?]);
9148                        };
9149                        // Metrics go out whole here. The supervisor's cached snapshot
9150                        // caps this blob (see truncate_health_metrics), and this path
9151                        // exists precisely to answer without that cap -- so applying it
9152                        // here would leave no way to see what the cached view drops.
9153                        let HealthReport {
9154                            status,
9155                            detail,
9156                            metrics,
9157                        } = report;
9158                        let capability_detail = self
9159                            .capability_evaluator
9160                            .required_problem_detail(&module_id);
9161                        let response = ClientControlResponse::SupervisorHealthProbe {
9162                            module_id,
9163                            status,
9164                            detail: append_capability_problem_detail(detail, capability_detail),
9165                            metrics,
9166                        };
9167                        Ok(vec![control_response_body_frame(
9168                            &frame,
9169                            &response,
9170                            "ClientControlResponse::SupervisorHealthProbe",
9171                        )?])
9172                    }
9173                    Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
9174                        guard.disarm();
9175                        // The module refused the health probe with its own body, so
9176                        // the refusal is relayed without the daemon-origin flag
9177                        // (bit 6): a client must not read it as the daemon's own
9178                        // refusal, which it may treat as never forwarded.
9179                        Ok(vec![control_relayed_error_body_frame(&frame, body)?])
9180                    }
9181                    Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
9182                        guard.disarm();
9183                        Ok(vec![control_error_frame(
9184                            &frame,
9185                            "target_unavailable",
9186                            message,
9187                        )?])
9188                    }
9189                    Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
9190                        guard.disarm();
9191                        Ok(vec![control_error_frame(
9192                            &frame,
9193                            "invalid_control_body",
9194                            message,
9195                        )?])
9196                    }
9197                    Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
9198                        guard.disarm();
9199                        Ok(vec![control_error_frame(
9200                            &frame,
9201                            "invalid_control_body",
9202                            format!("expected module-control op '{expected}', got '{actual}'"),
9203                        )?])
9204                    }
9205                    Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
9206                        guard.disarm();
9207                        Ok(vec![control_error_frame(
9208                            &frame,
9209                            "module_timeout",
9210                            format!(
9211                                "module_id '{module_id}' answered health.check after {:?}",
9212                                self.health_probe_timeout
9213                            ),
9214                        )?])
9215                    }
9216                    Ok(Err(_)) => Ok(vec![control_error_frame(
9217                        &frame,
9218                        "target_unavailable",
9219                        "health.check waiter was canceled before the module responded",
9220                    )?]),
9221                    Err(_) => Ok(vec![control_error_frame(
9222                        &frame,
9223                        "module_timeout",
9224                        format!(
9225                            "module_id '{module_id}' did not answer health.check within {:?}",
9226                            self.health_probe_timeout
9227                        ),
9228                    )?]),
9229                }
9230            },
9231        )
9232        .await
9233    }
9234
9235    fn supervisor_status(
9236        &self,
9237        module_id: &str,
9238        corr: u64,
9239    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
9240        self.supervisor
9241            .get(module_id)
9242            .map(|module| {
9243                let warming = module.is_warming_for_control("status").map_err(|err| {
9244                    RouterError::backend(
9245                        0,
9246                        corr,
9247                        format!(
9248                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
9249                        ),
9250                    )
9251                })?;
9252                module.status_for_control("status").map_err(|err| {
9253                    RouterError::backend(
9254                        0,
9255                        corr,
9256                        format!(
9257                            "failed to read supervisor status for module_id '{module_id}': {err}"
9258                        ),
9259                    )
9260                }).map(|status| (status, warming))
9261            })
9262            .transpose()
9263    }
9264
9265    fn guard_module_control_op(
9266        &self,
9267        frame: &Frame,
9268        module_id: &str,
9269        op: &str,
9270    ) -> Result<Option<Frame>, RouterError> {
9271        if self.module_grants_op(module_id, op, frame.header.corr)? {
9272            return Ok(None);
9273        }
9274
9275        Ok(Some(control_error_frame(
9276            frame,
9277            "op_not_allowed",
9278            format!("module_id '{module_id}' did not grant control op '{op}'"),
9279        )?))
9280    }
9281
9282    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
9283        let Some(registration) = self
9284            .registry
9285            .get_module(module_id)
9286            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
9287        else {
9288            return Ok(false);
9289        };
9290        Ok(module_registration_grants_op(&registration.control_ops, op))
9291    }
9292
9293    fn handle_status_update(
9294        &self,
9295        endpoint: ModuleEndpointId,
9296        frame: Frame,
9297    ) -> Result<Vec<Frame>, RouterError> {
9298        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
9299            Ok(update) => update,
9300            Err(err) => {
9301                // Forward-compat: a newer module may push a channel-0 op this subc
9302                // version doesn't know. The control contract says unknown push ops
9303                // are IGNORED, never answered with an error. Only a malformed body
9304                // for an op we DO know is a real error worth surfacing.
9305                if is_known_module_push_op(&frame.body) {
9306                    return Ok(vec![control_error_frame(
9307                        &frame,
9308                        "invalid_control_body",
9309                        format!("malformed module control push body: {err}"),
9310                    )?]);
9311                }
9312                return Ok(Vec::new());
9313            }
9314        };
9315
9316        match update {
9317            ModuleControlPush::RouteStatus {
9318                route_channel,
9319                route_epoch,
9320                status,
9321            } => {
9322                self.forwarding
9323                    .cache_status(endpoint, route_channel, route_epoch, status)
9324                    .map_err(RouterError::Forwarding)?;
9325            }
9326        }
9327        Ok(Vec::new())
9328    }
9329
9330    fn handle_route_poll(
9331        &self,
9332        ctx: &RouteCtx,
9333        frame: Frame,
9334        route_channel: u16,
9335        route_epoch: u32,
9336        kind: PollKind,
9337    ) -> Result<Vec<Frame>, RouterError> {
9338        let snapshot = self
9339            .forwarding
9340            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
9341            .map_err(RouterError::Forwarding)?;
9342        let response = match (kind, snapshot) {
9343            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
9344                ClientControlResponse::RoutePoll {
9345                    route_channel,
9346                    route_epoch,
9347                    status,
9348                    live: None,
9349                }
9350            }
9351            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9352                route_channel,
9353                route_epoch,
9354                status: None,
9355                live: None,
9356            },
9357            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
9358                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
9359                // is what makes reporting `true` correct rather than a
9360                // confident guess. `process_live` returns None only when the
9361                // module id has no supervisor snapshot at all -- an
9362                // externally-started module the daemon did not spawn -- and
9363                // for those the supervisor has no opinion to offer, ever. It
9364                // is never None for a supervised module in an unknown state:
9365                // a supervised module always has a snapshot, and the answer
9366                // comes from `state == Running && process_alive`.
9367                //
9368                // The route is Bound, so the module completed a HELLO on a
9369                // live connection; "the process this route points at is
9370                // running" is therefore attested by the binding rather than
9371                // assumed. Reporting `false` for an unsupervised module would
9372                // be the actual lie -- it would tell a client its healthy
9373                // route is dead because the daemon does not manage the
9374                // process.
9375                //
9376                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
9377                // module whose liveness is genuinely unknown, e.g. a snapshot
9378                // that has not been populated yet -- THIS DEFAULT BECOMES
9379                // WRONG and must split: unsupervised stays true, unknown
9380                // becomes null so the client can tell the two apart. The
9381                // response field is already `Option<bool>`, so the wire can
9382                // carry that distinction today.
9383                let live = self
9384                    .process_liveness
9385                    .as_ref()
9386                    .and_then(|source| source.process_live(&module_id))
9387                    .unwrap_or(true);
9388                ClientControlResponse::RoutePoll {
9389                    route_channel,
9390                    route_epoch,
9391                    status: None,
9392                    live: Some(live),
9393                }
9394            }
9395            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9396                route_channel,
9397                route_epoch,
9398                status: None,
9399                live: Some(false),
9400            },
9401        };
9402
9403        Ok(vec![control_response_body_frame(
9404            &frame,
9405            &response,
9406            "ClientControlResponse::RoutePoll",
9407        )?])
9408    }
9409
9410    pub(crate) fn observe_module_control_completion(
9411        &self,
9412        completion: ModuleControlRpcCompletion,
9413    ) -> bool {
9414        match completion {
9415            ModuleControlRpcCompletion::Unknown => false,
9416            ModuleControlRpcCompletion::Settled => true,
9417            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
9418                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
9419                info!(
9420                    module_id = %module_id,
9421                    latency_ms,
9422                    "late health.check answer proves the module is alive"
9423                );
9424                match self
9425                    .supervisor
9426                    .record_late_health_answer(&module_id, latency_ms)
9427                {
9428                    Ok(true) => {}
9429                    Ok(false) => debug!(
9430                        module_id = %module_id,
9431                        latency_ms,
9432                        "late health.check answer has no active supervisor snapshot"
9433                    ),
9434                    Err(err) => warn!(
9435                        module_id = %module_id,
9436                        latency_ms,
9437                        error = %err,
9438                        "failed to record late health.check answer"
9439                    ),
9440                }
9441                true
9442            }
9443        }
9444    }
9445
9446    /// Decide whether a failure while settling a relayed `route.bind` belongs to
9447    /// the module connection whose frame is being handled, or to the client that
9448    /// relay was opened for.
9449    ///
9450    /// This runs on the MODULE connection's frame handler, where returning `Err`
9451    /// ends that connection -- and a module connection carries every client's
9452    /// routes to that module, so ending it costs the whole fleet its tools.
9453    /// `ConnectionClosing` carries the id of the connection that is closing, and
9454    /// when that id is a CLIENT's, the condition is entirely about that one
9455    /// client's route.open. A client-scoped condition has no authority over a
9456    /// shared module connection, so it is logged and the single relay is dropped:
9457    /// the client is going away, and `complete_pending_relay` already removed the
9458    /// relay before failing, so there is nothing left to settle. Anything that
9459    /// relay still reserved is released by that client's own connection teardown,
9460    /// which is already under way -- that is what "closing" means.
9461    ///
9462    /// Every other failure is a statement about THIS connection and stays fatal:
9463    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
9464    /// id in `ConnectionClosing` all mean this connection cannot keep serving
9465    /// frames correctly.
9466    fn refuse_to_end_module_connection_for_a_client(
9467        &self,
9468        module_connection_id: ConnectionId,
9469        corr: u64,
9470        err: ForwardingError,
9471    ) -> Result<(), RouterError> {
9472        if let ForwardingError::ConnectionClosing { connection_id } = err {
9473            if connection_id != module_connection_id {
9474                warn!(
9475                    module_connection_id = module_connection_id.get(),
9476                    client_connection_id = connection_id.get(),
9477                    corr,
9478                    "dropping a route.bind response for a closing client; the module connection keeps serving"
9479                );
9480                return Ok(());
9481            }
9482        }
9483        Err(RouterError::Forwarding(err))
9484    }
9485
9486    fn handle_module_relay_response(
9487        &self,
9488        connection_id: ConnectionId,
9489        frame: Frame,
9490    ) -> Result<Vec<Frame>, RouterError> {
9491        let mut secondary_error = None;
9492        let outcome = match frame.header.ty {
9493            FrameType::Response => match module_control_response_probe(&frame.body) {
9494                Ok(probe) if probe.op == "route.bind" => {
9495                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9496                        Ok(ModuleControlResponse::RouteBindAck {}) => {
9497                            RouteBindRelayOutcome::Accepted
9498                        }
9499                        Ok(other) => {
9500                            let message =
9501                                format!("route.bind response carried unexpected body: {other:?}");
9502                            secondary_error = Some(control_error_frame(
9503                                &frame,
9504                                "invalid_control_body",
9505                                message.clone(),
9506                            )?);
9507                            RouteBindRelayOutcome::ModuleGone(message)
9508                        }
9509                        Err(err) => {
9510                            let message = format!("malformed route.bind response body: {err}");
9511                            secondary_error = Some(control_error_frame(
9512                                &frame,
9513                                "invalid_control_body",
9514                                message.clone(),
9515                            )?);
9516                            RouteBindRelayOutcome::ModuleGone(message)
9517                        }
9518                    }
9519                }
9520                Ok(probe) => {
9521                    let outcome = if matches!(probe.op.as_str(), "config.review" | "consent/v1") {
9522                        ModuleControlRpcOutcome::RawResponse(frame.body.clone())
9523                    } else {
9524                        match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9525                            Ok(response) => ModuleControlRpcOutcome::Response(response),
9526                            Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9527                                "malformed {} response body: {err}",
9528                                probe.op
9529                            )),
9530                        }
9531                    };
9532                    let completion = self
9533                        .forwarding
9534                        .complete_module_control_rpc(
9535                            connection_id,
9536                            frame.header.corr,
9537                            Some(&probe.op),
9538                            outcome,
9539                        )
9540                        .map_err(RouterError::Forwarding)?;
9541                    if !self.observe_module_control_completion(completion) {
9542                        debug!(
9543                            connection_id = connection_id.get(),
9544                            corr = frame.header.corr,
9545                            op = %probe.op,
9546                            "dropping late or unknown module-control RPC response"
9547                        );
9548                    }
9549                    return Ok(Vec::new());
9550                }
9551                Err(err) => {
9552                    if let Some(expected_op) = self
9553                        .forwarding
9554                        .pending_module_control_op(connection_id, frame.header.corr)
9555                        .map_err(RouterError::Forwarding)?
9556                    {
9557                        let completion = self
9558                            .forwarding
9559                            .complete_module_control_rpc(
9560                                connection_id,
9561                                frame.header.corr,
9562                                None,
9563                                ModuleControlRpcOutcome::MalformedResponse(format!(
9564                                    "malformed {expected_op} response body: {err}"
9565                                )),
9566                            )
9567                            .map_err(RouterError::Forwarding)?;
9568                        if !self.observe_module_control_completion(completion) {
9569                            debug!(
9570                                connection_id = connection_id.get(),
9571                                corr = frame.header.corr,
9572                                "dropping late malformed module-control RPC response"
9573                            );
9574                        }
9575                        return Ok(Vec::new());
9576                    }
9577                    let message = format!("malformed route.bind response body: {err}");
9578                    secondary_error = Some(control_error_frame(
9579                        &frame,
9580                        "invalid_control_body",
9581                        message.clone(),
9582                    )?);
9583                    RouteBindRelayOutcome::ModuleGone(message)
9584                }
9585            },
9586            FrameType::Error => {
9587                if self
9588                    .forwarding
9589                    .pending_module_control_op(connection_id, frame.header.corr)
9590                    .map_err(RouterError::Forwarding)?
9591                    .is_some()
9592                {
9593                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
9594                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
9595                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9596                            "malformed module-control ERROR body: {err}"
9597                        )),
9598                    };
9599                    let completion = self
9600                        .forwarding
9601                        .complete_module_control_rpc(
9602                            connection_id,
9603                            frame.header.corr,
9604                            None,
9605                            outcome,
9606                        )
9607                        .map_err(RouterError::Forwarding)?;
9608                    if !self.observe_module_control_completion(completion) {
9609                        debug!(
9610                            connection_id = connection_id.get(),
9611                            corr = frame.header.corr,
9612                            "dropping late or unknown module-control RPC error"
9613                        );
9614                    }
9615                    return Ok(Vec::new());
9616                }
9617                match serde_json::from_slice::<ErrorBody>(&frame.body) {
9618                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
9619                    Err(err) => {
9620                        let message = format!("malformed route.bind ERROR body: {err}");
9621                        secondary_error = Some(control_error_frame(
9622                            &frame,
9623                            "invalid_control_body",
9624                            message.clone(),
9625                        )?);
9626                        RouteBindRelayOutcome::ModuleGone(message)
9627                    }
9628                }
9629            }
9630            ty => {
9631                return Ok(vec![control_error_frame(
9632                    &frame,
9633                    "unsupported_control_frame",
9634                    format!("unsupported module channel-0 frame {ty:?}"),
9635                )?])
9636            }
9637        };
9638
9639        let settled =
9640            self.forwarding
9641                .complete_pending_relay(connection_id, frame.header.corr, outcome);
9642        let completion = match settled {
9643            Ok(completion) => completion,
9644            Err(err) => {
9645                self.refuse_to_end_module_connection_for_a_client(
9646                    connection_id,
9647                    frame.header.corr,
9648                    err,
9649                )?;
9650                return Ok(secondary_error.into_iter().collect());
9651            }
9652        };
9653        if let Some(target) = completion.abandoned.as_ref() {
9654            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
9655        }
9656        if !completion.settled {
9657            debug!(
9658                connection_id = connection_id.get(),
9659                corr = frame.header.corr,
9660                frame_type = ?frame.header.ty,
9661                "dropping late or unknown route.bind relay response"
9662            );
9663        }
9664        Ok(secondary_error.into_iter().collect())
9665    }
9666
9667    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
9668        debug!(connection_id = connection_id.get(), "handling GOODBYE");
9669        // GOODBYE ends the connection's logical session even when its socket
9670        // stays open. Use disconnect teardown so verdicts, client notices and
9671        // scope authority are released at the same lifecycle boundary.
9672        self.cleanup_connection_with_end_reason(
9673            connection_id,
9674            RegistrationEndReason::ExplicitGoodbye,
9675        )
9676        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
9677        Ok(Vec::new())
9678    }
9679}
9680
9681impl Default for ControlHandler {
9682    fn default() -> Self {
9683        Self::new(Arc::new(Registry::default()))
9684    }
9685}
9686
9687impl crate::supervise::SwapPromotionObserver for ControlHandler {
9688    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
9689        self.apply_registration_capabilities(registration);
9690    }
9691}
9692
9693fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
9694    CapabilityRequirementStatus {
9695        consumer: status.consumer,
9696        capability: status.capability,
9697        need: match status.need {
9698            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
9699            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
9700        },
9701        verdict: status.verdict.as_str().to_string(),
9702        episode_seq: status.episode_seq,
9703        config_satisfiable: status.config_satisfiable,
9704        runtime_available: status.runtime_available,
9705        detail: status.detail,
9706    }
9707}
9708
9709fn append_capability_problem_detail(
9710    detail: Option<String>,
9711    capability_detail: Option<String>,
9712) -> Option<String> {
9713    match (detail, capability_detail) {
9714        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
9715        (Some(detail), None) => Some(detail),
9716        (None, Some(capability_detail)) => Some(capability_detail),
9717        (None, None) => None,
9718    }
9719}
9720
9721fn subc_ops() -> Vec<String> {
9722    SUBC_CONTROL_OPS
9723        .iter()
9724        .map(|op| (*op).to_string())
9725        .collect()
9726}
9727
9728fn module_subc_ops() -> Vec<String> {
9729    SUBC_CONTROL_OPS
9730        .iter()
9731        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
9732        .map(|op| (*op).to_string())
9733        .collect()
9734}
9735
9736#[cfg(test)]
9737fn module_baseline_control_ops() -> Vec<String> {
9738    MODULE_BASELINE_CONTROL_OPS
9739        .iter()
9740        .map(|op| (*op).to_string())
9741        .collect()
9742}
9743
9744fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
9745    let mut seen = HashSet::new();
9746    let mut effective = Vec::new();
9747    for op in MODULE_BASELINE_CONTROL_OPS {
9748        if seen.insert((*op).to_string()) {
9749            effective.push((*op).to_string());
9750        }
9751    }
9752    for op in declared.unwrap_or_default() {
9753        if seen.insert(op.clone()) {
9754            effective.push(op);
9755        }
9756    }
9757    effective
9758}
9759
9760fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
9761    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
9762}
9763
9764fn target_module_id(target: &RouteTarget) -> &str {
9765    match target {
9766        RouteTarget::ToolProvider { module_id }
9767        | RouteTarget::ManagementSurface { module_id }
9768        | RouteTarget::InternalService { module_id, .. } => module_id,
9769    }
9770}
9771
9772fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
9773    roles.iter().any(|role| match (target, role) {
9774        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
9775        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
9776        (
9777            RouteTarget::InternalService { service_id, .. },
9778            ProviderRole::InternalService {
9779                service_id: provided,
9780                ..
9781            },
9782        ) => service_id == provided,
9783        _ => false,
9784    })
9785}
9786
9787fn is_routable_role(role: &ProviderRole) -> bool {
9788    matches!(
9789        role,
9790        ProviderRole::ToolProvider { .. }
9791            | ProviderRole::ManagementSurface { .. }
9792            | ProviderRole::InternalService { .. }
9793    )
9794}
9795
9796#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9797enum ControlRequestBodyError {
9798    UnknownOp,
9799    InvalidBody,
9800}
9801
9802#[derive(Debug, Deserialize)]
9803struct ControlOpProbe {
9804    op: String,
9805}
9806
9807fn module_control_response_probe(body: &[u8]) -> Result<ControlOpProbe, serde_json::Error> {
9808    serde_json::from_slice(body).or_else(|original| {
9809        #[derive(Deserialize)]
9810        struct ConsentProbe {
9811            protocol: String,
9812        }
9813        match serde_json::from_slice::<ConsentProbe>(body) {
9814            Ok(probe) if probe.protocol == subc_protocol::consent::CONSENT_V1 => {
9815                Ok(ControlOpProbe { op: probe.protocol })
9816            }
9817            _ => Err(original),
9818        }
9819    })
9820}
9821
9822/// Channel-0 push ops this subc version understands. A push whose `op` is not in
9823/// this set is treated as a forward-compat unknown and ignored rather than errored.
9824const MODULE_PUSH_OPS: &[&str] = &["route.status"];
9825
9826fn is_known_module_push_op(body: &[u8]) -> bool {
9827    serde_json::from_slice::<ControlOpProbe>(body)
9828        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
9829        .unwrap_or(false)
9830}
9831
9832fn is_known_module_request_op(body: &[u8]) -> bool {
9833    serde_json::from_slice::<ControlOpProbe>(body)
9834        .map(|probe| is_module_to_subc_op(&probe.op))
9835        .unwrap_or(false)
9836}
9837
9838fn is_module_to_subc_op(op: &str) -> bool {
9839    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
9840}
9841
9842fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
9843    debug!(
9844        op = %op,
9845        connection_id = connection_id.get(),
9846        corr,
9847        "control dispatch"
9848    );
9849}
9850
9851fn log_slow_control_dispatch(
9852    dispatch_started_at: Option<StdInstant>,
9853    op: &'static str,
9854    connection_id: ConnectionId,
9855    corr: u64,
9856    route_open_timings: Option<&RouteOpenTimings>,
9857) {
9858    let Some(dispatch_started_at) = dispatch_started_at else {
9859        return;
9860    };
9861    // Use the same monotonic clock as the phase timers and relay deadline.
9862    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
9863    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
9864        if let Some(timings) = route_open_timings {
9865            warn!(
9866                op = %op,
9867                connection_id = connection_id.get(),
9868                corr,
9869                elapsed_ms = elapsed.as_millis() as u64,
9870                module_id = ?timings.module_id,
9871                principal = %timings.principal,
9872                admission_ms = timings.admission.as_millis() as u64,
9873                project_root_ms = timings.project_root.as_millis() as u64,
9874                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
9875                bind_send_ms = timings.bind_send.as_millis() as u64,
9876                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
9877                "slow control dispatch"
9878            );
9879            return;
9880        }
9881        warn!(
9882            op = %op,
9883            connection_id = connection_id.get(),
9884            corr,
9885            elapsed_ms = elapsed.as_millis() as u64,
9886            "slow control dispatch"
9887        );
9888    }
9889}
9890
9891/// A test stand-in for the OS caller lookup, given the recorded endpoints.
9892#[cfg(test)]
9893type TestCallerLookup =
9894    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
9895
9896/// Every new client operation needs an explicit privilege decision here.
9897fn privileged_operation(request: &impl PrivilegedRequest) -> Option<PrivilegedOperation<'_>> {
9898    request.privileged_operation()
9899}
9900
9901trait PrivilegedRequest {
9902    fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>>;
9903}
9904
9905impl PrivilegedRequest for ClientControlRequest {
9906    fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>> {
9907        let request = self;
9908        match request {
9909        ClientControlRequest::SupervisorRestart { module_id, .. } => {
9910            Some(PrivilegedOperation::Restart { module_id })
9911        }
9912        ClientControlRequest::SupervisorSwap { module_id, .. } => {
9913            Some(PrivilegedOperation::Swap { module_id })
9914        }
9915        ClientControlRequest::SupervisorReload { module_id } => {
9916            Some(PrivilegedOperation::Reload { module_id })
9917        }
9918        ClientControlRequest::SupervisorRescan { preview } => {
9919            if *preview {
9920                None
9921            } else {
9922                Some(PrivilegedOperation::Rescan)
9923            }
9924        }
9925        // Only the well-formed apply asks for approval. A preview changes
9926        // nothing, and any other combination is refused before it touches
9927        // anything, so prompting the operator for it would ask about a request
9928        // that cannot run.
9929        ClientControlRequest::SupervisorRescanChecked {
9930            preview,
9931            candidate,
9932            expect,
9933        } => (!*preview && candidate.is_none() && expect.is_some())
9934            .then_some(PrivilegedOperation::Rescan),
9935        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
9936            Some(PrivilegedOperation::SetEnabled {
9937                module_id,
9938                enabled: *enabled,
9939            })
9940        }
9941        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
9942            Some(PrivilegedOperation::ReleaseReserved { module_id })
9943        }
9944        ClientControlRequest::ServerDescribe {}
9945        // The announcement uses the same client admission as supervisor.restart,
9946        // but cannot stop anything itself and needs no operator confirmation.
9947        | ClientControlRequest::DaemonShutdownIntent { .. }
9948        | ClientControlRequest::CatalogList { .. }
9949        | ClientControlRequest::RouteOpen { .. }
9950        | ClientControlRequest::RoutePoll { .. }
9951        | ClientControlRequest::SupervisorList {}
9952        | ClientControlRequest::SupervisorHealth {}
9953        | ClientControlRequest::SupervisorHealthProbe { .. }
9954        | ClientControlRequest::SupervisorRoutes { .. }
9955        | ClientControlRequest::SupervisorProvenance { .. }
9956        | ClientControlRequest::SupervisorStderrTail { .. }
9957        | ClientControlRequest::SupervisorTerminals { .. }
9958        | ClientControlRequest::SupervisorSpawnSnapshot {}
9959        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
9960    }
9961    }
9962}
9963
9964fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
9965    match request {
9966        ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
9967        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
9968        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
9969        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
9970        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
9971        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
9972        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
9973        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
9974        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
9975        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
9976        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
9977        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
9978        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
9979        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
9980        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
9981        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
9982        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
9983        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
9984        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
9985        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
9986        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
9987    }
9988}
9989
9990/// The caller of one `route.open`, as its `route.open refused` line names it.
9991///
9992/// A connection id alone cannot name a caller after the fact: ids restart at
9993/// every daemon restart, so the same number belongs to different modules
9994/// before and after one. These three fields are what the refused line needs
9995/// to be attributable on its own, and they are rendered with the same helpers
9996/// as the `route.open accepted` line so both directions grep alike.
9997struct RouteOpenCaller {
9998    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
9999    /// when the request presented a consumer identity whose launch nonce did
10000    /// not verify. The claimed module id is never printed as a principal.
10001    principal: String,
10002    /// `none` for an unscoped open; the admitted stamp's label once scope
10003    /// admission issued one; before that, the requested selector's label.
10004    scope: String,
10005    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
10006    /// because it is a string the caller chose.
10007    harness: String,
10008}
10009
10010impl RouteOpenCaller {
10011    fn new(
10012        principal: Option<&Principal>,
10013        scope: Option<&ScopeSelector>,
10014        identity: &BindIdentity,
10015    ) -> Self {
10016        Self {
10017            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
10018            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
10019            harness: bounded_log_text(&identity.harness),
10020        }
10021    }
10022
10023    /// For a refusal that has no parsed request at hand: names whatever the
10024    /// frame still says, and `unknown` where it says nothing parseable.
10025    fn unknown() -> Self {
10026        Self {
10027            principal: "unknown".to_string(),
10028            scope: "unknown".to_string(),
10029            harness: "unknown".to_string(),
10030        }
10031    }
10032}
10033
10034/// Longest client-chosen text (a scope ref, a harness) a route.open log line
10035/// prints before cutting it.
10036const MAX_LOGGED_TEXT_CHARS: usize = 64;
10037
10038/// `value` Debug-quoted, so control characters land escaped and cannot forge
10039/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
10040/// `…` inside the quotes so one request cannot flood the line.
10041fn bounded_log_text(value: &str) -> String {
10042    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
10043    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
10044        bounded.push('…');
10045    }
10046    format!("{bounded:?}")
10047}
10048
10049/// The `scope=` field of the `route.open accepted` line, from the stamp that
10050/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
10051/// so a ref holding control characters cannot forge a log line, and cut to 64
10052/// characters with a trailing `…` to bound the line. The bind proof and the
10053/// scope's attributes never appear.
10054fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
10055    use subc_protocol::scope::ScopeKind;
10056    let kind = match stamp.kind {
10057        ScopeKind::Head => "head",
10058        ScopeKind::Worker => "worker",
10059        ScopeKind::Ephemeral => "ephemeral",
10060        ScopeKind::Federated => "federated",
10061        _ => "unknown",
10062    };
10063    format!(
10064        "{kind}/{}/{}@{}",
10065        principal_label(&stamp.owner),
10066        bounded_log_text(&stamp.scope_ref),
10067        stamp.scope_epoch
10068    )
10069}
10070
10071/// The `scope=` field of a `route.open refused` line for an open refused
10072/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
10073/// The kind is not known yet, so the literal `requested` stands in its place.
10074/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
10075/// owner it has not been checked against the configured modules: a quoted
10076/// owner means the value was never validated. The bind proof never appears.
10077fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
10078    let epoch = selector
10079        .scope_epoch
10080        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
10081    format!(
10082        "requested/{}/{}@{epoch}",
10083        bounded_log_text(&principal_label(&selector.owner)),
10084        bounded_log_text(&selector.scope_ref),
10085    )
10086}
10087
10088fn principal_label(principal: &Principal) -> String {
10089    match principal {
10090        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
10091        Principal::Direct => "direct".to_string(),
10092        other => format!("{other:?}"),
10093    }
10094}
10095
10096fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
10097    match request {
10098        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
10099        ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { .. } => {
10100            MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS
10101        }
10102        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
10103        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
10104        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
10105        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
10106        ModuleControlRequestFromModule::RouteAuthorize { .. } => {
10107            subc_protocol::scope::ROUTE_AUTHORIZE_OP
10108        }
10109        ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
10110        _ => "unsupported",
10111    }
10112}
10113
10114fn parse_client_control_request(
10115    body: &[u8],
10116) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
10117    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
10118        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10119            Ok(probe)
10120                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
10121                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
10122            {
10123                ControlRequestBodyError::InvalidBody
10124            }
10125            Ok(_) => ControlRequestBodyError::UnknownOp,
10126            Err(_) => ControlRequestBodyError::InvalidBody,
10127        };
10128        (err, classification)
10129    })
10130}
10131
10132fn parse_module_control_request_from_module(
10133    body: &[u8],
10134) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
10135    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
10136        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10137            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
10138            Ok(_) => ControlRequestBodyError::UnknownOp,
10139            Err(_) => ControlRequestBodyError::InvalidBody,
10140        };
10141        (err, classification)
10142    })
10143}
10144
10145#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
10146enum ProviderRoleKind {
10147    ToolProvider,
10148    PipelineStage,
10149    ManagementSurface,
10150    InternalService,
10151}
10152
10153fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
10154    match role {
10155        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
10156        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
10157        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
10158        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
10159    }
10160}
10161
10162fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
10163    roles.iter().map(provider_role_kind).collect()
10164}
10165
10166/// Most refused scope records named individually in the log per sync; the
10167/// `refused` count on the accepted line is always complete.
10168const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
10169
10170/// Per-outcome counts of one accepted `scope.sync`, for its log line.
10171#[derive(Debug, Default, PartialEq, Eq)]
10172struct ScopeOutcomeCounts {
10173    created: usize,
10174    replaced: usize,
10175    updated: usize,
10176    unchanged: usize,
10177    refused: usize,
10178}
10179
10180impl ScopeOutcomeCounts {
10181    fn of(results: &[ScopeRecordResult]) -> Self {
10182        let mut counts = Self::default();
10183        for result in results {
10184            let slot = match result.outcome {
10185                ScopeRecordOutcome::Created => &mut counts.created,
10186                ScopeRecordOutcome::Replaced => &mut counts.replaced,
10187                ScopeRecordOutcome::Updated => &mut counts.updated,
10188                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
10189                ScopeRecordOutcome::Refused => &mut counts.refused,
10190            };
10191            *slot += 1;
10192        }
10193        counts
10194    }
10195}
10196
10197#[cfg(test)]
10198mod scope_outcome_count_tests {
10199    use super::*;
10200
10201    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
10202        ScopeRecordResult {
10203            scope_ref: "r".to_string(),
10204            scope_epoch: 1,
10205            outcome,
10206            code: None,
10207            message: None,
10208            version: None,
10209            parent_state: None,
10210        }
10211    }
10212
10213    /// Each outcome lands in its own count, so a refused record can never be
10214    /// hidden inside the total the log already printed.
10215    #[test]
10216    fn every_outcome_is_counted_in_its_own_field() {
10217        let results = [
10218            result(ScopeRecordOutcome::Created),
10219            result(ScopeRecordOutcome::Created),
10220            result(ScopeRecordOutcome::Replaced),
10221            result(ScopeRecordOutcome::Updated),
10222            result(ScopeRecordOutcome::Unchanged),
10223            result(ScopeRecordOutcome::Refused),
10224            result(ScopeRecordOutcome::Refused),
10225            result(ScopeRecordOutcome::Refused),
10226        ];
10227        assert_eq!(
10228            ScopeOutcomeCounts::of(&results),
10229            ScopeOutcomeCounts {
10230                created: 2,
10231                replaced: 1,
10232                updated: 1,
10233                unchanged: 1,
10234                refused: 3,
10235            }
10236        );
10237    }
10238}
10239
10240/// Return whether a catalog change can create a newly violating live route.
10241/// Removing an attested claim is intentionally excluded: it makes fewer routes
10242/// forbidden and therefore must leave the existing route census untouched.
10243fn capability_census_trigger(
10244    old: Option<&CapabilityDeclarations>,
10245    new: Option<&CapabilityDeclarations>,
10246) -> bool {
10247    let old_provides = old
10248        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
10249        .unwrap_or_default();
10250    let old_denies = old
10251        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
10252        .unwrap_or_default();
10253    let new = new.cloned().unwrap_or(CapabilityDeclarations {
10254        provides: Vec::new(),
10255        requires: Vec::new(),
10256        must_never_reach: Vec::new(),
10257    });
10258
10259    new.provides
10260        .iter()
10261        .any(|capability| !old_provides.contains(capability))
10262        || new
10263            .must_never_reach
10264            .iter()
10265            .any(|capability| !old_denies.contains(capability))
10266}
10267
10268fn catalog_update_frozen_field_message(
10269    registered: &ModuleManifest,
10270    provides: &[ProviderRole],
10271) -> Option<String> {
10272    let old_has_provides = !registered.provides.is_empty();
10273    let new_has_provides = !provides.is_empty();
10274    if old_has_provides != new_has_provides {
10275        return Some(format!(
10276            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
10277            registered.module_id
10278        ));
10279    }
10280
10281    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
10282        return Some(format!(
10283            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
10284            registered.module_id
10285        ));
10286    }
10287
10288    let registered_concurrency = manifest_concurrency(registered);
10289    let mut candidate = registered.clone();
10290    candidate.provides = provides.to_vec();
10291    let candidate_concurrency = manifest_concurrency(&candidate);
10292    if candidate_concurrency != registered_concurrency {
10293        return Some(format!(
10294            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
10295            registered.module_id, registered_concurrency, candidate_concurrency
10296        ));
10297    }
10298
10299    // control_ops live beside the manifest in the HELLO body, not inside
10300    // ModuleManifest, so a provides-only catalog.update cannot change them.
10301    None
10302}
10303
10304fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
10305    manifest.provides.iter().any(is_routable_role)
10306}
10307
10308/// Returns the routable-provider concurrency subc should enforce for this manifest.
10309///
10310/// ToolProvider and ManagementSurface store their delivery concurrency directly.
10311/// InternalService has no role-specific concurrency field, so it retains the
10312/// existing ModuleManaged default for backward compatibility.
10313fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
10314    manifest
10315        .provides
10316        .iter()
10317        .find_map(|provider| match provider {
10318            ProviderRole::ToolProvider { concurrency, .. }
10319            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
10320            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
10321        })
10322        .unwrap_or(Concurrency::ModuleManaged)
10323}
10324
10325/// True when the manifest carries a ManagementSurface role whose concurrency
10326/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
10327/// bytes because the typed manifest deliberately erases that distinction: the
10328/// default exists for wire compatibility, and this probe exists so the default
10329/// stays observable. Any parse irregularity returns false -- the caller only
10330/// logs, and a malformed body already failed registration upstream.
10331fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
10332    let has_management_surface = manifest
10333        .provides
10334        .iter()
10335        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
10336    if !has_management_surface {
10337        return false;
10338    }
10339    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
10340        return false;
10341    };
10342    let Some(provides) = raw
10343        .get("manifest")
10344        .and_then(|manifest| manifest.get("provides"))
10345        .and_then(serde_json::Value::as_array)
10346    else {
10347        return false;
10348    };
10349    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
10350    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
10351    // against the management_surface_manifest_without_concurrency golden, not
10352    // recalled (the externally-tagged guess was this function's first bug).
10353    provides.iter().any(|role| {
10354        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
10355            && role.get("concurrency").is_none()
10356    })
10357}
10358
10359fn negotiate_version(peer_version: u8) -> Result<u8, String> {
10360    if peer_version != PROTOCOL_VERSION {
10361        return Err(format!(
10362            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
10363        ));
10364    }
10365    Ok(PROTOCOL_VERSION)
10366}
10367
10368fn pong(frame: &Frame) -> Result<Frame, RouterError> {
10369    Frame::build_with_version(
10370        response_version(frame),
10371        FrameType::Pong,
10372        frame.header.flags,
10373        0,
10374        0,
10375        frame.header.corr,
10376        Vec::new(),
10377    )
10378    .map_err(RouterError::FrameBuild)
10379}
10380
10381pub(crate) fn control_error_frame(
10382    frame: &Frame,
10383    code: &'static str,
10384    message: impl Into<String>,
10385) -> Result<Frame, RouterError> {
10386    control_error_body_frame(
10387        frame,
10388        ErrorBody {
10389            code: code.to_string(),
10390            message: message.into(),
10391            detail: None,
10392        },
10393    )
10394}
10395
10396/// A channel-0 ERROR for a refusal the daemon decided itself: every
10397/// route.open refusal, the drain `module_reloading` and scope-change/end
10398/// bodies, spawn-subscription and supervisor refusals. Carries the
10399/// daemon-origin bit, which is what lets a client treat `module_reloading`
10400/// as proof that its request never reached a module. Response, push,
10401/// goodbye and relay-request frames keep the plain control flags in their
10402/// own builders.
10403pub(crate) fn control_error_body_frame(
10404    frame: &Frame,
10405    error: ErrorBody,
10406) -> Result<Frame, RouterError> {
10407    control_error_frame_with_flags(frame, error, daemon_error_flags())
10408}
10409
10410/// A channel-0 ERROR relaying a module's own rejection body unchanged: a
10411/// refused route.bind answered as a route.open refusal, or a refused
10412/// module-control RPC. Plain control flags — the code and message are
10413/// module-chosen, so the frame must not claim daemon authorship, even when
10414/// the module picks a daemon code like `module_reloading`.
10415fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
10416    control_error_frame_with_flags(frame, error, control_flags())
10417}
10418
10419fn control_error_frame_with_flags(
10420    frame: &Frame,
10421    error: ErrorBody,
10422    flags: Flags,
10423) -> Result<Frame, RouterError> {
10424    let body = serde_json::to_vec(&error).map_err(|err| {
10425        RouterError::backend(
10426            0,
10427            frame.header.corr,
10428            format!("failed to encode control ERROR: {err}"),
10429        )
10430    })?;
10431
10432    Frame::build_with_version(
10433        response_version(frame),
10434        FrameType::Error,
10435        flags,
10436        0,
10437        0,
10438        frame.header.corr,
10439        body,
10440    )
10441    .map_err(RouterError::FrameBuild)
10442}
10443
10444pub(crate) fn control_response_body_frame<T: Serialize>(
10445    frame: &Frame,
10446    reply: &T,
10447    label: &'static str,
10448) -> Result<Frame, RouterError> {
10449    let body = serde_json::to_vec(reply).map_err(|err| {
10450        RouterError::backend(
10451            0,
10452            frame.header.corr,
10453            format!("failed to encode {label}: {err}"),
10454        )
10455    })?;
10456
10457    Frame::build_with_version(
10458        response_version(frame),
10459        FrameType::Response,
10460        control_flags(),
10461        0,
10462        0,
10463        frame.header.corr,
10464        body,
10465    )
10466    .map_err(RouterError::FrameBuild)
10467}
10468
10469/// Map a forwarding failure to the wire code a client sees.
10470///
10471/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
10472/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
10473/// chosen here decides whether a caller retries or gives up.
10474///
10475/// That makes attribution the load-bearing property, not merely having a code. A
10476/// permanent fault published as a retryable one produces a fleet-wide retry storm
10477/// against something that can never recover; a transient fault published as
10478/// permanent gives up on work that would have succeeded. Both look correct in a
10479/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
10480/// the mapping per variant rather than merely asserting that some code exists.
10481///
10482/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
10483/// two codes on the same side of the boundary passes it. Measured rather than
10484/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
10485/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
10486/// a test named for something else that happens to assert the string.
10487///
10488/// That accidental coverage is deliberately left alone rather than promoted to a
10489/// named test, because it guards a property this function does not promise.
10490/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
10491/// specific code within a class, so identity is free to change and only the
10492/// partition is a contract. Splitting it out would assert a guarantee nothing
10493/// depends on — and a suite that promises more than the code does is the harder
10494/// thing to correct later, because the next reader cannot tell which assertions
10495/// are load-bearing.
10496///
10497/// Pin identity here the moment a consumer branches on a specific code.
10498/// The spawn a HELLO would register as and the provider declarations it
10499/// requests, carried from the HELLO's checks to the point the registry
10500/// accepts it.
10501struct HelloAuthority {
10502    record: Arc<crate::spawn_authority::SpawnAuthorityRecord>,
10503    requested: ProviderDeclarations,
10504    /// Whether admitting this HELLO froze the spawn's requested declarations,
10505    /// so a refusal after admission undoes that freeze and no other.
10506    froze: std::sync::atomic::AtomicBool,
10507}
10508
10509fn forwarding_error_code(err: &ForwardingError) -> &'static str {
10510    match err {
10511        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
10512        ForwardingError::NoModuleConnection => "target_unavailable",
10513        ForwardingError::ModuleReloading { .. } => "module_reloading",
10514        ForwardingError::ClientRouteChannelExhausted { .. }
10515        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
10516        ForwardingError::StaleModuleEndpoint
10517        | ForwardingError::UnknownReservation { .. }
10518        | ForwardingError::ConnectionClosing { .. }
10519        | ForwardingError::ClientEgressClosed { .. }
10520        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
10521        // Only a swap candidate's registration can produce this, and it means
10522        // exactly what a second active HELLO for a live id means.
10523        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
10524        ForwardingError::RelayCorrelationExhausted
10525        | ForwardingError::RouteOpenBuild(_)
10526        | ForwardingError::Poisoned => "forwarding_error",
10527    }
10528}
10529
10530fn response_version(frame: &Frame) -> u8 {
10531    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
10532        frame.header.ver
10533    } else {
10534        PROTOCOL_VERSION
10535    }
10536}
10537
10538fn control_flags() -> Flags {
10539    Flags::new(false, Priority::Passive, false)
10540}
10541
10542/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
10543/// channel. The target is the module (a client never saw the route), so this
10544/// takes the module path: delivered late rather than dropped when the module's
10545/// queue is momentarily full, and never closing its connection.
10546fn send_goodbye_target_best_effort(
10547    counters: &DaemonCounters,
10548    target: &GoodbyeTarget,
10549    context: &'static str,
10550) {
10551    let Ok(frame) = Frame::build_with_version(
10552        target.negotiated_ver,
10553        FrameType::Goodbye,
10554        control_flags(),
10555        target.channel,
10556        target.epoch,
10557        0,
10558        Vec::new(),
10559    ) else {
10560        return;
10561    };
10562    crate::forwarding::send_module_route_goodbye(
10563        counters,
10564        &target.sink,
10565        frame,
10566        target.module_id.as_deref(),
10567        context,
10568    );
10569}
10570
10571fn route_close_reason_for_opener(
10572    reason: RouteCloseReason,
10573    accepts_close_reasons: Option<&[String]>,
10574) -> RouteCloseReason {
10575    if reason == RouteCloseReason::AuthorityChanged
10576        && !accepts_close_reasons
10577            .is_some_and(|reasons| reasons.iter().any(|reason| reason == "authority_changed"))
10578    {
10579        // Older readers forbid reopening for unknown reasons. Reload already
10580        // permits reopening and, like an authority change, requires a new bind.
10581        RouteCloseReason::Reload
10582    } else {
10583        reason
10584    }
10585}
10586
10587pub(crate) fn send_route_control_pushes(
10588    forwarding: &ForwardingTable,
10589    routes: Vec<EndpointRoute>,
10590    push: ClientControlPush,
10591) {
10592    let reason = match &push {
10593        ClientControlPush::RouteClosing { reason, .. }
10594        | ClientControlPush::RouteClosed { reason, .. } => *reason,
10595    };
10596    let mut targets: Vec<(GoodbyeTarget, RouteCloseReason, Vec<u16>)> = Vec::new();
10597    for route in routes {
10598        let recipient_reason =
10599            route_close_reason_for_opener(reason, route.accepts_close_reasons.as_deref());
10600        let target = route.goodbye_target;
10601        if let Some((existing, _, channels)) = targets.iter_mut().find(|(existing, reason, _)| {
10602            existing.connection_id == target.connection_id && *reason == recipient_reason
10603        }) {
10604            debug_assert_eq!(
10605                existing.negotiated_ver, target.negotiated_ver,
10606                "one connection cannot negotiate multiple frame versions"
10607            );
10608            if !channels.contains(&target.channel) {
10609                channels.push(target.channel);
10610            }
10611            continue;
10612        }
10613        let channel = target.channel;
10614        targets.push((target, recipient_reason, vec![channel]));
10615    }
10616    for (target, recipient_reason, mut channels) in targets {
10617        channels.sort_unstable();
10618        let mut push = push.clone();
10619        match &mut push {
10620            ClientControlPush::RouteClosing {
10621                channels: covered,
10622                reason,
10623                ..
10624            }
10625            | ClientControlPush::RouteClosed {
10626                channels: covered,
10627                reason,
10628                ..
10629            } => {
10630                *covered = channels;
10631                *reason = recipient_reason;
10632            }
10633        }
10634        let body = match serde_json::to_vec(&push) {
10635            Ok(body) => body,
10636            Err(err) => {
10637                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
10638                continue;
10639            }
10640        };
10641        let frame = match Frame::build_with_version(
10642            target.negotiated_ver,
10643            FrameType::Push,
10644            control_flags(),
10645            0,
10646            0,
10647            0,
10648            body.clone(),
10649        ) {
10650            Ok(frame) => frame,
10651            Err(err) => {
10652                warn!(
10653                    route_channel = target.channel,
10654                    error = %err,
10655                    "failed to build route lifecycle control PUSH frame"
10656                );
10657                continue;
10658            }
10659        };
10660        if let Err(err) = target.sink.try_send(frame) {
10661            if target.close_on_delivery_failure() {
10662                warn!(
10663                    target_connection_id = target.connection_id.get(),
10664                    route_channel = target.channel,
10665                    error = %err,
10666                    "route lifecycle control PUSH was not delivered to client; closing target connection"
10667                );
10668                let _ = forwarding.escalate_client_delivery_failure(
10669                    target.connection_id,
10670                    target.channel,
10671                    target.epoch,
10672                    CloseReason::new(
10673                        "route_lifecycle_push_delivery_failed",
10674                        format!(
10675                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
10676                            target.channel
10677                        ),
10678                    ),
10679                    crate::forwarding::UndeliveredFrame {
10680                        module_id: target.module_id.as_deref(),
10681                        sink: &target.sink,
10682                    },
10683                );
10684            }
10685        }
10686    }
10687}
10688
10689#[cfg(test)]
10690mod tests {
10691    include!("control/agent_commands_tests.rs");
10692    include!("control/resource_revision_tests.rs");
10693    include!("control/route_open_recovery_tests.rs");
10694    #[tokio::test]
10695    async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
10696        let registry = Arc::new(Registry::default());
10697        let handler = ControlHandler::new(registry);
10698        assert_eq!(
10699            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10700            None
10701        );
10702        let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
10703        hello_via_sink(
10704            &handler,
10705            &ctx,
10706            &mut rx,
10707            hello_frame("daemon", PROTOCOL_VERSION, 1),
10708        )
10709        .await;
10710        assert_eq!(
10711            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10712            Some("daemon".into())
10713        );
10714        assert_eq!(
10715            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
10716            None
10717        );
10718        assert_eq!(
10719            handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
10720            None,
10721            "a registered module is not necessarily supervised"
10722        );
10723        assert_eq!(
10724            handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
10725            None,
10726            "aggregate provenance has no single target"
10727        );
10728        assert_eq!(
10729            handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
10730            None
10731        );
10732    }
10733
10734    mod peer_manifest {
10735        include!("peer_manifest_tests.rs");
10736    }
10737    mod catalog_declarations {
10738        include!("control/catalog_tests.rs");
10739    }
10740    mod activity_read {
10741        include!("activity_read_tests.rs");
10742    }
10743
10744    mod lifecycle_activity {
10745        include!("control_lifecycle_tests.rs");
10746    }
10747    #[cfg(unix)]
10748    #[tokio::test]
10749    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
10750        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
10751        let path = dir.join("subc.jsonc");
10752        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
10753            "program":"/bin/sleep","args":["60"],"protocol":"none",
10754            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
10755        }}}).to_string()).unwrap();
10756        let mut configured = crate::daemon_config::load(&path)
10757            .unwrap()
10758            .unwrap()
10759            .modules
10760            .pop()
10761            .unwrap();
10762        let registry = std::sync::Arc::new(crate::Registry::default());
10763        let handle = crate::SupervisorHandle::new();
10764        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
10765            .with_handle(handle.clone());
10766        let module = supervisor
10767            .supervise_configured_with_health(
10768                configured.module_spec(),
10769                true,
10770                configured.health.clone(),
10771                None,
10772                configured.restart,
10773            )
10774            .unwrap();
10775        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
10776        let before = module.status().unwrap().pid;
10777        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
10778        configured.health.cadence = std::time::Duration::from_secs(3600);
10779        let health_only = handler
10780            .reconcile_supervised_modules(
10781                &supervisor,
10782                vec![configured.clone()],
10783                false,
10784                Principal::Direct,
10785            )
10786            .await
10787            .unwrap();
10788        assert!(
10789            health_only.0.changed_pending_reload.is_empty(),
10790            "health policy is already applied live"
10791        );
10792        assert_eq!(module.status().unwrap().pid, before);
10793        assert_eq!(
10794            module.configuration().unwrap().1.http,
10795            configured.health.http
10796        );
10797        configured.args = vec!["61".into()];
10798        let launch = handler
10799            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
10800            .await
10801            .unwrap();
10802        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
10803        assert_eq!(
10804            module.status().unwrap().pid,
10805            before,
10806            "a launch edit is stored until reload"
10807        );
10808        module.drain().await.unwrap();
10809    }
10810    use cortexkit_test_support::ScratchDir;
10811    use std::{
10812        collections::BTreeMap,
10813        fmt,
10814        path::PathBuf,
10815        sync::{Arc, Mutex},
10816        time::Duration,
10817    };
10818
10819    use serde_json::{json, Value};
10820    use subc_protocol::{
10821        manifest::{
10822            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
10823            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
10824        },
10825        session::HealthStatus,
10826        FrameType,
10827    };
10828
10829    use super::*;
10830    use crate::{
10831        forwarding::{DataRoute, DataRouteState},
10832        registry::ChannelState,
10833        router::FrameSink,
10834        stderr_tail::DEFAULT_MAX_LINE_BYTES,
10835        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
10836        RouteCtx, Router,
10837    };
10838    use tokio::{
10839        sync::mpsc,
10840        time::{sleep, Instant},
10841    };
10842    use tracing::{
10843        field::{Field, Visit},
10844        Event, Subscriber,
10845    };
10846    use tracing_subscriber::{layer::Context, prelude::*, Layer};
10847
10848    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
10849    ///
10850    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
10851    /// is only populated for `tests/*.rs` integration test binaries -- this file
10852    /// compiles as part of the library target, which gets neither. The binary
10853    /// lives in the profile folder this test was built into,
10854    /// `<target-dir>/<profile>/fake-aft-stub`, which `cargo_profile_bin` finds
10855    /// from the test's own path.
10856    ///
10857    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
10858    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
10859    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
10860    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
10861    /// `NotFound`, which reads as a broken test rather than an unbuilt
10862    /// dependency -- so state the cause and the remedy instead. Deliberately a
10863    /// panic and not a silent skip: a test that quietly passes when it could not
10864    /// run is worse than one that fails, because it reports health it never
10865    /// verified.
10866    fn fake_aft_stub_path() -> PathBuf {
10867        let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
10868        assert!(
10869            path.exists(),
10870            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
10871             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
10872            path.display()
10873        );
10874        path
10875    }
10876
10877    /// Whether clients retry `code` in place: the predicate itself, never a copy
10878    /// of its set. A copied list breaks silently when a code is added to or
10879    /// removed from the real one, and a stale copy here would let exactly the
10880    /// failure this test exists to catch pass.
10881    fn client_retries(code: &str) -> bool {
10882        subc_protocol::error_codes::is_retryable_route_open(code)
10883    }
10884
10885    /// A code is not a label — clients branch on it, so publishing the wrong KIND
10886    /// of failure is worse than publishing none. A permanent fault dressed as
10887    /// retryable makes every client in the fleet retry forever against something
10888    /// that cannot recover; a transient fault dressed as permanent abandons work
10889    /// that would have succeeded.
10890    ///
10891    /// Asserting "a code exists" cannot catch either, because the string is free
10892    /// to say anything. This enumerates every variant and pins which side of the
10893    /// retry boundary it lands on, so a new variant must be classified here
10894    /// deliberately rather than inheriting whichever arm it was appended to.
10895    #[test]
10896    fn retryability_of_forwarding_codes_matches_the_failure() {
10897        // Transient by nature: the target is booting, reloading, or its endpoint
10898        // was swapped mid-flight. Retrying is how these resolve.
10899        let transient = [
10900            ForwardingError::NoModuleConnection,
10901            ForwardingError::ModuleReloading {
10902                module_id: "m".into(),
10903            },
10904            ForwardingError::StaleModuleEndpoint,
10905            ForwardingError::UnknownReservation {
10906                client_channel: 1,
10907                module_channel: 1,
10908            },
10909            ForwardingError::ConnectionClosing {
10910                connection_id: ConnectionId::new(1),
10911            },
10912            ForwardingError::ClientEgressClosed {
10913                connection_id: ConnectionId::new(1),
10914            },
10915            ForwardingError::ModuleEgressUnavailable {
10916                connection_id: ConnectionId::new(1),
10917            },
10918        ];
10919        for err in transient {
10920            let code = forwarding_error_code(&err);
10921            assert!(
10922                client_retries(code),
10923                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
10924            );
10925        }
10926
10927        // Not fixed by retrying. Channel and correlation exhaustion need the
10928        // caller to close routes, and a poisoned lock is a daemon that cannot
10929        // recover at all — the worst thing to advertise as retryable, since every
10930        // client would storm a daemon that will never answer.
10931        let permanent = [
10932            ForwardingError::ConnectionRoleConflict {
10933                connection_id: ConnectionId::new(1),
10934            },
10935            ForwardingError::ClientRouteChannelExhausted {
10936                connection_id: ConnectionId::new(1),
10937            },
10938            ForwardingError::ModuleRouteChannelExhausted {
10939                endpoint: ModuleEndpointId {
10940                    connection_id: ConnectionId::new(1),
10941                    generation: 1,
10942                },
10943            },
10944            ForwardingError::RelayCorrelationExhausted,
10945            ForwardingError::RouteOpenBuild("x".into()),
10946            ForwardingError::Poisoned,
10947        ];
10948        for err in permanent {
10949            let code = forwarding_error_code(&err);
10950            assert!(
10951                !client_retries(code),
10952                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
10953            );
10954        }
10955    }
10956
10957    /// The principal is the daemon's answer to "who is calling", and modules
10958    /// branch on it: aft gates bash on it, cerebellum gates browser control,
10959    /// plexus gates connector invocation. So a stamp is an authorization input in
10960    /// another process, not a label — and both possible answers SUCCEED, which is
10961    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
10962    /// first-party capability to something that never proved it; a supervised one
10963    /// stamped `Direct` silently strips a module of capability it is entitled to.
10964    ///
10965    /// Neither shows up in a test that only checks the bind succeeded. Before this
10966    /// test the only coverage was accidental —
10967    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
10968    /// stamped principal on its way past, so narrowing that wire-shape test to its
10969    /// stated subject would have deleted the last assertion on this value. It
10970    /// still asserts the stamp, which is now redundancy rather than the only
10971    /// guard: both fail under the same mutation, and this one names the reason.
10972    /// SCOPE: this handler's supervisor has spawned nothing, so
10973    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
10974    /// is unreachable here. Both assertions below are refusals, and a mutant that
10975    /// refuses everything would satisfy them.
10976    ///
10977    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
10978    /// spawns a supervised consumer, reads its live nonce, and asserts the module
10979    /// observed `principal.kind == "reserved"` carrying that module_id — verified
10980    /// at source rather than assumed, since a citation is a claim about another
10981    /// file and ages like one. Recorded because a harness that structurally
10982    /// cannot reach an arm reports "none" for that arm identically to one that
10983    /// covers it and found nothing.
10984    #[tokio::test]
10985    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
10986        let handler = ControlHandler::default();
10987        let frame =
10988            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
10989
10990        // Absent consumer_identity is the ordinary case: a human at a terminal, or
10991        // any process holding the connection file. Nothing was proved, so nothing
10992        // may be granted beyond the unattested floor.
10993        let stamped = handler
10994            .route_open_principal(ConnectionId::new(1), &frame, None)
10995            .unwrap()
10996            .unwrap();
10997        assert_eq!(
10998            stamped,
10999            Principal::Direct,
11000            "a caller that proved nothing must not be stamped as a supervised module"
11001        );
11002
11003        // A claimed module_id with a nonce no supervised child was given is a
11004        // forgery attempt, not a weaker caller: it must be REFUSED rather than
11005        // quietly demoted to Direct, or an impersonation attempt looks identical
11006        // to an ordinary unattested connection.
11007        let forged = handler
11008            .route_open_principal(
11009                ConnectionId::new(1),
11010                &frame,
11011                Some(ConsumerIdentity {
11012                    module_id: "aft".to_string(),
11013                    launch_nonce: "not-a-real-nonce".to_string(),
11014                }),
11015            )
11016            .unwrap();
11017        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
11018        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
11019    }
11020
11021    /// The test above hands `route_open_principal` an identity it built itself,
11022    /// which proves the stamping rule and nothing about where the identity comes
11023    /// from. The real producer is a wire body, and the two are joined by a serde
11024    /// field name that nothing else asserts.
11025    ///
11026    /// That join fails quietly in one specific way: an unrecognised key is simply
11027    /// absent after parsing, so a renamed or misspelled `consumer_identity`
11028    /// yields `None` and every supervised module silently drops to `Direct`.
11029    /// Capability-wise that is the safe direction, but it surfaces far from its
11030    /// cause — as a module mysteriously refused bash — and it would pass every
11031    /// test that builds its own input.
11032    ///
11033    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
11034    /// would break every client the moment the daemon gains a field, trading a
11035    /// quiet demotion for a hard refusal on additive change. Asserting the join
11036    /// instead means a rename breaks a test here rather than the fleet.
11037    #[test]
11038    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
11039        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
11040        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
11041        let ClientControlRequest::RouteOpen {
11042            consumer_identity, ..
11043        } = parsed
11044        else {
11045            panic!("route.open body must parse as RouteOpen");
11046        };
11047        assert_eq!(
11048            consumer_identity,
11049            Some(ConsumerIdentity {
11050                module_id: "aft".to_string(),
11051                launch_nonce: "n".to_string(),
11052            }),
11053            "the wire field name must reach the value route_open_principal reads"
11054        );
11055    }
11056
11057    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
11058        ModuleManifest::builder(module_id, "0.1.0")
11059            .protocol_ver(protocol_ver)
11060            .provides(vec![ProviderRole::ToolProvider {
11061                tools: vec![Tool {
11062                    name: "read".to_string(),
11063                    description: None,
11064                    execution_mode: ExecutionMode::Pure,
11065                    schema: json!({"type": "object"}),
11066                }],
11067                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
11068                concurrency: Concurrency::ModuleManaged,
11069                emits_push: true,
11070                sub_supervises: true,
11071            }])
11072            .build()
11073    }
11074
11075    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
11076        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
11077    }
11078
11079    fn hello_frame_with_control_ops(
11080        module_id: &str,
11081        protocol_ver: u8,
11082        corr: u64,
11083        control_ops: Option<Vec<String>>,
11084    ) -> Frame {
11085        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
11086    }
11087
11088    fn hello_frame_with_nonce(
11089        module_id: &str,
11090        protocol_ver: u8,
11091        corr: u64,
11092        launch_nonce: Option<&str>,
11093    ) -> Frame {
11094        hello_frame_full(
11095            module_id,
11096            protocol_ver,
11097            corr,
11098            None,
11099            launch_nonce.map(ToOwned::to_owned),
11100        )
11101    }
11102
11103    fn hello_frame_full(
11104        module_id: &str,
11105        protocol_ver: u8,
11106        corr: u64,
11107        control_ops: Option<Vec<String>>,
11108        launch_nonce: Option<String>,
11109    ) -> Frame {
11110        let body = serde_json::to_vec(&ModuleHelloBody {
11111            manifest: manifest(module_id, protocol_ver),
11112            protocol_ver,
11113            control_ops,
11114            launch_nonce,
11115        })
11116        .unwrap();
11117        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11118    }
11119
11120    fn hello_frame_with_rich_ui(
11121        module_id: &str,
11122        protocol_ver: u8,
11123        corr: u64,
11124        launch_nonce: Option<&str>,
11125        rich_ui: Value,
11126    ) -> Frame {
11127        let mut body = serde_json::to_value(ModuleHelloBody {
11128            manifest: manifest(module_id, protocol_ver),
11129            protocol_ver,
11130            control_ops: None,
11131            launch_nonce: launch_nonce.map(str::to_string),
11132        })
11133        .unwrap();
11134        body["manifest"]["rich_ui"] = rich_ui;
11135        Frame::build(
11136            FrameType::Hello,
11137            control_flags(),
11138            0,
11139            0,
11140            corr,
11141            serde_json::to_vec(&body).unwrap(),
11142        )
11143        .unwrap()
11144    }
11145
11146    fn non_routable_hello_frame_with_control_ops(
11147        module_id: &str,
11148        corr: u64,
11149        control_ops: Option<Vec<String>>,
11150    ) -> Frame {
11151        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11152        manifest.provides.clear();
11153        let body = serde_json::to_vec(&ModuleHelloBody {
11154            manifest,
11155            protocol_ver: PROTOCOL_VERSION,
11156            control_ops,
11157            launch_nonce: None,
11158        })
11159        .unwrap();
11160        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11161    }
11162
11163    fn capability_grammar_hello_frame(
11164        capabilities: Value,
11165        runtime_computed: Option<Value>,
11166        corr: u64,
11167    ) -> Frame {
11168        let mut body = serde_json::to_value(ModuleHelloBody {
11169            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
11170            protocol_ver: PROTOCOL_VERSION,
11171            control_ops: None,
11172            launch_nonce: None,
11173        })
11174        .expect("HELLO body serializes");
11175        body["manifest"]["capabilities"] = capabilities;
11176        if let Some(runtime_computed) = runtime_computed {
11177            body["runtime_computed"] = runtime_computed;
11178        }
11179        Frame::build(
11180            FrameType::Hello,
11181            control_flags(),
11182            0,
11183            0,
11184            corr,
11185            serde_json::to_vec(&body).expect("HELLO body reserializes"),
11186        )
11187        .expect("HELLO frame builds")
11188    }
11189
11190    fn channel_request(channel: u16, corr: u64) -> Frame {
11191        Frame::build(
11192            FrameType::Request,
11193            Flags::new(true, Priority::Interactive, false),
11194            channel,
11195            0,
11196            corr,
11197            b"opaque".to_vec(),
11198        )
11199        .unwrap()
11200    }
11201
11202    fn route_ctx(
11203        connection_id: ConnectionId,
11204    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11205        let (tx, rx) = mpsc::channel(8);
11206        (
11207            RouteCtx {
11208                connection_id,
11209                egress: FrameSink::new(tx),
11210            },
11211            rx,
11212        )
11213    }
11214
11215    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
11216        serde_json::from_slice(&frame.body).unwrap()
11217    }
11218
11219    /// Register a module over a connection that has a sink and return the
11220    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
11221    /// module's own sink rather than returning it as a reply, so the ack is
11222    /// taken off `rx` here and whatever the test reads next is what followed it.
11223    async fn hello_via_sink(
11224        handler: &ControlHandler,
11225        ctx: &RouteCtx,
11226        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11227        hello: Frame,
11228    ) -> Frame {
11229        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
11230        assert!(
11231            replies.is_empty(),
11232            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
11233        );
11234        let ack = rx
11235            .try_recv()
11236            .expect("HELLO_ACK is queued on the module sink")
11237            .frame;
11238        assert_eq!(ack.header.ty, FrameType::HelloAck);
11239        ack
11240    }
11241
11242    fn parse_error(frame: &Frame) -> Value {
11243        serde_json::from_slice(&frame.body).unwrap()
11244    }
11245
11246    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
11247        serde_json::from_slice(&frame.body).unwrap()
11248    }
11249
11250    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
11251        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
11252            route_channel,
11253            route_epoch: 0,
11254            kind,
11255        })
11256        .unwrap();
11257        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11258    }
11259
11260    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
11261        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
11262            module_id: module_id.to_string(),
11263        })
11264        .unwrap();
11265        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11266    }
11267
11268    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
11269        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
11270    }
11271
11272    // Exercise retired opener fields without putting them back in BindIdentity.
11273    fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
11274        let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
11275        let mut frame = route_open_frame(corr, module_id, root);
11276        let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
11277        body["identity"]["project_root"] = serde_json::json!(path.as_path());
11278        frame.body = serde_json::to_vec(&body).unwrap();
11279        frame
11280    }
11281
11282    fn route_open_frame_with_consumer_capabilities(
11283        corr: u64,
11284        module_id: &str,
11285        _project_root: ScratchDir,
11286        consumer_capabilities: Option<Vec<String>>,
11287    ) -> Frame {
11288        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11289            busy_refusals: None,
11290            target: RouteTarget::ToolProvider {
11291                module_id: module_id.to_string(),
11292            },
11293            identity: BindIdentity::new("unit".to_string(), "session".to_string()),
11294            consumer_identity: None,
11295            consumer_capabilities,
11296            role_versions: None,
11297            admission_facts: None,
11298            scope: None,
11299        })
11300        .unwrap();
11301        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11302    }
11303
11304    fn route_open_frame_with_role_versions(
11305        corr: u64,
11306        module_id: &str,
11307        _project_root: ScratchDir,
11308        role_versions: Option<BTreeMap<String, String>>,
11309    ) -> Frame {
11310        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11311            busy_refusals: None,
11312            target: RouteTarget::ToolProvider {
11313                module_id: module_id.to_string(),
11314            },
11315            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11316            consumer_identity: None,
11317            consumer_capabilities: None,
11318            role_versions,
11319            admission_facts: None,
11320            scope: None,
11321        })
11322        .unwrap();
11323        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11324    }
11325
11326    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
11327        entries
11328            .iter()
11329            .map(|(role, version)| (role.to_string(), version.to_string()))
11330            .collect()
11331    }
11332
11333    fn route_open_frame_with_admission_facts(
11334        corr: u64,
11335        module_id: &str,
11336        _project_root: ScratchDir,
11337        consumer_identity: Option<subc_control::ConsumerIdentity>,
11338        facts: Option<Value>,
11339    ) -> Frame {
11340        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11341            busy_refusals: None,
11342            target: RouteTarget::ToolProvider {
11343                module_id: module_id.to_string(),
11344            },
11345            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11346            consumer_identity,
11347            consumer_capabilities: None,
11348            role_versions: None,
11349            admission_facts: facts,
11350            scope: None,
11351        })
11352        .unwrap();
11353        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11354    }
11355
11356    #[derive(Clone, Default)]
11357    struct EventCapture {
11358        events: Arc<Mutex<Vec<CapturedEvent>>>,
11359    }
11360
11361    #[derive(Clone, Debug)]
11362    struct CapturedEvent {
11363        target: String,
11364        level: tracing::Level,
11365        fields: BTreeMap<String, String>,
11366    }
11367
11368    impl EventCapture {
11369        fn events(&self) -> Vec<CapturedEvent> {
11370            self.events.lock().unwrap().clone()
11371        }
11372    }
11373
11374    impl<S> Layer<S> for EventCapture
11375    where
11376        S: Subscriber,
11377    {
11378        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
11379            let mut visitor = EventFieldVisitor::default();
11380            event.record(&mut visitor);
11381            self.events.lock().unwrap().push(CapturedEvent {
11382                target: event.metadata().target().to_string(),
11383                level: *event.metadata().level(),
11384                fields: visitor.fields,
11385            });
11386        }
11387    }
11388
11389    #[derive(Default)]
11390    struct EventFieldVisitor {
11391        fields: BTreeMap<String, String>,
11392    }
11393
11394    impl Visit for EventFieldVisitor {
11395        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
11396            self.fields
11397                .insert(field.name().to_string(), format!("{value:?}"));
11398        }
11399    }
11400
11401    fn health_response(corr: u64, status: HealthStatus) -> Frame {
11402        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
11403            status,
11404            detail: Some("warming".to_string()),
11405            metrics: Some(json!({"queue_depth": 3})),
11406        })
11407        .unwrap();
11408        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11409    }
11410
11411    fn route_bind_ack(corr: u64) -> Frame {
11412        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
11413        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11414    }
11415
11416    fn unique_project_root(label: &str) -> ScratchDir {
11417        ScratchDir::new(label)
11418    }
11419
11420    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
11421        match parse_route_poll(frame) {
11422            ClientControlResponse::RoutePoll {
11423                status: None,
11424                live: Some(live),
11425                ..
11426            } => assert_eq!(live, expected_live),
11427            other => panic!("unexpected route.poll response: {other:?}"),
11428        }
11429    }
11430
11431    fn bind_liveness_route(
11432        registry: &Registry,
11433        forwarding: &ForwardingTable,
11434        module_id: &str,
11435    ) -> (RouteCtx, u16, u32) {
11436        let module_connection = ConnectionId::new(101);
11437        let client_connection = ConnectionId::new(202);
11438        let registration = registry
11439            .register_with_control_ops(
11440                manifest(module_id, PROTOCOL_VERSION),
11441                PROTOCOL_VERSION,
11442                module_connection,
11443                module_baseline_control_ops(),
11444            )
11445            .unwrap();
11446        let (module_tx, _module_rx) = mpsc::channel(8);
11447        let endpoint = forwarding
11448            .register_module_connection(
11449                module_connection,
11450                module_id.to_string(),
11451                PROTOCOL_VERSION,
11452                manifest_concurrency(&registration.manifest),
11453                FrameSink::new(module_tx),
11454            )
11455            .unwrap();
11456        let (client_ctx, _client_rx) = route_ctx(client_connection);
11457        let pending = forwarding
11458            .begin_route_bind_relay_for_test(
11459                client_connection,
11460                client_ctx.egress.clone(),
11461                1,
11462                module_id,
11463            )
11464            .unwrap();
11465        assert_eq!(pending.endpoint, endpoint);
11466        let route_channel = pending.client_channel;
11467        let route_epoch = pending.client_epoch;
11468        forwarding
11469            .complete_pending_relay(
11470                module_connection,
11471                pending.corr,
11472                RouteBindRelayOutcome::Accepted,
11473            )
11474            .unwrap();
11475        (client_ctx, route_channel, route_epoch)
11476    }
11477
11478    struct FakeProcessLiveness {
11479        live: Option<bool>,
11480    }
11481
11482    impl ModuleProcessLiveness for FakeProcessLiveness {
11483        fn process_live(&self, _module_id: &str) -> Option<bool> {
11484            self.live
11485        }
11486    }
11487
11488    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11489    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
11490    {
11491        let registry = Arc::new(Registry::default());
11492        let supervisor_handle = SupervisorHandle::new();
11493        let supervisor = Supervisor::new_for_test(
11494            Arc::clone(&registry),
11495            RestartPolicy::new(1, Duration::from_millis(10)),
11496        )
11497        .with_handle(supervisor_handle.clone());
11498        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
11499        let module = supervisor
11500            .spawn(ModuleSpec {
11501                carry_containment: false,
11502                module_id: "stderr-tail-wire".to_string(),
11503                program: fake_aft_stub_path(),
11504                args: Vec::new(),
11505                env: vec![
11506                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
11507                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
11508                ],
11509                reserved: false,
11510                reserved_prefixes: Vec::new(),
11511                protocol: ModuleProtocol::Subc,
11512                overlap: Default::default(),
11513            })
11514            .unwrap();
11515
11516        let deadline = Instant::now() + Duration::from_secs(5);
11517        loop {
11518            let tail = module.stderr_tail(None, None);
11519            if tail
11520                .entries
11521                .iter()
11522                .any(|entry| matches!(entry, TailEntry::ProcessStart))
11523                && tail.entries.iter().any(|entry| {
11524                    matches!(
11525                        entry,
11526                        TailEntry::Line {
11527                            truncated: true,
11528                            ..
11529                        }
11530                    )
11531                })
11532            {
11533                break;
11534            }
11535            assert!(
11536                Instant::now() < deadline,
11537                "module did not produce a truncated line and restart boundary: {tail:?}"
11538            );
11539            sleep(Duration::from_millis(10)).await;
11540        }
11541
11542        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11543        let request = ClientControlRequest::SupervisorStderrTail {
11544            module_id: "stderr-tail-wire".to_string(),
11545            max_lines: None,
11546            max_bytes: None,
11547        };
11548        let frame = Frame::build(
11549            FrameType::Request,
11550            control_flags(),
11551            0,
11552            0,
11553            1,
11554            serde_json::to_vec(&request).unwrap(),
11555        )
11556        .unwrap();
11557        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11558        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11559        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
11560            serde_json::from_slice(&responses[0].body).unwrap()
11561        else {
11562            panic!("expected supervisor.stderr_tail response");
11563        };
11564
11565        assert!(
11566            tail.entries
11567                .iter()
11568                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
11569            "the control response lost the restart boundary"
11570        );
11571        let Some(StderrTailEntry::Line {
11572            text,
11573            truncated,
11574            at_ms,
11575        }) = tail.entries.iter().find(|entry| {
11576            matches!(
11577                entry,
11578                StderrTailEntry::Line {
11579                    truncated: true,
11580                    ..
11581                }
11582            )
11583        })
11584        else {
11585            panic!("the control response lost the truncated line");
11586        };
11587        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
11588        assert!(*truncated);
11589        assert!(
11590            at_ms.is_some(),
11591            "the control response lost the line's capture time"
11592        );
11593    }
11594
11595    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
11596    /// read done on the worker thread would stall every other task until it
11597    /// finished; the read must run off the worker so this test's own task keeps
11598    /// running while the read is paused.
11599    #[tokio::test(flavor = "current_thread")]
11600    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
11601        let dir = ScratchDir::new("terminals-off-worker");
11602        let journal_path = dir.join("terminals.jsonl");
11603        let registry = Arc::new(Registry::default());
11604        let supervisor_handle = SupervisorHandle::new();
11605        let supervisor =
11606            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
11607                .with_handle(supervisor_handle.clone())
11608                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
11609        let module = supervisor
11610            .spawn(ModuleSpec {
11611                carry_containment: false,
11612                module_id: "terminal-off-worker".to_string(),
11613                program: fake_aft_stub_path(),
11614                args: Vec::new(),
11615                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11616                reserved: false,
11617                reserved_prefixes: Vec::new(),
11618                protocol: ModuleProtocol::Subc,
11619                overlap: Default::default(),
11620            })
11621            .unwrap();
11622        let deadline = Instant::now() + Duration::from_secs(5);
11623        while module.terminal_history().entries.len() != 2 {
11624            assert!(Instant::now() < deadline, "module did not record two exits");
11625            sleep(Duration::from_millis(10)).await;
11626        }
11627
11628        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
11629        let handler =
11630            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
11631        let frame = Frame::build(
11632            FrameType::Request,
11633            control_flags(),
11634            0,
11635            0,
11636            1,
11637            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
11638                module_id: "terminal-off-worker".to_string(),
11639            })
11640            .unwrap(),
11641        )
11642        .unwrap();
11643        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11644        let spawned_at = std::time::Instant::now();
11645        let read = tokio::spawn({
11646            let handler = Arc::clone(&handler);
11647            async move { handler.handle_control_frame(&ctx, frame).await }
11648        });
11649        // Waiting for the pause from a blocking thread keeps this task pending,
11650        // so the runtime's single worker is free to run the read task.
11651        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
11652            .await
11653            .unwrap()
11654            .expect("the history read reached its pause");
11655        let elapsed = spawned_at.elapsed();
11656        assert!(
11657            elapsed < Duration::from_secs(2) && !read.is_finished(),
11658            "this task could not run while the history read was paused \
11659             (resumed after {elapsed:?}, read finished: {})",
11660            read.is_finished()
11661        );
11662
11663        drop(release);
11664        let responses = read.await.unwrap().unwrap();
11665        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11666        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
11667            panic!("expected supervisor.terminals response");
11668        };
11669        assert_eq!(terminals.entries.len(), 2);
11670        assert_eq!(terminals.journal_skipped_lines, 0);
11671        assert_eq!(terminals.journal_read_errors, 0);
11672    }
11673
11674    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11675    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
11676        let registry = Arc::new(Registry::default());
11677        let supervisor_handle = SupervisorHandle::new();
11678        let supervisor =
11679            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
11680                .with_handle(supervisor_handle.clone());
11681        let module = supervisor
11682            .spawn(ModuleSpec {
11683                carry_containment: false,
11684                module_id: "terminal-golden".to_string(),
11685                program: fake_aft_stub_path(),
11686                args: Vec::new(),
11687                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11688                reserved: false,
11689                reserved_prefixes: Vec::new(),
11690                protocol: ModuleProtocol::Subc,
11691                overlap: Default::default(),
11692            })
11693            .unwrap();
11694
11695        let deadline = Instant::now() + Duration::from_secs(5);
11696        while module.terminal_history().entries.len() != 2 {
11697            assert!(
11698                Instant::now() < deadline,
11699                "module did not retain two terminal exits: {:?}",
11700                module.terminal_history()
11701            );
11702            sleep(Duration::from_millis(10)).await;
11703        }
11704
11705        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11706        let request = ClientControlRequest::SupervisorTerminals {
11707            module_id: "terminal-golden".to_string(),
11708        };
11709        let frame = Frame::build(
11710            FrameType::Request,
11711            control_flags(),
11712            0,
11713            0,
11714            1,
11715            serde_json::to_vec(&request).unwrap(),
11716        )
11717        .unwrap();
11718        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11719        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11720        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11721        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
11722            panic!("expected supervisor.terminals response");
11723        };
11724        assert_eq!(terminals.entries.len(), 2);
11725        assert_eq!(terminals.dropped, 0);
11726
11727        let mut rendered = serde_json::to_value(response).unwrap();
11728        // Wall-clock fields are the observation contract, but not stable fixture
11729        // bytes; normalize only them after the real handler has shaped the response.
11730        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
11731        for (index, entry) in rendered["entries"]
11732            .as_array_mut()
11733            .expect("terminal response entries array")
11734            .iter_mut()
11735            .enumerate()
11736        {
11737            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
11738        }
11739
11740        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11741            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
11742        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
11743        if std::env::var_os("UPDATE_GOLDEN").is_some() {
11744            std::fs::write(&golden_path, &serialized).unwrap();
11745        }
11746        let expected: Value =
11747            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
11748        assert_eq!(rendered, expected);
11749    }
11750
11751    #[test]
11752    fn hello_registers_manifest_and_returns_ack() {
11753        let registry = Arc::new(Registry::default());
11754        let handler = ControlHandler::new(Arc::clone(&registry));
11755        let conn = ConnectionId::new(1);
11756
11757        let responses = handler
11758            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
11759            .unwrap();
11760
11761        assert_eq!(responses.len(), 1);
11762        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11763        assert_eq!(responses[0].header.channel, 0);
11764        assert_eq!(responses[0].header.corr, 7);
11765        let ack = parse_ack(&responses[0]);
11766        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11767        assert!(ack
11768            .subc_capabilities
11769            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
11770        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
11771        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
11772        assert!(ack
11773            .subc_ops
11774            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
11775        assert!(ack
11776            .subc_ops
11777            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
11778        assert!(ack
11779            .subc_ops
11780            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS.to_string()));
11781
11782        let registration = registry.get_module("aft").unwrap().unwrap();
11783        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
11784        assert_eq!(registration.state, ChannelState::Active);
11785        assert_eq!(registration.connection_id, conn);
11786        assert_eq!(registration.control_ops, module_baseline_control_ops());
11787        assert!(registration.manifest.display_name.is_none());
11788    }
11789
11790    #[test]
11791    fn hello_display_name_refuses_invalid_manifest_by_field() {
11792        let registry = Arc::new(Registry::default());
11793        let handler = ControlHandler::new(Arc::clone(&registry));
11794        for (index, invalid) in [
11795            json!(""),
11796            json!("a".repeat(41)),
11797            json!(" Routing"),
11798            json!("a\nb"),
11799            json!("a\u{202e}b"),
11800            json!(7),
11801        ]
11802        .into_iter()
11803        .enumerate()
11804        {
11805            let module_id = format!("bad-display-name-{index}");
11806            let mut body = serde_json::to_value(ModuleHelloBody {
11807                manifest: manifest(&module_id, PROTOCOL_VERSION),
11808                protocol_ver: PROTOCOL_VERSION,
11809                control_ops: None,
11810                launch_nonce: None,
11811            })
11812            .unwrap();
11813            body["manifest"]["display_name"] = invalid;
11814            let hello = Frame::build(
11815                FrameType::Hello,
11816                control_flags(),
11817                0,
11818                0,
11819                7,
11820                serde_json::to_vec(&body).unwrap(),
11821            )
11822            .unwrap();
11823            let response = handler
11824                .handle_control(ConnectionId::new(100 + index as u64), hello)
11825                .unwrap();
11826            assert_eq!(response[0].header.ty, FrameType::Error);
11827            let error = parse_error(&response[0]);
11828            assert_eq!(error["code"], "invalid_manifest");
11829            assert!(error["message"].as_str().unwrap().contains("display_name"));
11830            assert!(registry.get_module(&module_id).unwrap().is_none());
11831        }
11832    }
11833
11834    #[test]
11835    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
11836        let invalid_identifiers = [
11837            ("case_change", "credentials-Provider/v1"),
11838            ("leading_zero", "credentials-provider/v01"),
11839            ("trailing_hyphen", "credentials-provider-/v1"),
11840            ("consecutive_hyphens", "credentials--provider/v1"),
11841            ("uppercase", "Credentials-provider/v1"),
11842            ("missing_v", "credentials-provider/1"),
11843            ("whitespace", "credentials provider/v1"),
11844            ("zero_version", "credentials-provider/v0"),
11845            ("out_of_range_version", "credentials-provider/v4294967296"),
11846            (
11847                "overlength_name",
11848                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
11849            ),
11850        ];
11851        let mut cases = invalid_identifiers
11852            .into_iter()
11853            .map(|(name, identifier)| {
11854                (
11855                    format!("identifier_{name}"),
11856                    "capabilities.provides[0]".to_string(),
11857                    identifier.to_string(),
11858                    json!({ "provides": [identifier] }),
11859                    None,
11860                )
11861            })
11862            .collect::<Vec<_>>();
11863        cases.extend([
11864            (
11865                "unknown_need".to_string(),
11866                "capabilities.requires[0].need".to_string(),
11867                "deferred".to_string(),
11868                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
11869                None,
11870            ),
11871            (
11872                "duplicate_provides".to_string(),
11873                "capabilities.provides[1]".to_string(),
11874                "credentials-provider/v1".to_string(),
11875                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
11876                None,
11877            ),
11878            (
11879                "duplicate_must_never_reach".to_string(),
11880                "capabilities.must_never_reach[1]".to_string(),
11881                "credentials-provider/v1".to_string(),
11882                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
11883                None,
11884            ),
11885            (
11886                "duplicate_requires_same_need".to_string(),
11887                "capabilities.requires[1]".to_string(),
11888                "credentials-provider/v1".to_string(),
11889                json!({ "requires": [
11890                    { "capability": "credentials-provider/v1", "need": "required" },
11891                    { "capability": "credentials-provider/v1", "need": "required" }
11892                ] }),
11893                None,
11894            ),
11895            (
11896                "duplicate_requires_conflicting_need".to_string(),
11897                "capabilities.requires[1]".to_string(),
11898                "credentials-provider/v1".to_string(),
11899                json!({ "requires": [
11900                    { "capability": "credentials-provider/v1", "need": "required" },
11901                    { "capability": "credentials-provider/v1", "need": "optional" }
11902                ] }),
11903                None,
11904            ),
11905            (
11906                "capabilities_root_pointer".to_string(),
11907                "runtime_computed[0]".to_string(),
11908                "/capabilities".to_string(),
11909                json!({}),
11910                Some(json!(["/capabilities"])),
11911            ),
11912            (
11913                "capabilities_descendant_pointer".to_string(),
11914                "runtime_computed[0]".to_string(),
11915                "/capabilities/provides".to_string(),
11916                json!({}),
11917                Some(json!(["/capabilities/provides"])),
11918            ),
11919            (
11920                "malformed_pointer_without_leading_slash".to_string(),
11921                "runtime_computed[0]".to_string(),
11922                "capabilities".to_string(),
11923                json!({}),
11924                Some(json!(["capabilities"])),
11925            ),
11926            (
11927                "malformed_pointer_escape".to_string(),
11928                "runtime_computed[0]".to_string(),
11929                "/roles/~2/tools".to_string(),
11930                json!({}),
11931                Some(json!(["/roles/~2/tools"])),
11932            ),
11933            (
11934                "unknown_capabilities_field".to_string(),
11935                "capabilities.future".to_string(),
11936                "<array>".to_string(),
11937                json!({ "future": [] }),
11938                None,
11939            ),
11940        ]);
11941
11942        for (index, (name, field, value, capabilities, runtime_computed)) in
11943            cases.into_iter().enumerate()
11944        {
11945            let registry = Arc::new(Registry::default());
11946            let handler = ControlHandler::new(Arc::clone(&registry));
11947            let response = handler
11948                .handle_control(
11949                    ConnectionId::new((index + 1) as u64),
11950                    capability_grammar_hello_frame(
11951                        capabilities,
11952                        runtime_computed,
11953                        index as u64 + 1,
11954                    ),
11955                )
11956                .expect("invalid HELLO returns a refusal");
11957
11958            assert_eq!(response.len(), 1, "{name} must emit one refusal");
11959            let error = parse_error(&response[0]);
11960            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
11961            let message = error["message"]
11962                .as_str()
11963                .expect("error message is a string");
11964            assert!(
11965                message.contains(&field),
11966                "{name}: field missing from {message}"
11967            );
11968            assert!(
11969                message.contains(&value),
11970                "{name}: value missing from {message}"
11971            );
11972            assert_eq!(
11973                registry
11974                    .active_registration_count()
11975                    .expect("registry reads"),
11976                0,
11977                "{name}: refused HELLO must not create a catalog entry"
11978            );
11979        }
11980    }
11981
11982    #[test]
11983    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
11984        let registry = Arc::new(Registry::default());
11985        let handler = ControlHandler::new(Arc::clone(&registry));
11986        let capabilities = json!({
11987            "provides": ["credentials-provider/v1"],
11988            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
11989            "must_never_reach": ["federation-transport/v1"]
11990        });
11991        let response = handler
11992            .handle_control(
11993                ConnectionId::new(99),
11994                capability_grammar_hello_frame(
11995                    capabilities.clone(),
11996                    Some(json!(["/roles/0/tools"])),
11997                    99,
11998                ),
11999            )
12000            .expect("valid HELLO registers");
12001        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12002
12003        let request = Frame::build(
12004            FrameType::Request,
12005            control_flags(),
12006            0,
12007            0,
12008            100,
12009            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12010                .expect("catalog request serializes"),
12011        )
12012        .expect("catalog request frame builds");
12013        let response = handler
12014            .handle_catalog_list(request, None)
12015            .expect("catalog list succeeds");
12016        let ClientControlResponse::CatalogList { modules, .. } =
12017            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
12018        else {
12019            panic!("catalog request must return catalog.list");
12020        };
12021        assert_eq!(modules.len(), 1);
12022        assert_eq!(
12023            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
12024            capabilities
12025        );
12026    }
12027
12028    #[test]
12029    fn catalog_list_mirrors_management_operation_description() {
12030        let registry = Arc::new(Registry::default());
12031        let handler = ControlHandler::new(Arc::clone(&registry));
12032        let description = "List managed records and return their identifiers and metadata.";
12033        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
12034        manifest.provides = vec![ProviderRole::ManagementSurface {
12035            operations: vec![ManagementOperation {
12036                name: "records.list".to_string(),
12037                kind: ManagementOperationKind::Query,
12038                description: Some(description.to_string()),
12039            }],
12040            config_schema: json!({"type": "object"}),
12041            observability: vec![ObservabilitySurface {
12042                name: "records.stats".to_string(),
12043                kind: ObservabilityKind::Snapshot,
12044            }],
12045            identity_scope: vec![IdentityScope::Project],
12046            concurrency: Concurrency::ModuleManaged,
12047        }];
12048        registry
12049            .register_with_control_ops(
12050                manifest,
12051                PROTOCOL_VERSION,
12052                ConnectionId::new(99),
12053                Vec::new(),
12054            )
12055            .expect("described management manifest registers");
12056
12057        let request = Frame::build(
12058            FrameType::Request,
12059            control_flags(),
12060            0,
12061            0,
12062            100,
12063            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12064                .expect("catalog request serializes"),
12065        )
12066        .expect("catalog request frame builds");
12067        let response = handler
12068            .handle_catalog_list(request, None)
12069            .expect("catalog list succeeds");
12070        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
12071        assert_eq!(
12072            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
12073            "catalog.list must preserve the declared operation description verbatim"
12074        );
12075    }
12076
12077    #[test]
12078    fn catalog_list_mirrors_registered_operation_delivery() {
12079        let registry = Arc::new(Registry::default());
12080        let handler = ControlHandler::new(Arc::clone(&registry));
12081        let delivery =
12082            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
12083        let mut body = serde_json::to_value(ModuleHelloBody {
12084            manifest: manifest("watch-management", PROTOCOL_VERSION),
12085            protocol_ver: PROTOCOL_VERSION,
12086            control_ops: None,
12087            launch_nonce: None,
12088        })
12089        .unwrap();
12090        body["manifest"]["provides"] = json!([{
12091            "role": "management_surface",
12092            "operations": [
12093                {"name": "status", "kind": "query"},
12094                {"name": "future", "kind": "query"},
12095                {"name": "reset", "kind": "mutate"}
12096            ],
12097            "config_schema": {}, "observability": [], "identity_scope": []
12098        }]);
12099        body["manifest"]["operation_delivery"] = delivery.clone();
12100        let hello = Frame::build(
12101            FrameType::Hello,
12102            control_flags(),
12103            0,
12104            0,
12105            99,
12106            serde_json::to_vec(&body).unwrap(),
12107        )
12108        .unwrap();
12109        let response = handler
12110            .handle_control(ConnectionId::new(99), hello)
12111            .unwrap();
12112        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12113
12114        let (_, registrations) = registry.list_modules().unwrap();
12115        let registered = &registrations[0].manifest;
12116        assert!(registered.is_watch_operation("status"));
12117        assert!(!registered.is_watch_operation("future"));
12118        assert!(!registered.is_watch_operation("reset"));
12119        assert!(!registered.is_watch_operation("missing"));
12120
12121        let request = Frame::build(
12122            FrameType::Request,
12123            control_flags(),
12124            0,
12125            0,
12126            100,
12127            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12128        )
12129        .unwrap();
12130        let response = handler.handle_catalog_list(request, None).unwrap();
12131        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
12132        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
12133        let ClientControlResponse::CatalogList { modules, .. } =
12134            serde_json::from_slice(&response[0].body).unwrap()
12135        else {
12136            panic!("expected catalog.list");
12137        };
12138        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
12139    }
12140
12141    #[test]
12142    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
12143        let registry = Arc::new(Registry::default());
12144        let supervisor = SupervisorHandle::new();
12145        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12146        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
12147        let descriptor = json!({
12148            "schema": "ckios.richui.manifest/1",
12149            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12150            "bytes": 4096,
12151            "chunks": 4
12152        });
12153        let connection = ConnectionId::new(201);
12154        let hello = handler
12155            .handle_control(
12156                connection,
12157                hello_frame_with_rich_ui(
12158                    "vault",
12159                    PROTOCOL_VERSION,
12160                    201,
12161                    Some("the-real-nonce"),
12162                    descriptor.clone(),
12163                ),
12164            )
12165            .unwrap();
12166        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12167
12168        let request =
12169            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
12170        let provides = registry
12171            .get_module_by_connection(connection)
12172            .unwrap()
12173            .unwrap()
12174            .manifest
12175            .provides;
12176        handler
12177            .handle_catalog_update(connection, request, provides, None, Some(false))
12178            .unwrap();
12179
12180        let request = Frame::build(
12181            FrameType::Request,
12182            control_flags(),
12183            0,
12184            0,
12185            203,
12186            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12187        )
12188        .unwrap();
12189        let response = handler.handle_catalog_list(request, None).unwrap();
12190        let ClientControlResponse::CatalogList { modules, .. } =
12191            serde_json::from_slice(&response[0].body).unwrap()
12192        else {
12193            panic!("expected catalog.list");
12194        };
12195        assert_eq!(
12196            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
12197            descriptor
12198        );
12199        assert!(modules[0].nonce_verified);
12200    }
12201
12202    fn hello_with_role_declarations(declarations: Value) -> Frame {
12203        let mut body = serde_json::to_value(ModuleHelloBody {
12204            manifest: manifest("role-test", PROTOCOL_VERSION),
12205            protocol_ver: PROTOCOL_VERSION,
12206            control_ops: None,
12207            launch_nonce: None,
12208        })
12209        .unwrap();
12210        body["manifest"]["role_declarations"] = declarations;
12211        Frame::build(
12212            FrameType::Hello,
12213            control_flags(),
12214            0,
12215            0,
12216            301,
12217            serde_json::to_vec(&body).unwrap(),
12218        )
12219        .unwrap()
12220    }
12221
12222    fn update_role_declarations_frame(declarations: Value) -> Frame {
12223        Frame::build(
12224            FrameType::Request,
12225            control_flags(),
12226            0,
12227            0,
12228            303,
12229            serde_json::to_vec(&json!({
12230                "op": "catalog.update_role_declarations",
12231                "role_declarations": declarations,
12232            }))
12233            .unwrap(),
12234        )
12235        .unwrap()
12236    }
12237
12238    #[tokio::test]
12239    async fn role_declarations_update_replaces_then_clears_catalog() {
12240        let registry = Arc::new(Registry::default());
12241        let handler = ControlHandler::new(Arc::clone(&registry));
12242        let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12243        let initial = json!({"media-provider/v1": {"engines": ["old"]}, "old/v1": {}});
12244        hello_via_sink(
12245            &handler,
12246            &ctx,
12247            &mut rx,
12248            hello_with_role_declarations(initial.clone()),
12249        )
12250        .await;
12251        assert_eq!(
12252            role_declarations_catalog(&handler).0["role_declarations"],
12253            initial
12254        );
12255        let before = registry.get_module("role-test").unwrap().unwrap();
12256        for declarations in [
12257            json!({"media-provider/v1": {"engines": ["new"]}}),
12258            json!({}),
12259        ] {
12260            let response = handler
12261                .handle_control_frame(&ctx, update_role_declarations_frame(declarations.clone()))
12262                .await
12263                .unwrap();
12264            assert_eq!(response[0].header.ty, FrameType::Response);
12265            assert_eq!(
12266                serde_json::from_slice::<Value>(&response[0].body).unwrap(),
12267                json!({"op": "catalog.update_role_declarations"})
12268            );
12269            let (raw, entry) = role_declarations_catalog(&handler);
12270            assert_eq!(
12271                serde_json::to_value(entry.role_declarations).unwrap(),
12272                declarations
12273            );
12274            if declarations.as_object().unwrap().is_empty() {
12275                assert!(raw.get("role_declarations").is_none());
12276            } else {
12277                assert_eq!(raw["role_declarations"], declarations);
12278            }
12279            let after = registry.get_module("role-test").unwrap().unwrap();
12280            let mut expected = before.manifest.clone();
12281            expected.role_declarations = serde_json::from_value(declarations).unwrap();
12282            assert_eq!(after.manifest, expected);
12283            assert_eq!(after.ready, before.ready);
12284        }
12285    }
12286
12287    #[tokio::test]
12288    async fn role_declarations_update_refuses_invalid_maps_without_changing_catalog() {
12289        let registry = Arc::new(Registry::default());
12290        let handler = ControlHandler::new(registry);
12291        let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12292        let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12293        hello_via_sink(
12294            &handler,
12295            &ctx,
12296            &mut rx,
12297            hello_with_role_declarations(initial.clone()),
12298        )
12299        .await;
12300        for (invalid, reason) in [
12301            (json!(null), "role_declarations must be an object"),
12302            (json!([]), "role_declarations must be an object"),
12303            (
12304                json!({"bad key": {}}),
12305                "role_declarations key must use capability-name grammar",
12306            ),
12307            (
12308                json!({"media-provider/v1": []}),
12309                "role_declarations values must be objects",
12310            ),
12311            (
12312                json!({"media-provider/v1": {"padding": "x".repeat(16384)}}),
12313                "role_declarations exceeds 16384 compact JSON bytes",
12314            ),
12315        ] {
12316            let response = handler
12317                .handle_control_frame(&ctx, update_role_declarations_frame(invalid))
12318                .await
12319                .unwrap();
12320            let error = parse_error(&response[0]);
12321            assert_eq!(error["code"], "invalid_role_declarations");
12322            assert_eq!(error["message"], reason);
12323            assert_eq!(
12324                role_declarations_catalog(&handler).0["role_declarations"],
12325                initial
12326            );
12327        }
12328    }
12329
12330    #[tokio::test]
12331    async fn role_declarations_update_is_owned_by_the_registered_connection() {
12332        let registry = Arc::new(Registry::default());
12333        let handler = ControlHandler::new(Arc::clone(&registry));
12334        let (owner, mut owner_rx) = route_ctx(ConnectionId::new(301));
12335        let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12336        hello_via_sink(
12337            &handler,
12338            &owner,
12339            &mut owner_rx,
12340            hello_with_role_declarations(initial.clone()),
12341        )
12342        .await;
12343        let (other, mut other_rx) = route_ctx(ConnectionId::new(302));
12344        let replacement = json!({"media-provider/v1": {"engines": ["other"]}});
12345        let response = handler
12346            .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12347            .await
12348            .unwrap();
12349        assert_eq!(parse_error(&response[0])["code"], "not_registered");
12350        hello_via_sink(
12351            &handler,
12352            &other,
12353            &mut other_rx,
12354            hello_frame("other", PROTOCOL_VERSION, 304),
12355        )
12356        .await;
12357        let response = handler
12358            .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12359            .await
12360            .unwrap();
12361        assert_eq!(response[0].header.ty, FrameType::Response);
12362        assert_eq!(
12363            serde_json::to_value(
12364                registry
12365                    .get_module("role-test")
12366                    .unwrap()
12367                    .unwrap()
12368                    .manifest
12369                    .role_declarations
12370            )
12371            .unwrap(),
12372            initial
12373        );
12374        assert_eq!(
12375            serde_json::to_value(
12376                registry
12377                    .get_module("other")
12378                    .unwrap()
12379                    .unwrap()
12380                    .manifest
12381                    .role_declarations
12382            )
12383            .unwrap(),
12384            replacement
12385        );
12386    }
12387
12388    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
12389        let request = Frame::build(
12390            FrameType::Request,
12391            control_flags(),
12392            0,
12393            0,
12394            302,
12395            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12396        )
12397        .unwrap();
12398        let response = handler.handle_catalog_list(request, None).unwrap();
12399        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
12400        let ClientControlResponse::CatalogList { mut modules, .. } =
12401            serde_json::from_slice(&response[0].body).unwrap()
12402        else {
12403            panic!("expected catalog.list");
12404        };
12405        (raw["modules"][0].clone(), modules.remove(0))
12406    }
12407
12408    #[test]
12409    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
12410        let registry = Arc::new(Registry::default());
12411        let handler = ControlHandler::new(Arc::clone(&registry));
12412        let declarations = json!({
12413            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
12414            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
12415        });
12416        let connection = ConnectionId::new(301);
12417        let response = handler
12418            .handle_control(
12419                connection,
12420                hello_with_role_declarations(declarations.clone()),
12421            )
12422            .unwrap();
12423        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12424        let registered = registry
12425            .get_module_by_connection(connection)
12426            .unwrap()
12427            .unwrap()
12428            .manifest;
12429        // Compared as JSON values, not bytes: object key order is not part of
12430        // the relay's promise, and workspace builds unify serde_json's
12431        // preserve_order feature, which changes the order a literal keeps.
12432        assert_eq!(
12433            serde_json::to_value(&registered.role_declarations).unwrap(),
12434            declarations
12435        );
12436        let (raw, entry) = role_declarations_catalog(&handler);
12437        assert_eq!(raw["role_declarations"], declarations);
12438        assert_eq!(entry.role_declarations, registered.role_declarations);
12439
12440        let request =
12441            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
12442        handler
12443            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
12444            .unwrap();
12445        let (raw, entry) = role_declarations_catalog(&handler);
12446        assert_eq!(raw["role_declarations"], declarations);
12447        assert_eq!(entry.role_declarations, registered.role_declarations);
12448    }
12449
12450    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
12451        let registry = Arc::new(Registry::default());
12452        let handler = ControlHandler::new(Arc::clone(&registry));
12453        let capture = EventCapture::default();
12454        let _subscriber =
12455            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12456        let response = handler
12457            .handle_control(
12458                ConnectionId::new(301),
12459                hello_with_role_declarations(declarations),
12460            )
12461            .unwrap();
12462        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12463        assert!(registry
12464            .get_module_by_connection(ConnectionId::new(301))
12465            .unwrap()
12466            .unwrap()
12467            .manifest
12468            .role_declarations
12469            .is_empty());
12470        let (raw, entry) = role_declarations_catalog(&handler);
12471        assert!(raw.get("role_declarations").is_none());
12472        assert!(entry.role_declarations.is_empty());
12473        let warnings: Vec<_> = capture
12474            .events()
12475            .into_iter()
12476            .filter(|event| {
12477                event.fields.get("message").map(String::as_str)
12478                    == Some("dropping whole role_declarations map from HELLO")
12479            })
12480            .collect();
12481        assert_eq!(warnings.len(), 1);
12482        assert_eq!(warnings[0].level, tracing::Level::WARN);
12483        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
12484        assert_eq!(warnings[0].fields["reason"], reason);
12485    }
12486
12487    #[test]
12488    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
12489        for value in [
12490            json!(null),
12491            json!(false),
12492            json!(0),
12493            json!("opaque"),
12494            json!([]),
12495        ] {
12496            assert_hello_drops_role_declarations_with_warning(
12497                json!({"routing/v1": {}, "broken/v1": value}),
12498                "role_declarations values must be objects",
12499            );
12500        }
12501        assert_hello_drops_role_declarations_with_warning(
12502            json!([]),
12503            "role_declarations must be an object",
12504        );
12505    }
12506
12507    #[test]
12508    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
12509        assert_hello_drops_role_declarations_with_warning(
12510            json!({"routing/v1": {}, "bad key": {}}),
12511            "role_declarations key must use capability-name grammar",
12512        );
12513    }
12514
12515    #[test]
12516    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
12517        assert_hello_drops_role_declarations_with_warning(
12518            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
12519            "role_declarations exceeds 16384 compact JSON bytes",
12520        );
12521    }
12522
12523    fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
12524        let mut body = serde_json::to_value(ModuleHelloBody {
12525            manifest: manifest("flow-ops", PROTOCOL_VERSION),
12526            protocol_ver: PROTOCOL_VERSION,
12527            control_ops: None,
12528            launch_nonce: None,
12529        })
12530        .unwrap();
12531        body["manifest"]["provides"] = json!([{
12532            "role": "management_surface",
12533            "operations": [
12534                {"name": "diagnostics.findings", "kind": "query"},
12535                {"name": "reset", "kind": "mutate"}
12536            ],
12537            "config_schema": {}, "observability": [], "identity_scope": []
12538        }]);
12539        body["manifest"]["capabilities"] = json!({ "provides": capabilities });
12540        if let Some(flow_operations) = flow_operations {
12541            body["manifest"]["flow_operations"] = flow_operations;
12542        }
12543        Frame::build(
12544            FrameType::Hello,
12545            control_flags(),
12546            0,
12547            0,
12548            401,
12549            serde_json::to_vec(&body).unwrap(),
12550        )
12551        .unwrap()
12552    }
12553
12554    #[test]
12555    fn catalog_list_mirrors_flow_operations_unchanged() {
12556        // Order is kept as declared (not sorted) and an empty list stays an
12557        // affirmative "none", distinct from an absent declaration.
12558        for (capability, flow_operations) in [
12559            ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
12560            ("call-modes/v1", json!(["diagnostics.findings"])),
12561            ("flow-scopes/v1", json!([])),
12562        ] {
12563            let registry = Arc::new(Registry::default());
12564            let handler = ControlHandler::new(Arc::clone(&registry));
12565            let connection = ConnectionId::new(401);
12566            let response = handler
12567                .handle_control(
12568                    connection,
12569                    hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
12570                )
12571                .unwrap();
12572            assert_eq!(response[0].header.ty, FrameType::HelloAck);
12573            let registered = registry
12574                .get_module_by_connection(connection)
12575                .unwrap()
12576                .unwrap()
12577                .manifest;
12578            assert_eq!(
12579                serde_json::to_value(&registered.flow_operations).unwrap(),
12580                flow_operations
12581            );
12582            let (raw, entry) = role_declarations_catalog(&handler);
12583            assert_eq!(raw["flow_operations"], flow_operations);
12584            assert_eq!(entry.flow_operations, registered.flow_operations);
12585        }
12586
12587        let registry = Arc::new(Registry::default());
12588        let handler = ControlHandler::new(Arc::clone(&registry));
12589        let response = handler
12590            .handle_control(
12591                ConnectionId::new(401),
12592                hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
12593            )
12594            .unwrap();
12595        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12596        let (raw, entry) = role_declarations_catalog(&handler);
12597        assert!(raw.get("flow_operations").is_none());
12598        assert_eq!(entry.flow_operations, None);
12599    }
12600
12601    #[test]
12602    fn operation_deferral_hello_accepts_valid_and_invalid_maps_and_catalog_relays_only_valid() {
12603        let oversized: serde_json::Map<String, Value> = (0..257)
12604            .map(|n| (format!("tool:t{n}"), json!(true)))
12605            .collect();
12606        for (capabilities, map, valid) in [
12607            (
12608                json!(["scope-federated/v1"]),
12609                json!({"tool:read":true,"management:reset":true}),
12610                true,
12611            ),
12612            (json!(["scope-federated/v1"]), json!({}), true),
12613            (json!([]), json!({"tool:read":true}), false),
12614            (
12615                json!(["scope-federated/v1"]),
12616                json!({"tool:valid":true,"tool:bad":false}),
12617                false,
12618            ),
12619            (
12620                json!(["scope-federated/v1"]),
12621                json!({"tool:valid":true,"management:missing":true}),
12622                false,
12623            ),
12624            (
12625                json!(["scope-federated/v1"]),
12626                Value::Object(oversized),
12627                false,
12628            ),
12629        ] {
12630            let capture = EventCapture::default();
12631            let _subscriber = tracing::subscriber::set_default(
12632                tracing_subscriber::registry().with(capture.clone()),
12633            );
12634            let registry = Arc::new(Registry::default());
12635            let handler = ControlHandler::new(Arc::clone(&registry));
12636            let hello = hello_with_flow_operations(capabilities, None);
12637            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12638            body["manifest"]["operation_deferral"] = map.clone();
12639            let hello = Frame::build(
12640                FrameType::Hello,
12641                control_flags(),
12642                0,
12643                0,
12644                401,
12645                serde_json::to_vec(&body).unwrap(),
12646            )
12647            .unwrap();
12648            let response = handler
12649                .handle_control(ConnectionId::new(401), hello)
12650                .unwrap();
12651            assert_eq!(response[0].header.ty, FrameType::HelloAck);
12652            let (raw, entry) = role_declarations_catalog(&handler);
12653            if valid {
12654                assert_eq!(raw["operation_deferral"], map);
12655                assert_eq!(
12656                    serde_json::to_value(entry.operation_deferral.unwrap()).unwrap(),
12657                    map
12658                );
12659            } else {
12660                assert!(entry.operation_deferral.is_none());
12661                assert!(raw.get("operation_deferral").is_none());
12662                assert_eq!(
12663                    capture
12664                        .events()
12665                        .iter()
12666                        .filter(|event| event.fields.get("message").map(String::as_str)
12667                            == Some("dropping operation_deferral from HELLO"))
12668                        .count(),
12669                    1
12670                );
12671            }
12672        }
12673    }
12674
12675    fn assert_hello_drops_flow_operations_with_warning(
12676        capabilities: Value,
12677        flow_operations: Value,
12678        reason: &str,
12679    ) {
12680        let registry = Arc::new(Registry::default());
12681        let handler = ControlHandler::new(Arc::clone(&registry));
12682        let capture = EventCapture::default();
12683        let _subscriber =
12684            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12685        let response = handler
12686            .handle_control(
12687                ConnectionId::new(401),
12688                hello_with_flow_operations(capabilities, Some(flow_operations)),
12689            )
12690            .unwrap();
12691        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12692        let registered = registry
12693            .get_module_by_connection(ConnectionId::new(401))
12694            .unwrap()
12695            .expect("the module still registers")
12696            .manifest;
12697        assert_eq!(registered.module_id, "flow-ops");
12698        assert_eq!(registered.flow_operations, None);
12699        let (raw, entry) = role_declarations_catalog(&handler);
12700        assert_eq!(raw["module_id"], "flow-ops");
12701        assert!(raw.get("flow_operations").is_none());
12702        assert_eq!(entry.flow_operations, None);
12703        let warnings: Vec<_> = capture
12704            .events()
12705            .into_iter()
12706            .filter(|event| {
12707                event.fields.get("message").map(String::as_str)
12708                    == Some("dropping flow_operations from HELLO")
12709            })
12710            .collect();
12711        assert_eq!(warnings.len(), 1);
12712        assert_eq!(warnings[0].level, tracing::Level::WARN);
12713        assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
12714        assert_eq!(warnings[0].fields["reason"], reason);
12715    }
12716
12717    #[test]
12718    fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
12719        assert_hello_drops_flow_operations_with_warning(
12720            json!(["flow-scopes/v1"]),
12721            json!(["diagnostics.findings", "missing"]),
12722            "flow_operations names 'missing', which is not a declared management operation",
12723        );
12724    }
12725
12726    #[test]
12727    fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
12728        assert_hello_drops_flow_operations_with_warning(
12729            json!([]),
12730            json!(["diagnostics.findings"]),
12731            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12732        );
12733        // A run-only capability does not admit flows either.
12734        assert_hello_drops_flow_operations_with_warning(
12735            json!(["agent-run-scopes/v1"]),
12736            json!([]),
12737            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12738        );
12739    }
12740
12741    #[test]
12742    fn hello_malformed_flow_operations_is_dropped_with_warning() {
12743        assert_hello_drops_flow_operations_with_warning(
12744            json!(["flow-scopes/v1"]),
12745            json!(["reset", "reset"]),
12746            "flow_operations names 'reset' more than once",
12747        );
12748        assert_hello_drops_flow_operations_with_warning(
12749            json!(["flow-scopes/v1"]),
12750            json!(vec!["reset"; 257]),
12751            "flow_operations lists 257 names, more than 256",
12752        );
12753        assert_hello_drops_flow_operations_with_warning(
12754            json!(["flow-scopes/v1"]),
12755            json!("reset"),
12756            "flow_operations must be an array of operation names",
12757        );
12758    }
12759
12760    #[test]
12761    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
12762        let registry = Arc::new(Registry::default());
12763        let handler = ControlHandler::new(registry.clone());
12764        let descriptor = json!({
12765            "schema": "ckios.richui.manifest/1",
12766            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12767            "bytes": 128,
12768            "chunks": 2
12769        });
12770        let malformed = [
12771            ("schema", json!("")),
12772            ("schema", json!("a".repeat(65))),
12773            ("schema", json!("Bad schema")),
12774            (
12775                "manifest_digest",
12776                json!(format!("sha256:{}", "A".repeat(64))),
12777            ),
12778            ("manifest_digest", json!("sha256:abc")),
12779            ("bytes", json!(0)),
12780            ("bytes", json!(16_777_217)),
12781            ("bytes", json!("128")),
12782            ("chunks", json!(0)),
12783            ("chunks", json!(1025)),
12784            ("chunks", json!("2")),
12785        ];
12786        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
12787            let mut invalid = descriptor.clone();
12788            invalid[field] = invalid_value;
12789            let response = handler
12790                .handle_control(
12791                    ConnectionId::new(210 + index as u64),
12792                    hello_frame_with_rich_ui(
12793                        &format!("bad-rich-ui-{index}"),
12794                        PROTOCOL_VERSION,
12795                        210 + index as u64,
12796                        None,
12797                        invalid,
12798                    ),
12799                )
12800                .unwrap();
12801            assert_eq!(response[0].header.ty, FrameType::Error);
12802            let error = parse_error(&response[0]);
12803            assert_eq!(error["code"], "invalid_manifest");
12804            assert!(error["message"]
12805                .as_str()
12806                .unwrap()
12807                .contains(&format!("rich_ui.{field}")));
12808            assert!(registry
12809                .get_module(&format!("bad-rich-ui-{index}"))
12810                .unwrap()
12811                .is_none());
12812        }
12813    }
12814
12815    #[test]
12816    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
12817        let registry = Arc::new(Registry::default());
12818        let handler = ControlHandler::new(Arc::clone(&registry));
12819        let connection = ConnectionId::new(219);
12820        registry
12821            .register_with_control_ops(
12822                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
12823                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
12824                        "bad schema",
12825                        format!("sha256:{}", "a".repeat(64)),
12826                        128,
12827                        2,
12828                    )))
12829                    .build(),
12830                PROTOCOL_VERSION,
12831                connection,
12832                Vec::new(),
12833            )
12834            .unwrap();
12835        let request =
12836            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
12837        let response = handler
12838            .handle_catalog_update(connection, request, Vec::new(), None, None)
12839            .unwrap();
12840        assert_eq!(response[0].header.ty, FrameType::Error);
12841        let error = parse_error(&response[0]);
12842        assert_eq!(error["code"], "invalid_manifest");
12843        assert!(error["message"]
12844            .as_str()
12845            .unwrap()
12846            .contains("rich_ui.schema"));
12847    }
12848
12849    #[test]
12850    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
12851        let registry = Arc::new(Registry::default());
12852        let supervisor = SupervisorHandle::new();
12853        supervisor.open_swap("provider", "candidate-nonce".to_string());
12854        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
12855        let connection = ConnectionId::new(218);
12856        let hello = handler
12857            .handle_control(
12858                connection,
12859                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
12860            )
12861            .unwrap();
12862        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12863        assert!(registry.list_modules().unwrap().1.is_empty());
12864
12865        registry.promote_candidate("provider").unwrap().unwrap();
12866        let request = Frame::build(
12867            FrameType::Request,
12868            control_flags(),
12869            0,
12870            0,
12871            219,
12872            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12873        )
12874        .unwrap();
12875        let response = handler.handle_catalog_list(request, None).unwrap();
12876        let ClientControlResponse::CatalogList { modules, .. } =
12877            serde_json::from_slice(&response[0].body).unwrap()
12878        else {
12879            panic!("expected catalog.list");
12880        };
12881        assert_eq!(modules.len(), 1);
12882        assert!(modules[0].nonce_verified);
12883    }
12884
12885    #[test]
12886    fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
12887        for provenance_json in [
12888            r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
12889            r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
12890        ] {
12891            let registry = Arc::new(Registry::default());
12892            let handler = ControlHandler::new(registry);
12893            let hello = Frame::build(
12894                FrameType::Hello,
12895                control_flags(),
12896                0,
12897                0,
12898                220,
12899                serde_json::to_vec(&ModuleHelloBody {
12900                    manifest: ModuleManifest::builder("built", "0.1.0")
12901                        .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
12902                        .build(),
12903                    protocol_ver: PROTOCOL_VERSION,
12904                    control_ops: None,
12905                    launch_nonce: None,
12906                })
12907                .unwrap(),
12908            )
12909            .unwrap();
12910            assert_eq!(
12911                handler
12912                    .handle_control(ConnectionId::new(220), hello)
12913                    .unwrap()[0]
12914                    .header
12915                    .ty,
12916                FrameType::HelloAck
12917            );
12918            let request = Frame::build(
12919                FrameType::Request,
12920                control_flags(),
12921                0,
12922                0,
12923                221,
12924                serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12925            )
12926            .unwrap();
12927            let response = handler.handle_catalog_list(request, None).unwrap();
12928            let ClientControlResponse::CatalogList { modules, .. } =
12929                serde_json::from_slice(&response[0].body).unwrap()
12930            else {
12931                panic!("expected catalog.list");
12932            };
12933            assert_eq!(modules.len(), 1);
12934            assert_eq!(
12935                serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
12936                format!(
12937                    r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
12938                )
12939                .as_bytes()
12940            );
12941        }
12942    }
12943
12944    #[test]
12945    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
12946        let registry = Arc::new(Registry::default());
12947        let handler = ControlHandler::new(Arc::clone(&registry));
12948        let hello = Frame::build(
12949            FrameType::Hello,
12950            control_flags(),
12951            0,
12952            0,
12953            220,
12954            serde_json::to_vec(&ModuleHelloBody {
12955                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
12956                protocol_ver: PROTOCOL_VERSION,
12957                control_ops: None,
12958                launch_nonce: Some("unverified".to_string()),
12959            })
12960            .unwrap(),
12961        )
12962        .unwrap();
12963        assert_eq!(
12964            handler
12965                .handle_control(ConnectionId::new(220), hello)
12966                .unwrap()[0]
12967                .header
12968                .ty,
12969            FrameType::HelloAck
12970        );
12971        let request = Frame::build(
12972            FrameType::Request,
12973            control_flags(),
12974            0,
12975            0,
12976            221,
12977            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12978        )
12979        .unwrap();
12980        let response = handler.handle_catalog_list(request, None).unwrap();
12981        let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
12982        assert!(wire["modules"][0].get("provenance").is_none());
12983        let ClientControlResponse::CatalogList { modules, .. } =
12984            serde_json::from_slice(&response[0].body).unwrap()
12985        else {
12986            panic!("expected catalog.list");
12987        };
12988        assert_eq!(
12989            serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
12990            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
12991        );
12992        assert!(!modules[0].nonce_verified);
12993        assert!(modules[0].provenance.is_none());
12994    }
12995
12996    // Preserve the legacy-field byte pins while discovery adds optional facts.
12997    fn catalog_legacy_projection(mut entry: CatalogEntry) -> CatalogEntry {
12998        assert_eq!(entry.first_party, Some(false));
12999        entry.incarnation = None;
13000        entry.spawn = None;
13001        entry.first_party = None;
13002        entry.origin = None;
13003        entry.requested_declarations_digest = None;
13004        entry.approved_declarations_digest = None;
13005        entry
13006    }
13007
13008    #[test]
13009    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
13010        let registry = Arc::new(Registry::default());
13011        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
13012            [("vault".to_string(), true), ("squatter".to_string(), true)],
13013            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13014        );
13015        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
13016        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13017            provides: vec!["credentials-provider/v1".to_string()],
13018            requires: Vec::new(),
13019            must_never_reach: Vec::new(),
13020        });
13021        let frame = Frame::build(
13022            FrameType::Hello,
13023            control_flags(),
13024            0,
13025            0,
13026            77,
13027            serde_json::to_vec(&ModuleHelloBody {
13028                manifest: squatter,
13029                protocol_ver: PROTOCOL_VERSION,
13030                control_ops: None,
13031                launch_nonce: None,
13032            })
13033            .expect("HELLO serializes"),
13034        )
13035        .expect("HELLO frame builds");
13036        let response = handler
13037            .handle_control(ConnectionId::new(77), frame)
13038            .expect("reserved claim receives a typed refusal");
13039        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13040        assert_eq!(
13041            registry
13042                .active_registration_count()
13043                .expect("registry reads"),
13044            0,
13045            "a reserved capability refusal must not leave a catalog entry"
13046        );
13047    }
13048
13049    #[test]
13050    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
13051        for settlement in ["timeout", "inconclusive", "drop"] {
13052            let breakers = RouteBindBreakers::default();
13053            let RouteBindAdmission::Admitted {
13054                guard: mut old,
13055                probe: false,
13056            } = breakers.admit("prov")
13057            else {
13058                panic!("ordinary relay admitted")
13059            };
13060            let RouteBindAdmission::Admitted {
13061                guard: mut opener, ..
13062            } = breakers.admit("prov")
13063            else {
13064                panic!("second relay admitted")
13065            };
13066            assert!(
13067                !opener
13068                    .record_timeout(1, Duration::ZERO)
13069                    .unwrap()
13070                    .reopened_after_probe
13071            );
13072            let RouteBindAdmission::Admitted {
13073                guard: mut probe,
13074                probe: true,
13075            } = breakers.admit("prov")
13076            else {
13077                panic!("one cooldown probe admitted")
13078            };
13079            match settlement {
13080                "timeout" => assert!(
13081                    !old.record_timeout(1, Duration::ZERO)
13082                        .unwrap()
13083                        .reopened_after_probe
13084                ),
13085                "inconclusive" => old.record_inconclusive(),
13086                "drop" => drop(old),
13087                _ => unreachable!(),
13088            }
13089            assert!(
13090                matches!(
13091                    breakers.admit("prov"),
13092                    RouteBindAdmission::Refused {
13093                        probe_in_flight: true,
13094                        ..
13095                    }
13096                ),
13097                "{settlement} of a pre-open relay cannot release the real probe"
13098            );
13099            assert!(
13100                probe
13101                    .record_timeout(1, Duration::ZERO)
13102                    .unwrap()
13103                    .reopened_after_probe
13104            );
13105            assert!(matches!(
13106                breakers.admit("prov"),
13107                RouteBindAdmission::Admitted { probe: true, .. }
13108            ));
13109        }
13110        let breakers = RouteBindBreakers::default();
13111        let admit = || match breakers.admit("prov") {
13112            RouteBindAdmission::Admitted { guard, .. } => guard,
13113            _ => panic!("relay admitted"),
13114        };
13115        admit().record_timeout(1, Duration::ZERO);
13116        let mut old_probe = admit();
13117        breakers.reset_for_new_module_connection("prov");
13118        admit().record_timeout(1, Duration::ZERO);
13119        let _new_probe = admit();
13120        old_probe.record_inconclusive();
13121        assert!(matches!(
13122            breakers.admit("prov"),
13123            RouteBindAdmission::Refused {
13124                probe_in_flight: true,
13125                ..
13126            }
13127        ));
13128    }
13129
13130    #[tokio::test]
13131    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
13132        for candidate in [false, true] {
13133            let registry = Arc::new(Registry::default());
13134            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
13135                [("vault".to_string(), true), ("squatter".to_string(), true)],
13136                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13137            );
13138            let conn = ConnectionId::new(77);
13139            let (ctx, mut rx) = route_ctx(conn);
13140            let initial = capability_manifest("squatter", &[], &[]);
13141            if candidate {
13142                registry
13143                    .register_candidate_with_control_ops(
13144                        initial.clone(),
13145                        PROTOCOL_VERSION,
13146                        conn,
13147                        module_baseline_control_ops(),
13148                    )
13149                    .unwrap();
13150                handler
13151                    .forwarding
13152                    .register_candidate_module_connection(
13153                        conn,
13154                        "squatter".to_string(),
13155                        PROTOCOL_VERSION,
13156                        manifest_concurrency(&initial),
13157                        ctx.egress.clone(),
13158                    )
13159                    .unwrap();
13160            } else {
13161                hello_via_sink(
13162                    &handler,
13163                    &ctx,
13164                    &mut rx,
13165                    hello_frame_with_manifest(initial.clone(), 1),
13166                )
13167                .await;
13168            }
13169            let response = handler
13170                .handle_control_frame(
13171                    &ctx,
13172                    catalog_update_with_capabilities_frame(
13173                        2,
13174                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
13175                            .capabilities
13176                            .unwrap(),
13177                    ),
13178                )
13179                .await
13180                .unwrap();
13181            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13182            assert_eq!(
13183                registry
13184                    .get_module_by_connection(conn)
13185                    .unwrap()
13186                    .unwrap()
13187                    .manifest,
13188                initial
13189            );
13190        }
13191    }
13192
13193    #[test]
13194    #[cfg(not(feature = "dev-desktop-approval"))]
13195    fn server_describe_default_build_advertises_no_config_capability() {
13196        let assert_default = |handler: &ControlHandler| {
13197            let request = Frame::build(
13198                FrameType::Request,
13199                control_flags(),
13200                0,
13201                0,
13202                79,
13203                serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13204            )
13205            .unwrap();
13206            let responses = handler.handle_server_describe(request).unwrap();
13207            assert_eq!(responses.len(), 1);
13208            let response: Value = serde_json::from_slice(&responses[0].body).unwrap();
13209            assert_eq!(response["op"], "server.describe");
13210            let capabilities = response["capabilities"]
13211                .as_array()
13212                .expect("capability array");
13213            assert!(
13214                !capabilities.is_empty(),
13215                "exercise an actual describe response, not an empty fixture"
13216            );
13217            assert!(
13218                capabilities
13219                    .iter()
13220                    .all(|capability| !capability.as_str().unwrap().starts_with("config.")),
13221                "a default build must not announce configuration authority: {capabilities:?}"
13222            );
13223        };
13224        assert_default(&ControlHandler::new(Arc::new(Registry::default())));
13225        #[cfg(unix)]
13226        {
13227            // A local store does not prove that project-membership validation,
13228            // active-scope usage tracking and approved-program launch selection
13229            // are connected. Defaults must not advertise that authority.
13230            let (_rig, _owner, _supervisor, handler, _spec) =
13231                crate::config_gate::runtime::RuntimeOwner::unactivated_supervisor_fixture_for_test(
13232                );
13233            assert_default(&handler);
13234        }
13235    }
13236
13237    #[test]
13238    fn server_describe_surfaces_required_capability_verdict_fields() {
13239        let registry = Arc::new(Registry::default());
13240        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
13241            [
13242                ("consumer".to_string(), true),
13243                ("provider".to_string(), false),
13244            ],
13245            BTreeMap::new(),
13246        );
13247        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13248        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13249            provides: Vec::new(),
13250            requires: vec![subc_protocol::manifest::CapabilityRequirement {
13251                capability: "credentials-provider/v1".to_string(),
13252                need: subc_protocol::manifest::CapabilityNeed::Required,
13253            }],
13254            must_never_reach: Vec::new(),
13255        });
13256        let hello = Frame::build(
13257            FrameType::Hello,
13258            control_flags(),
13259            0,
13260            0,
13261            78,
13262            serde_json::to_vec(&ModuleHelloBody {
13263                manifest: consumer,
13264                protocol_ver: PROTOCOL_VERSION,
13265                control_ops: None,
13266                launch_nonce: None,
13267            })
13268            .expect("HELLO serializes"),
13269        )
13270        .expect("HELLO frame builds");
13271        handler
13272            .handle_control(ConnectionId::new(78), hello)
13273            .expect("consumer registers");
13274        let describe = Frame::build(
13275            FrameType::Request,
13276            control_flags(),
13277            0,
13278            0,
13279            79,
13280            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
13281                .expect("request serializes"),
13282        )
13283        .expect("describe frame builds");
13284        let response = handler
13285            .handle_server_describe(describe)
13286            .expect("server.describe succeeds");
13287        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
13288        let requirement = &rendered["capability_requirements"][0];
13289        assert_eq!(requirement["consumer"], "consumer");
13290        assert_eq!(requirement["verdict"], "never_provided");
13291        assert_eq!(requirement["episode_seq"], 1);
13292        assert_eq!(requirement["config_satisfiable"], false);
13293        assert_eq!(requirement["runtime_available"], false);
13294        assert!(requirement["detail"]
13295            .as_str()
13296            .expect("detail string")
13297            .contains("credentials-provider/v1"));
13298    }
13299
13300    /// Register a consumer that requires `project-identity/v1` and the provider
13301    /// of it, optionally mark daemon shutdown as begun, then deregister the
13302    /// provider as its connection closes. Returns the `capability.requirement`
13303    /// events that deregistration logged and the verdict the evaluator then
13304    /// holds for the requirement (the one `route.open` refusals read).
13305    fn deregister_required_provider(
13306        daemon_shutdown_begun: bool,
13307    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
13308        const CAPABILITY: &str = "project-identity/v1";
13309        let registry = Arc::new(Registry::default());
13310        let daemon_shutdown = DaemonShutdownFlag::default();
13311        let handler = ControlHandler::new(Arc::clone(&registry))
13312            .with_capability_config(
13313                [
13314                    ("consumer".to_string(), true),
13315                    ("provider".to_string(), true),
13316                ],
13317                BTreeMap::new(),
13318            )
13319            .with_daemon_shutdown_flag(daemon_shutdown.clone());
13320        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13321        consumer.capabilities = Some(CapabilityDeclarations {
13322            provides: Vec::new(),
13323            requires: vec![subc_protocol::manifest::CapabilityRequirement {
13324                capability: CAPABILITY.to_string(),
13325                need: subc_protocol::manifest::CapabilityNeed::Required,
13326            }],
13327            must_never_reach: Vec::new(),
13328        });
13329        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
13330        let provider_conn = ConnectionId::new(182);
13331        for (conn, manifest, corr) in [
13332            (ConnectionId::new(181), consumer, 181),
13333            (provider_conn, provider, 182),
13334        ] {
13335            let hello = handler
13336                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
13337                .expect("module registers");
13338            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13339        }
13340        assert_eq!(
13341            handler.capability_evaluator.verdict("consumer", CAPABILITY),
13342            Some(CapabilityVerdict::Provided),
13343            "the registered provider must satisfy the requirement first"
13344        );
13345        if daemon_shutdown_begun {
13346            daemon_shutdown.set_for_test();
13347        }
13348
13349        let capture = EventCapture::default();
13350        let _subscriber =
13351            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13352        handler
13353            .cleanup_connection(provider_conn)
13354            .expect("provider deregisters");
13355        let events = capture
13356            .events()
13357            .into_iter()
13358            .filter(|event| {
13359                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
13360            })
13361            .collect();
13362        (
13363            events,
13364            handler.capability_evaluator.verdict("consumer", CAPABILITY),
13365        )
13366    }
13367
13368    #[test]
13369    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
13370        let (events, verdict) = deregister_required_provider(true);
13371        assert_eq!(
13372            verdict,
13373            Some(CapabilityVerdict::NeverProvided),
13374            "shutdown must not change the verdict route.open refusals read"
13375        );
13376        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13377        let event = &events[0];
13378        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
13379        assert_eq!(
13380            event.fields.get("cause").map(String::as_str),
13381            Some("\"daemon_shutdown\""),
13382            "{event:?}"
13383        );
13384        assert_eq!(
13385            event.fields.get("verdict").map(String::as_str),
13386            Some("\"never_provided\""),
13387            "{event:?}"
13388        );
13389    }
13390
13391    #[test]
13392    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
13393        let (events, verdict) = deregister_required_provider(false);
13394        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
13395        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13396        let event = &events[0];
13397        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
13398        assert_eq!(event.fields.get("cause"), None, "{event:?}");
13399        assert_eq!(
13400            event.fields.get("verdict").map(String::as_str),
13401            Some("\"never_provided\""),
13402            "{event:?}"
13403        );
13404    }
13405
13406    #[test]
13407    fn catalog_list_omits_capabilities_for_legacy_manifest() {
13408        let registry = Arc::new(Registry::default());
13409        let handler = ControlHandler::new(Arc::clone(&registry));
13410        let hello = handler
13411            .handle_control(
13412                ConnectionId::new(101),
13413                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
13414            )
13415            .expect("legacy HELLO registers");
13416        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13417
13418        let request = Frame::build(
13419            FrameType::Request,
13420            control_flags(),
13421            0,
13422            0,
13423            102,
13424            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
13425                .expect("catalog request serializes"),
13426        )
13427        .expect("catalog request frame builds");
13428        let response = handler
13429            .handle_catalog_list(request, None)
13430            .expect("catalog list succeeds");
13431        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
13432        assert!(
13433            body["modules"][0].get("capabilities").is_none(),
13434            "legacy manifest must retain an absent capabilities field on catalog.list"
13435        );
13436    }
13437
13438    #[test]
13439    fn hello_ack_omits_storage_when_no_storage_config() {
13440        let registry = Arc::new(Registry::default());
13441        let handler = ControlHandler::new(Arc::clone(&registry));
13442        let responses = handler
13443            .handle_control(
13444                ConnectionId::new(1),
13445                hello_frame("aft", PROTOCOL_VERSION, 7),
13446            )
13447            .unwrap();
13448        let ack = parse_ack(&responses[0]);
13449        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
13450        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
13451    }
13452
13453    #[tokio::test]
13454    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
13455        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
13456        let registry = Arc::new(Registry::default());
13457        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
13458        let responses = handler
13459            .handle_control(
13460                ConnectionId::new(1),
13461                hello_frame("aft", PROTOCOL_VERSION, 7),
13462            )
13463            .unwrap();
13464        let ack = parse_ack(&responses[0]);
13465        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
13466
13467        let described = handler
13468            .handle_control_frame(
13469                &route_ctx(ConnectionId::new(2)).0,
13470                Frame::build(
13471                    FrameType::Request,
13472                    control_flags(),
13473                    0,
13474                    0,
13475                    9,
13476                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13477                )
13478                .unwrap(),
13479            )
13480            .await
13481            .unwrap();
13482        let ClientControlResponse::ServerDescribe { machine_id, .. } =
13483            serde_json::from_slice(&described[0].body).unwrap()
13484        else {
13485            panic!("server.describe answered with another shape");
13486        };
13487        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
13488    }
13489
13490    #[test]
13491    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
13492        // With a central sqlite storage policy, each registering module gets its
13493        // own resolved descriptor in HELLO_ACK, keyed by its module id.
13494        let registry = Arc::new(Registry::default());
13495        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
13496            crate::daemon_config::StorageConfig::Sqlite {
13497                data_home: std::path::PathBuf::from("/data"),
13498            },
13499        ));
13500
13501        let responses = handler
13502            .handle_control(
13503                ConnectionId::new(1),
13504                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
13505            )
13506            .unwrap();
13507        let ack = parse_ack(&responses[0]);
13508        assert_eq!(
13509            ack.storage,
13510            Some(serde_json::json!({
13511                "module_id": "alfonso-routing",
13512                "storage_namespace": "default",
13513                "isolation": { "kind": "module" },
13514                "backend": {
13515                    "backend": "sqlite",
13516                    "path": "/data/cortexkit/alfonso-routing/store.db"
13517                }
13518            })),
13519            "the delivered descriptor is the module's own sqlite store path"
13520        );
13521    }
13522
13523    #[test]
13524    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
13525        let registry = Arc::new(Registry::default());
13526        let handler = ControlHandler::new(Arc::clone(&registry));
13527        let conn = ConnectionId::new(1);
13528        let responses = handler
13529            .handle_control(
13530                conn,
13531                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13532            )
13533            .unwrap();
13534        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
13535        let registration = registry.get_module("aft").unwrap().unwrap();
13536        assert_eq!(registration.control_ops, module_baseline_control_ops());
13537
13538        let frame =
13539            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
13540        assert!(handler
13541            .guard_module_control_op(&frame, "aft", "route.bind")
13542            .unwrap()
13543            .is_none());
13544        let error = handler
13545            .guard_module_control_op(&frame, "aft", "test.synthetic")
13546            .unwrap()
13547            .expect("synthetic ungranted op should be rejected");
13548        assert_eq!(error.header.ty, FrameType::Error);
13549        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
13550    }
13551
13552    #[test]
13553    fn hello_control_ops_some_adds_optional_grants() {
13554        let registry = Arc::new(Registry::default());
13555        let handler = ControlHandler::new(Arc::clone(&registry));
13556        handler
13557            .handle_control(
13558                ConnectionId::new(1),
13559                hello_frame_with_control_ops(
13560                    "aft",
13561                    PROTOCOL_VERSION,
13562                    7,
13563                    Some(vec![
13564                        "future.synthetic".to_string(),
13565                        "route.bind".to_string(),
13566                    ]),
13567                ),
13568            )
13569            .unwrap();
13570        let registration = registry.get_module("aft").unwrap().unwrap();
13571        assert_eq!(
13572            registration.control_ops,
13573            vec![
13574                "route.bind".to_string(),
13575                "route.status".to_string(),
13576                "future.synthetic".to_string(),
13577            ]
13578        );
13579        let frame =
13580            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
13581        assert!(handler
13582            .guard_module_control_op(&frame, "aft", "future.synthetic")
13583            .unwrap()
13584            .is_none());
13585    }
13586
13587    #[tokio::test]
13588    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
13589        let registry = Arc::new(Registry::default());
13590        let forwarding = Arc::new(ForwardingTable::default());
13591        let handler =
13592            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13593        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
13594        hello_via_sink(
13595            &handler,
13596            &module_ctx,
13597            &mut module_rx,
13598            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13599        )
13600        .await;
13601
13602        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
13603        let responses = handler
13604            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
13605            .await
13606            .unwrap();
13607        assert_eq!(responses.len(), 1);
13608        assert_eq!(responses[0].header.ty, FrameType::Error);
13609        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
13610        assert!(module_rx.try_recv().is_err());
13611    }
13612
13613    #[tokio::test]
13614    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
13615        let registry = Arc::new(Registry::default());
13616        let forwarding = Arc::new(ForwardingTable::default());
13617        let handler =
13618            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13619        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
13620        hello_via_sink(
13621            &handler,
13622            &module_ctx,
13623            &mut module_rx,
13624            hello_frame_with_control_ops(
13625                "aft",
13626                PROTOCOL_VERSION,
13627                7,
13628                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13629            ),
13630        )
13631        .await;
13632
13633        let project_root = unique_project_root("demux");
13634        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
13635        let route_handler = handler.clone();
13636        let route_task = tokio::spawn(async move {
13637            route_handler
13638                .handle_control_frame(
13639                    &route_client_ctx,
13640                    route_open_frame(100, "aft", project_root),
13641                )
13642                .await
13643                .unwrap()
13644        });
13645        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13646            .await
13647            .unwrap()
13648            .unwrap();
13649        assert!(matches!(
13650            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
13651            ModuleControlRequest::RouteBind { .. }
13652        ));
13653
13654        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
13655        let health_handler = handler.clone();
13656        let health_task = tokio::spawn(async move {
13657            health_handler
13658                .handle_control_frame(
13659                    &health_client_ctx,
13660                    supervisor_health_probe_frame(101, "aft"),
13661                )
13662                .await
13663                .unwrap()
13664        });
13665        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13666            .await
13667            .unwrap()
13668            .unwrap();
13669        assert_eq!(
13670            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13671            ModuleControlRequest::HealthCheck {}
13672        );
13673
13674        handler
13675            .handle_control_frame(
13676                &module_ctx,
13677                health_response(health_frame.header.corr, HealthStatus::Degraded),
13678            )
13679            .await
13680            .unwrap();
13681        let health_response = health_task.await.unwrap();
13682        assert_eq!(health_response.len(), 1);
13683        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13684            ClientControlResponse::SupervisorHealthProbe {
13685                module_id,
13686                status,
13687                detail,
13688                metrics,
13689            } => {
13690                assert_eq!(module_id, "aft");
13691                assert_eq!(status, HealthStatus::Degraded);
13692                assert_eq!(detail.as_deref(), Some("warming"));
13693                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
13694            }
13695            other => panic!("unexpected health response: {other:?}"),
13696        }
13697
13698        handler
13699            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13700            .await
13701            .unwrap();
13702        let route_response = route_task.await.unwrap();
13703        assert!(route_response.is_empty());
13704        let published = route_client_rx.recv().await.unwrap();
13705        assert!(matches!(
13706            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13707            ClientControlResponse::RouteOpen { .. }
13708        ));
13709    }
13710
13711    /// Start one `route.open` on `client_connection` and return its still-running
13712    /// handler task together with the `route.bind` the module received for it.
13713    /// The handler blocks until the module answers, so it has to run as a task
13714    /// while the test drives the module side.
13715    async fn relay_route_open(
13716        handler: &ControlHandler,
13717        client_connection: ConnectionId,
13718        client_egress: &FrameSink,
13719        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13720        corr: u64,
13721        module_id: &str,
13722        project_root_label: &str,
13723    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
13724        let ctx = RouteCtx {
13725            connection_id: client_connection,
13726            egress: client_egress.clone(),
13727        };
13728        let handler = handler.clone();
13729        let project_root = unique_project_root(project_root_label);
13730        let module_id = module_id.to_string();
13731        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
13732        let task = tokio::spawn(async move {
13733            let _guard = tracing::dispatcher::set_default(&dispatch);
13734            handler
13735                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
13736                .await
13737                .unwrap()
13738        });
13739        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13740            .await
13741            .expect("module receives the relayed route.bind")
13742            .expect("module egress is open");
13743        (task, bind.frame)
13744    }
13745
13746    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
13747        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
13748            ModuleControlRequest::RouteBind {
13749                route_channel,
13750                epoch,
13751                ..
13752            } => (route_channel, epoch),
13753            other => panic!("expected a route.bind request, got {other:?}"),
13754        }
13755    }
13756
13757    fn published_route(frame: &Frame) -> (u16, u32) {
13758        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
13759            ClientControlResponse::RouteOpen {
13760                route_channel,
13761                route_epoch,
13762                ..
13763            } => (route_channel, route_epoch),
13764            other => panic!("expected a route.open response, got {other:?}"),
13765        }
13766    }
13767
13768    /// Reproduction of a production outage. A client had `route.open`s in
13769    /// flight to a module and was already marked closing -- its egress had refused a
13770    /// module frame, so the daemon asked its connection to end -- while its sink
13771    /// was still open. When the module acked those binds, the daemon refused to
13772    /// commit a route for a closing client, and that refusal was returned from
13773    /// the MODULE connection's frame handler, where a router error that has no
13774    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
13775    /// the supervisor correctly did not respawn a clean exit, and every seat lost
13776    /// its tools for hours -- one client's teardown took down a connection
13777    /// carrying ~170 other routes.
13778    ///
13779    /// The window is opened here by calling the production path that opens it
13780    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
13781    /// state that matters is "in `closing_connections`, sink still open, relay
13782    /// still pending", and it lasts only from the close request until the
13783    /// connection loop reacts to it; a socket-level test can flood a client into
13784    /// that escalation but cannot pin the module's ack inside the window. Closing
13785    /// the socket instead takes the other path entirely -- connection teardown
13786    /// removes the pending relay under the same lock, so the ack finds nothing.
13787    #[tokio::test]
13788    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
13789        let registry = Arc::new(Registry::default());
13790        let forwarding = Arc::new(ForwardingTable::default());
13791        let handler =
13792            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13793
13794        let module_connection = ConnectionId::new(30);
13795        let (module_ctx, mut module_rx) = route_ctx(module_connection);
13796        hello_via_sink(
13797            &handler,
13798            &module_ctx,
13799            &mut module_rx,
13800            hello_frame("aft", PROTOCOL_VERSION, 7),
13801        )
13802        .await;
13803
13804        let dying_client = ConnectionId::new(31);
13805        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
13806
13807        // A published route on the dying client. The escalation below only marks
13808        // a connection closing for a route it has already published.
13809        let (first_task, first_bind) = relay_route_open(
13810            &handler,
13811            dying_client,
13812            &dying_ctx.egress,
13813            &mut module_rx,
13814            100,
13815            "aft",
13816            "closing-first",
13817        )
13818        .await;
13819        handler
13820            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
13821            .await
13822            .unwrap();
13823        assert!(first_task.await.unwrap().is_empty());
13824        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
13825
13826        // A second route.open from the same client, relayed and awaiting its ack.
13827        let (second_task, second_bind) = relay_route_open(
13828            &handler,
13829            dying_client,
13830            &dying_ctx.egress,
13831            &mut module_rx,
13832            101,
13833            "aft",
13834            "closing-second",
13835        )
13836        .await;
13837        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
13838
13839        // The window: the client is closing, its sink is still open, and its
13840        // second bind is still pending.
13841        assert!(forwarding
13842            .escalate_client_delivery_failure(
13843                dying_client,
13844                first_channel,
13845                first_epoch,
13846                CloseReason::new(
13847                    "module_to_client_delivery_failed",
13848                    "client egress refused a module frame",
13849                ),
13850                crate::forwarding::UndeliveredFrame {
13851                    module_id: None,
13852                    sink: &dying_ctx.egress,
13853                },
13854            )
13855            .unwrap());
13856        assert!(!dying_ctx.egress.is_closed());
13857
13858        // The frame that used to end the module connection.
13859        let ack = handler
13860            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
13861            .await;
13862        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13863        if module_loop_error.is_some() {
13864            // What the server's connection loop does with a router error that has
13865            // no ERROR-frame translation: end the connection, which releases the
13866            // module's registration and every route on it.
13867            handler.cleanup_connection(module_connection).unwrap();
13868        }
13869        // Read the module's next frame before opening the co-tenant's route, so
13870        // the GOODBYE assertion below is about THIS ack and not about later
13871        // traffic. `None` means the module was told nothing.
13872        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13873            .await
13874            .ok()
13875            .flatten();
13876
13877        // 1. The module connection is still registered.
13878        assert!(
13879            registry
13880                .get_module_by_connection(module_connection)
13881                .unwrap()
13882                .is_some(),
13883            "one client's closing connection ended the shared module connection: \
13884             {module_loop_error:?}"
13885        );
13886        // ...and still serving: another client can open and use a route on it.
13887        let cotenant = ConnectionId::new(32);
13888        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
13889        let (cotenant_task, cotenant_bind) = relay_route_open(
13890            &handler,
13891            cotenant,
13892            &cotenant_ctx.egress,
13893            &mut module_rx,
13894            102,
13895            "aft",
13896            "closing-cotenant",
13897        )
13898        .await;
13899        handler
13900            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
13901            .await
13902            .unwrap();
13903        assert!(cotenant_task.await.unwrap().is_empty());
13904        let (cotenant_channel, cotenant_epoch) =
13905            published_route(&cotenant_rx.recv().await.unwrap());
13906        assert!(matches!(
13907            forwarding
13908                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
13909                .unwrap(),
13910            DataRoute::Client(DataRouteState::Bound(_))
13911        ));
13912
13913        // 2. The module was told to drop the binding it created for the route
13914        //    that will never be published.
13915        let goodbye = post_ack_module_frame
13916            .expect("module receives a GOODBYE for the abandoned route channel");
13917        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13918        assert_eq!(goodbye.header.channel, abandoned_channel);
13919        assert_eq!(goodbye.header.epoch, abandoned_epoch);
13920
13921        // 3. The dying client received nothing: no route was ever published to
13922        //    it. Its route.open is answered as unavailable, which the connection
13923        //    loop would write to a socket that is already going away.
13924        assert!(dying_rx.try_recv().is_err());
13925        let second_response = second_task.await.unwrap();
13926        assert_eq!(second_response.len(), 1);
13927        assert_eq!(
13928            parse_error(&second_response[0])["code"],
13929            "target_unavailable"
13930        );
13931    }
13932
13933    /// One `catalog.list` as a client sees it: the generation and the listed
13934    /// module ids with their readiness.
13935    async fn catalog_snapshot(
13936        handler: &ControlHandler,
13937        client_ctx: &RouteCtx,
13938        corr: u64,
13939    ) -> (u64, Vec<(String, bool)>) {
13940        let body =
13941            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
13942        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
13943        let reply = handler
13944            .handle_control_frame(client_ctx, request)
13945            .await
13946            .unwrap()
13947            .pop()
13948            .unwrap();
13949        let ClientControlResponse::CatalogList {
13950            generation,
13951            modules,
13952            ..
13953        } = serde_json::from_slice(&reply.body).unwrap()
13954        else {
13955            panic!("not a catalog.list reply");
13956        };
13957        (
13958            generation,
13959            modules
13960                .into_iter()
13961                .map(|entry| (entry.module_id, entry.ready))
13962                .collect(),
13963        )
13964    }
13965
13966    /// A handler whose HELLO for module "aft" on connection 40 is parked
13967    /// between reserving its registry entry and installing its forwarding
13968    /// endpoint. Returns the handler, the two-party barrier (wait once to know
13969    /// the HELLO is parked, once more to release it), the HELLO's thread, and
13970    /// the module's context and receiver.
13971    fn hello_held_before_forwarding() -> (
13972        ControlHandler,
13973        Arc<std::sync::Barrier>,
13974        std::thread::JoinHandle<Vec<Frame>>,
13975        RouteCtx,
13976        mpsc::Receiver<crate::router::OutboundFrame>,
13977    ) {
13978        let hold = Arc::new(std::sync::Barrier::new(2));
13979        let handler = ControlHandler::with_forwarding(
13980            Arc::new(Registry::default()),
13981            Arc::new(ForwardingTable::default()),
13982        )
13983        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
13984        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
13985        let hello = {
13986            let handler = handler.clone();
13987            let module_ctx = module_ctx.clone();
13988            std::thread::spawn(move || {
13989                handler
13990                    .handle_hello(
13991                        module_ctx.connection_id,
13992                        Some(module_ctx.egress.clone()),
13993                        hello_frame("aft", PROTOCOL_VERSION, 7),
13994                    )
13995                    .unwrap()
13996            })
13997        };
13998        (handler, hold, hello, module_ctx, module_rx)
13999    }
14000
14001    /// If a lookup can acquire the registry while publication is parked, let
14002    /// the lookup finish before releasing HELLO. Otherwise release HELLO only
14003    /// after the lookup has tried the lock. Both schedules avoid timing guesses
14004    /// and return the actual lookup result, including an unpublished snapshot
14005    /// when the publication lock is missing.
14006    fn lookup_during_hello_publication<T: Send + 'static>(
14007        registry: &Registry,
14008        hold: &std::sync::Barrier,
14009        lookup: impl FnOnce() -> T + Send + 'static,
14010    ) -> T {
14011        let attempted = registry.observe_next_lock_attempt();
14012        let reader = std::thread::spawn(lookup);
14013        if attempted.recv().unwrap() {
14014            let result = reader.join().unwrap();
14015            hold.wait();
14016            result
14017        } else {
14018            hold.wait();
14019            reader.join().unwrap()
14020        }
14021    }
14022
14023    #[test]
14024    fn catalog_read_after_hello_ack_waits_for_publication() {
14025        let registry = Arc::new(Registry::default());
14026        let hold = Arc::new(std::sync::Barrier::new(2));
14027        let mut handler = ControlHandler::with_forwarding(
14028            Arc::clone(&registry),
14029            Arc::new(ForwardingTable::default()),
14030        );
14031        handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
14032        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14033        let hello = {
14034            let handler = handler.clone();
14035            std::thread::spawn(move || {
14036                handler
14037                    .handle_hello(
14038                        module_ctx.connection_id,
14039                        Some(module_ctx.egress),
14040                        hello_frame("aft", PROTOCOL_VERSION, 7),
14041                    )
14042                    .unwrap()
14043            })
14044        };
14045        hold.wait();
14046        assert_eq!(
14047            module_rx.try_recv().unwrap().frame.header.ty,
14048            FrameType::HelloAck
14049        );
14050        let replies = lookup_during_hello_publication(&registry, &hold, move || {
14051            // A different connection can issue catalog.list as soon as the
14052            // module has read its ACK, even while HELLO has not returned.
14053            let request = Frame::build(
14054                FrameType::Request,
14055                control_flags(),
14056                0,
14057                0,
14058                300,
14059                serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
14060            )
14061            .unwrap();
14062            handler.handle_catalog_list(request, None).unwrap()
14063        });
14064        assert!(hello.join().unwrap().is_empty());
14065        let ClientControlResponse::CatalogList {
14066            generation,
14067            modules,
14068            ..
14069        } = serde_json::from_slice(&replies[0].body).unwrap()
14070        else {
14071            panic!("not a catalog.list reply");
14072        };
14073        assert_eq!(
14074            modules
14075                .iter()
14076                .map(|entry| entry.module_id.as_str())
14077                .collect::<Vec<_>>(),
14078            vec!["aft"]
14079        );
14080        assert_eq!(generation, 1);
14081    }
14082
14083    /// While a HELLO has reserved its registry entry but its forwarding
14084    /// endpoint is not installed yet, `route.open` to the module can only be
14085    /// refused. `catalog.list` must not list the module in that window: a
14086    /// client that opens a route as soon as it sees a module listed and ready
14087    /// would be refused for a module the daemon had just advertised. The
14088    /// refusal in the window stays the retryable `target_unavailable`, never
14089    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
14090    /// Once the HELLO completes, the module is listed and a route to it opens.
14091    #[tokio::test]
14092    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
14093        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
14094        let client = ConnectionId::new(41);
14095        let (client_ctx, mut client_rx) = route_ctx(client);
14096
14097        hold.wait();
14098        // Parked: the registry holds "aft", forwarding has no endpoint for it.
14099        assert!(handler.registry.get_module("aft").unwrap().is_some());
14100        assert!(!handler
14101            .forwarding
14102            .has_live_module_connection("aft")
14103            .unwrap());
14104        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
14105        assert!(
14106            listed.is_empty(),
14107            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
14108        );
14109        let refused = handler
14110            .handle_control_frame(
14111                &client_ctx,
14112                route_open_frame(301, "aft", unique_project_root("listed-window")),
14113            )
14114            .await
14115            .unwrap();
14116        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
14117        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
14118        let duplicate = handler
14119            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
14120            .await
14121            .unwrap();
14122        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14123
14124        hold.wait();
14125        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14126        let ack = module_rx.try_recv().unwrap().frame;
14127        assert_eq!(ack.header.ty, FrameType::HelloAck);
14128        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14129        assert_eq!(listed, vec![("aft".to_string(), true)]);
14130
14131        let (open_task, bind) = relay_route_open(
14132            &handler,
14133            client,
14134            &client_ctx.egress,
14135            &mut module_rx,
14136            303,
14137            "aft",
14138            "listed-after",
14139        )
14140        .await;
14141        assert!(handler
14142            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14143            .await
14144            .unwrap()
14145            .is_empty());
14146        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
14147        published_route(&client_rx.try_recv().unwrap().frame);
14148    }
14149
14150    /// A catalog watcher keys on the generation: when it changes, it re-reads
14151    /// the listing. So the generation must advance exactly when the module
14152    /// appears. If it advanced while the module was still hidden, a watcher
14153    /// would read the new generation without the module, and the module would
14154    /// later appear under that same generation with nothing telling the
14155    /// watcher to look again.
14156    #[tokio::test]
14157    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
14158        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
14159        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14160        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14161
14162        hold.wait();
14163        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
14164        assert!(listed.is_empty());
14165        assert_eq!(
14166            during, before,
14167            "the generation moved while the module was not listed"
14168        );
14169
14170        hold.wait();
14171        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14172        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14173        assert_eq!(listed, vec![("aft".to_string(), true)]);
14174        assert_eq!(
14175            after,
14176            before + 1,
14177            "the module appeared without exactly one generation bump"
14178        );
14179    }
14180
14181    /// A HELLO whose forwarding endpoint cannot be installed (here the
14182    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
14183    /// refused and leaves nothing behind: no registry entry, no catalog entry,
14184    /// and the catalog generation where it was.
14185    #[tokio::test]
14186    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
14187        let registry = Arc::new(Registry::default());
14188        let handler = ControlHandler::with_forwarding(
14189            Arc::clone(&registry),
14190            Arc::new(ForwardingTable::default()),
14191        );
14192        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14193        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14194        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14195        drop(module_rx);
14196
14197        let replies = handler
14198            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
14199            .await
14200            .unwrap();
14201        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
14202        assert!(registry.get_module("aft").unwrap().is_none());
14203        assert!(registry
14204            .get_module_by_connection(module_ctx.connection_id)
14205            .unwrap()
14206            .is_none());
14207        assert_eq!(
14208            catalog_snapshot(&handler, &client_ctx, 301).await,
14209            (before, Vec::new())
14210        );
14211    }
14212
14213    /// A handler whose `route.open` parks after queueing the route.bind and
14214    /// before marking it enqueued, with module "aft" registered on connection
14215    /// 40. Returns the two-party barrier that parks and releases it.
14216    async fn route_open_held_after_bind_send() -> (
14217        ControlHandler,
14218        Arc<tokio::sync::Barrier>,
14219        RouteCtx,
14220        mpsc::Receiver<crate::router::OutboundFrame>,
14221    ) {
14222        let hold = Arc::new(tokio::sync::Barrier::new(2));
14223        let handler = ControlHandler::with_forwarding(
14224            Arc::new(Registry::default()),
14225            Arc::new(ForwardingTable::default()),
14226        )
14227        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
14228        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14229        hello_via_sink(
14230            &handler,
14231            &module_ctx,
14232            &mut module_rx,
14233            hello_frame("aft", PROTOCOL_VERSION, 7),
14234        )
14235        .await;
14236        (handler, hold, module_ctx, module_rx)
14237    }
14238
14239    /// A fast module acks the route.bind, and the reader commits the route,
14240    /// before `route.open` has marked the relay enqueued. The daemon used to
14241    /// read the missing relay as an abandoned bind and send the module a
14242    /// GOODBYE for the route it had just committed: the client was told its
14243    /// route was open, the module unbound the channel, and the client's first
14244    /// request was dropped on the module side as unbound.
14245    #[tokio::test]
14246    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
14247        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14248        let client = ConnectionId::new(41);
14249        let (client_ctx, mut client_rx) = route_ctx(client);
14250
14251        let (open_task, bind) = relay_route_open(
14252            &handler,
14253            client,
14254            &client_ctx.egress,
14255            &mut module_rx,
14256            200,
14257            "aft",
14258            "acked-before-mark",
14259        )
14260        .await;
14261        let (module_channel, module_epoch) = route_bind_channel(&bind);
14262        // route.open is now parked between queueing the bind and marking it.
14263        hold.wait().await;
14264
14265        // The module answers inside that window, and the route commits.
14266        assert!(handler
14267            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14268            .await
14269            .unwrap()
14270            .is_empty());
14271        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
14272
14273        hold.wait().await;
14274        assert!(
14275            open_task.await.unwrap().is_empty(),
14276            "route.open answers accepted"
14277        );
14278
14279        // Every frame route.open could have sent is already queued, because it
14280        // has returned; the module must have been sent nothing.
14281        match module_rx.try_recv() {
14282            Err(mpsc::error::TryRecvError::Empty) => {}
14283            Ok(outbound) => panic!(
14284                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
14285                outbound.frame.header.ty,
14286                outbound.frame.header.channel,
14287                outbound.frame.header.epoch
14288            ),
14289            Err(err) => panic!("module egress closed: {err}"),
14290        }
14291
14292        // The client's first request on the route is the next thing the module
14293        // reads, on the binding it just created.
14294        let router = Router::with_control_handler(Arc::new(handler.clone()));
14295        let request = Frame::build(
14296            FrameType::Request,
14297            Flags::new(true, Priority::Interactive, false),
14298            client_channel,
14299            client_epoch,
14300            201,
14301            b"first request".to_vec(),
14302        )
14303        .unwrap();
14304        router
14305            .route_for_connection(&client_ctx, request)
14306            .await
14307            .unwrap();
14308        let delivered = module_rx.try_recv().unwrap().frame;
14309        assert_eq!(delivered.header.ty, FrameType::Request);
14310        assert_eq!(
14311            (delivered.header.channel, delivered.header.epoch),
14312            (module_channel, module_epoch)
14313        );
14314        assert_eq!(delivered.body, b"first request");
14315    }
14316
14317    /// The client goes away while `route.open` is still queueing the bind, so
14318    /// the relay is dropped with no module reply. The teardown cannot send the
14319    /// GOODBYE itself (it could reach the module ahead of the bind), so
14320    /// `route.open` must send it once the bind is queued; otherwise the module
14321    /// keeps a binding the daemon will never commit.
14322    #[tokio::test]
14323    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
14324        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14325        let client = ConnectionId::new(42);
14326        let (client_ctx, _client_rx) = route_ctx(client);
14327
14328        let (open_task, bind) = relay_route_open(
14329            &handler,
14330            client,
14331            &client_ctx.egress,
14332            &mut module_rx,
14333            300,
14334            "aft",
14335            "released-before-mark",
14336        )
14337        .await;
14338        let (module_channel, module_epoch) = route_bind_channel(&bind);
14339        hold.wait().await;
14340
14341        handler.cleanup_connection(client).unwrap();
14342        assert!(
14343            module_rx.try_recv().is_err(),
14344            "client teardown must not GOODBYE a bind route.open is still queueing"
14345        );
14346
14347        hold.wait().await;
14348        let response = open_task.await.unwrap();
14349        assert_eq!(response.len(), 1);
14350        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14351
14352        let goodbye = module_rx
14353            .try_recv()
14354            .expect("module is told to drop the binding the daemon abandoned")
14355            .frame;
14356        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14357        assert_eq!(
14358            (goodbye.header.channel, goodbye.header.epoch),
14359            (module_channel, module_epoch)
14360        );
14361        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14362    }
14363
14364    /// The module acks inside the same window, but the client's egress closed
14365    /// first, so the reader refuses to commit. The ack proves the module holds
14366    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
14367    /// send a second one.
14368    #[tokio::test]
14369    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
14370        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14371        let client = ConnectionId::new(43);
14372        let (client_ctx, client_rx) = route_ctx(client);
14373
14374        let (open_task, bind) = relay_route_open(
14375            &handler,
14376            client,
14377            &client_ctx.egress,
14378            &mut module_rx,
14379            400,
14380            "aft",
14381            "gone-client-before-mark",
14382        )
14383        .await;
14384        let (module_channel, module_epoch) = route_bind_channel(&bind);
14385        hold.wait().await;
14386
14387        drop(client_rx);
14388        assert!(client_ctx.egress.is_closed());
14389        handler
14390            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14391            .await
14392            .unwrap();
14393
14394        hold.wait().await;
14395        let response = open_task.await.unwrap();
14396        assert_eq!(response.len(), 1);
14397        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14398
14399        let goodbye = module_rx
14400            .try_recv()
14401            .expect("module is told to drop the binding it acked")
14402            .frame;
14403        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14404        assert_eq!(
14405            (goodbye.header.channel, goodbye.header.epoch),
14406            (module_channel, module_epoch)
14407        );
14408        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14409        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14410    }
14411
14412    /// The fence at the module-loop boundary, stated as its own contract: which
14413    /// forwarding failures are allowed to end the module connection that is being
14414    /// served. A `ConnectionClosing` naming some client is about that client, and
14415    /// a module connection is shared; the same error naming the module's own
14416    /// connection is about this connection and must stay fatal, as must failures
14417    /// that are about the forwarding table itself.
14418    #[test]
14419    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
14420        let handler = ControlHandler::default();
14421        let module_connection = ConnectionId::new(30);
14422        let client_connection = ConnectionId::new(31);
14423
14424        handler
14425            .refuse_to_end_module_connection_for_a_client(
14426                module_connection,
14427                77,
14428                ForwardingError::ConnectionClosing {
14429                    connection_id: client_connection,
14430                },
14431            )
14432            .expect("a closing client must never end the module connection");
14433
14434        assert!(matches!(
14435            handler.refuse_to_end_module_connection_for_a_client(
14436                module_connection,
14437                78,
14438                ForwardingError::ConnectionClosing {
14439                    connection_id: module_connection,
14440                },
14441            ),
14442            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
14443                connection_id
14444            })) if connection_id == module_connection
14445        ));
14446        assert!(matches!(
14447            handler.refuse_to_end_module_connection_for_a_client(
14448                module_connection,
14449                79,
14450                ForwardingError::Poisoned,
14451            ),
14452            Err(RouterError::Forwarding(ForwardingError::Poisoned))
14453        ));
14454        assert!(matches!(
14455            handler.refuse_to_end_module_connection_for_a_client(
14456                module_connection,
14457                80,
14458                ForwardingError::StaleModuleEndpoint,
14459            ),
14460            Err(RouterError::Forwarding(
14461                ForwardingError::StaleModuleEndpoint
14462            ))
14463        ));
14464    }
14465
14466    /// The spawn-attestation guard is what stops a connected module from claiming
14467    /// another module's identity and being stamped `Reserved` for it. Every other
14468    /// test that supplies a consumer_identity supplies a CORRECT one, because a
14469    /// correct one is what the rest of the flow needs -- so the guard's rejection
14470    /// branch was never the subject of an assertion, only its acceptance branch.
14471    ///
14472    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
14473    /// whole subc-core library suite green; only the forwarding integration tests
14474    /// notice, and they notice for unrelated reasons. This test exists so the
14475    /// refusal itself is asserted where the guard lives: it fails if the identity
14476    /// check stops refusing, which is the direction that matters, since a guard
14477    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
14478    #[tokio::test]
14479    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
14480        let registry = Arc::new(Registry::default());
14481        let forwarding = Arc::new(ForwardingTable::default());
14482        let supervisor = SupervisorHandle::new();
14483        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14484        let handler =
14485            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14486                .with_supervisor(supervisor);
14487
14488        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
14489        hello_via_sink(
14490            &handler,
14491            &target_ctx,
14492            &mut target_rx,
14493            hello_frame("target", PROTOCOL_VERSION, 1),
14494        )
14495        .await;
14496
14497        // A real supervised module id presenting the wrong nonce. This is the
14498        // impersonation case: the attacker knows a privileged module_id, which is
14499        // public, and guesses at the nonce, which is not.
14500        let wrong_nonce = handler
14501            .handle_control_frame(
14502                &route_ctx(ConnectionId::new(91)).0,
14503                route_open_frame_with_admission_facts(
14504                    20,
14505                    "target",
14506                    unique_project_root("admission-facts"),
14507                    Some(subc_control::ConsumerIdentity {
14508                        module_id: "fed".to_string(),
14509                        launch_nonce: "not-the-real-nonce".to_string(),
14510                    }),
14511                    None,
14512                ),
14513            )
14514            .await
14515            .unwrap();
14516        assert_eq!(
14517            parse_error(&wrong_nonce[0])["code"],
14518            "bad_consumer_identity",
14519            "a mismatched launch nonce must be refused, not stamped Reserved"
14520        );
14521
14522        // A module id the supervisor never spawned at all, so no nonce exists to
14523        // compare against. An implementation that treats "no record" as "nothing
14524        // to check" fails open here while passing the case above.
14525        let never_spawned = handler
14526            .handle_control_frame(
14527                &route_ctx(ConnectionId::new(92)).0,
14528                route_open_frame_with_admission_facts(
14529                    21,
14530                    "target",
14531                    unique_project_root("admission-facts"),
14532                    Some(subc_control::ConsumerIdentity {
14533                        module_id: "never-spawned".to_string(),
14534                        launch_nonce: "any-nonce".to_string(),
14535                    }),
14536                    None,
14537                ),
14538            )
14539            .await
14540            .unwrap();
14541        assert_eq!(
14542            parse_error(&never_spawned[0])["code"],
14543            "bad_consumer_identity",
14544            "an unspawned module_id must be refused rather than accepted for lack of a record"
14545        );
14546    }
14547
14548    /// The refusal test above proves the guard says NO. Nothing proved it can say
14549    /// YES, and the difference is not academic: replacing the whole authorization
14550    /// with `false` -- admitting no consumer identity at all, revoking Reserved
14551    /// standing for every supervised module in the fleet -- leaves 110 of the 111
14552    /// library tests GREEN. The one that notices does so by HANGING, because it
14553    /// waits for a bind that can no longer happen.
14554    ///
14555    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
14556    /// or flaky test, invites a RETRY rather than an investigation, and the retry
14557    /// hangs too and gets blamed on the runner. So a total revocation of the
14558    /// daemon's trust grant would have shipped behind a symptom nobody attributes
14559    /// to code.
14560    ///
14561    /// The bias is structural rather than accidental. A REFUSAL looks like a
14562    /// failure someone writes a test for; a GRANT looks like the happy path. Every
14563    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
14564    /// suite for that reason, and this one is the purest case in the daemon.
14565    ///
14566    /// This test asserts the EFFECT rather than the absence of an error: the module
14567    /// receives a RouteBind and it carries `Reserved` naming the attested module.
14568    /// A guard that admitted nobody would produce no bind at all; one that admitted
14569    /// everybody would stamp the wrong principal, which the refusal test catches.
14570    #[tokio::test]
14571    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
14572        let registry = Arc::new(Registry::default());
14573        let forwarding = Arc::new(ForwardingTable::default());
14574        let supervisor = SupervisorHandle::new();
14575        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14576        let handler =
14577            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14578                .with_supervisor(supervisor);
14579
14580        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
14581        hello_via_sink(
14582            &handler,
14583            &target_ctx,
14584            &mut target_rx,
14585            hello_frame("target", PROTOCOL_VERSION, 1),
14586        )
14587        .await;
14588
14589        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
14590        let route_handler = handler.clone();
14591        let route_task = tokio::spawn(async move {
14592            route_handler
14593                .handle_control_frame(
14594                    &client_ctx,
14595                    route_open_frame_with_admission_facts(
14596                        30,
14597                        "target",
14598                        unique_project_root("admission-facts"),
14599                        Some(subc_control::ConsumerIdentity {
14600                            module_id: "fed".to_string(),
14601                            launch_nonce: "fed-nonce".to_string(),
14602                        }),
14603                        None,
14604                    ),
14605                )
14606                .await
14607                .unwrap()
14608        });
14609
14610        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
14611        // the very mutation it exists to catch -- a guard that admits nobody -- no
14612        // bind is ever sent, so it HUNG rather than failing. That reproduces the
14613        // exact defect being fixed: a total revocation detected only as a stalled
14614        // suite, which reads as flakiness and invites a retry. An acceptance test
14615        // that waits for an effect must bound the wait, or a red becomes a hang.
14616        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14617            .await
14618            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
14619            .expect("module control channel closed before route.bind");
14620        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
14621        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14622            panic!("expected route.bind")
14623        };
14624        assert_eq!(
14625            principal,
14626            Some(Principal::Reserved {
14627                module_id: "fed".to_string()
14628            }),
14629            "a correctly attested consumer must be stamped Reserved for its own id"
14630        );
14631
14632        handler
14633            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
14634            .await
14635            .unwrap();
14636        assert!(route_task.await.unwrap().is_empty());
14637        assert!(
14638            matches!(
14639                serde_json::from_slice::<ClientControlResponse>(
14640                    &client_rx.recv().await.unwrap().body
14641                )
14642                .unwrap(),
14643                ClientControlResponse::RouteOpen { .. }
14644            ),
14645            "the route must actually open, not merely avoid an error"
14646        );
14647    }
14648
14649    #[tokio::test(start_paused = true)]
14650    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
14651        let registry = Arc::new(Registry::default());
14652        let forwarding = Arc::new(ForwardingTable::default());
14653        let supervisor = SupervisorHandle::new();
14654        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14655        let handler =
14656            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14657                .with_supervisor(supervisor);
14658
14659        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
14660        hello_via_sink(
14661            &handler,
14662            &target_ctx,
14663            &mut target_rx,
14664            hello_frame("target", PROTOCOL_VERSION, 1),
14665        )
14666        .await;
14667
14668        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
14669        let direct_handler = handler.clone();
14670        let direct_open = tokio::spawn(async move {
14671            direct_handler
14672                .handle_control_frame(
14673                    &direct_ctx,
14674                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
14675                )
14676                .await
14677                .unwrap()
14678        });
14679        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14680            .await
14681            .expect("no direct route.bind within 5s")
14682            .expect("target control channel closed before direct route.bind");
14683        handler
14684            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
14685            .await
14686            .unwrap();
14687        assert!(direct_open.await.unwrap().is_empty());
14688        let _ = direct_rx.recv().await.unwrap();
14689
14690        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
14691        let reserved_handler = handler.clone();
14692        let reserved_open = tokio::spawn(async move {
14693            reserved_handler
14694                .handle_control_frame(
14695                    &reserved_ctx,
14696                    route_open_frame_with_admission_facts(
14697                        3,
14698                        "target",
14699                        unique_project_root("admission-facts"),
14700                        Some(ConsumerIdentity {
14701                            module_id: "fed".to_string(),
14702                            launch_nonce: "fed-nonce".to_string(),
14703                        }),
14704                        None,
14705                    ),
14706                )
14707                .await
14708                .unwrap()
14709        });
14710        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14711            .await
14712            .expect("no reserved route.bind within 5s")
14713            .expect("target control channel closed before reserved route.bind");
14714        handler
14715            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
14716            .await
14717            .unwrap();
14718        assert!(reserved_open.await.unwrap().is_empty());
14719        let _ = reserved_rx.recv().await.unwrap();
14720
14721        forwarding
14722            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
14723            .unwrap();
14724        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
14725        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
14726            module_id: Some("target".to_string()),
14727        })
14728        .unwrap();
14729        let census_frame =
14730            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
14731        let response = handler
14732            .handle_control_frame(&census_ctx, census_frame)
14733            .await
14734            .unwrap()
14735            .pop()
14736            .unwrap();
14737        let actual: Value = serde_json::from_slice(&response.body).unwrap();
14738        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
14739        assert!(matches!(
14740            decoded,
14741            ClientControlResponse::SupervisorRoutes { .. }
14742        ));
14743        let routes = actual["modules"][0]["routes"].as_array().unwrap();
14744        assert_eq!(routes.len(), 2);
14745        assert!(routes.iter().all(|route| route["draining"] == true));
14746        // The census carries WHY: the reason the drain was begun with, in the
14747        // route.closing vocabulary, on every draining route this drain marked.
14748        assert!(
14749            routes.iter().all(|route| route["drain_reason"] == "reload"),
14750            "draining routes must name the drain's reason: {routes:?}"
14751        );
14752        assert!(routes.iter().any(|route| {
14753            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
14754        }));
14755        assert!(routes.iter().any(|route| {
14756            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
14757        }));
14758
14759        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
14760            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
14761        if std::env::var_os("UPDATE_GOLDEN").is_some() {
14762            std::fs::write(
14763                &golden_path,
14764                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
14765            )
14766            .unwrap();
14767        }
14768        let expected: Value =
14769            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
14770        assert_eq!(actual, expected);
14771    }
14772
14773    async fn query_live_roots(
14774        handler: &ControlHandler,
14775        module_ctx: &RouteCtx,
14776    ) -> ModuleControlResponseToModule {
14777        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
14778        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
14779        let response = handler
14780            .handle_control_frame(module_ctx, frame)
14781            .await
14782            .unwrap()
14783            .pop()
14784            .unwrap();
14785        serde_json::from_slice(&response.body).unwrap()
14786    }
14787
14788    #[tokio::test(start_paused = true)]
14789    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
14790        let registry = Arc::new(Registry::default());
14791        let forwarding = Arc::new(ForwardingTable::default());
14792        let handler = ControlHandler::with_forwarding(registry, forwarding);
14793        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
14794        hello_via_sink(
14795            &handler,
14796            &target_ctx,
14797            &mut target_rx,
14798            hello_frame("target", PROTOCOL_VERSION, 1),
14799        )
14800        .await;
14801        let root = unique_project_root("live-roots-known");
14802        let path = ProjectRootId::from_path_allowing_missing(root.path())
14803            .unwrap()
14804            .as_path()
14805            .to_path_buf();
14806        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
14807        let open_handler = handler.clone();
14808        let opened = tokio::spawn(async move {
14809            open_handler
14810                .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
14811                .await
14812                .unwrap()
14813        });
14814        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14815            .await
14816            .unwrap()
14817            .unwrap();
14818        handler
14819            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
14820            .await
14821            .unwrap();
14822        assert!(opened.await.unwrap().is_empty());
14823        let _ = client_rx.recv().await.unwrap();
14824
14825        let root = unique_project_root("live-roots-pending");
14826        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
14827            .unwrap()
14828            .as_path()
14829            .to_path_buf();
14830        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
14831        let open_handler = handler.clone();
14832        let pending = tokio::spawn(async move {
14833            open_handler
14834                .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
14835                .await
14836                .unwrap()
14837        });
14838        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14839            .await
14840            .unwrap()
14841            .unwrap();
14842        let actual = query_live_roots(&handler, &target_ctx).await;
14843        let ModuleControlResponseToModule::LiveRoots {
14844            roots,
14845            unknown_root_bindings,
14846            total_bindings,
14847        } = actual
14848        else {
14849            panic!("expected live roots")
14850        };
14851        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
14852        assert_eq!(unknown_root_bindings, 0);
14853        assert_eq!(
14854            roots.len(),
14855            2,
14856            "root-known arm must retain each canonical root"
14857        );
14858        assert_eq!(
14859            total_bindings,
14860            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
14861        );
14862        let counts = roots
14863            .iter()
14864            .map(|root| (root.project_root.clone(), root.bound, root.pending))
14865            .collect::<Vec<_>>();
14866        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
14867        expected.sort_by(|a, b| a.0.cmp(&b.0));
14868        assert_eq!(
14869            counts, expected,
14870            "roots must sort by path and count pending separately"
14871        );
14872        handler
14873            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
14874            .await
14875            .unwrap();
14876        assert!(pending.await.unwrap().is_empty());
14877    }
14878
14879    #[tokio::test(start_paused = true)]
14880    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
14881        let forwarding = Arc::new(ForwardingTable::default());
14882        let handler =
14883            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
14884        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
14885        hello_via_sink(
14886            &handler,
14887            &target_ctx,
14888            &mut target_rx,
14889            hello_frame("target", PROTOCOL_VERSION, 1),
14890        )
14891        .await;
14892        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
14893        let pending = forwarding
14894            .begin_route_bind_relay_for_test(
14895                client_ctx.connection_id,
14896                client_ctx.egress.clone(),
14897                2,
14898                "target",
14899            )
14900            .unwrap();
14901        forwarding
14902            .complete_pending_relay(
14903                target_ctx.connection_id,
14904                pending.corr,
14905                RouteBindRelayOutcome::Accepted,
14906            )
14907            .unwrap();
14908        let actual = query_live_roots(&handler, &target_ctx).await;
14909        let ModuleControlResponseToModule::LiveRoots {
14910            roots,
14911            unknown_root_bindings,
14912            total_bindings,
14913        } = actual
14914        else {
14915            panic!("expected live roots")
14916        };
14917        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
14918        assert_eq!(
14919            unknown_root_bindings, 1,
14920            "unknown-root arm must not read as no bindings"
14921        );
14922        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
14923        assert_eq!(
14924            total_bindings,
14925            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
14926        );
14927    }
14928
14929    /// A module reads HELLO_ACK as its first frame and exits on anything else,
14930    /// so the ack has to be on its outbound queue before the module is
14931    /// routable. The connection loop writes a handler's replies only after the
14932    /// handler returns; this test stops in exactly that gap, runs a real
14933    /// route.open from another connection, and only then writes whatever the
14934    /// HELLO handler returned, the way the loop would. If the ack were still a
14935    /// reply, the route.bind request would reach the module first.
14936    #[tokio::test(start_paused = true)]
14937    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
14938        let forwarding = Arc::new(ForwardingTable::default());
14939        let handler =
14940            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
14941        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
14942        let replies = handler
14943            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
14944            .await
14945            .unwrap();
14946        let queued_by_hello = module_rx.len();
14947
14948        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
14949        let open_handler = handler.clone();
14950        let open = tokio::spawn(async move {
14951            open_handler
14952                .handle_control_frame(
14953                    &client_ctx,
14954                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
14955                )
14956                .await
14957                .unwrap()
14958        });
14959        // Let the route.open run until its route.bind is on the module's queue.
14960        let mut spins = 0;
14961        while module_rx.len() == queued_by_hello {
14962            spins += 1;
14963            assert!(spins < 10_000, "route.open never queued a route.bind");
14964            tokio::task::yield_now().await;
14965        }
14966
14967        // Now the connection loop's half: write the HELLO handler's replies.
14968        for reply in replies {
14969            module_ctx.egress.send(reply).await.unwrap();
14970        }
14971
14972        let first = module_rx.recv().await.unwrap().frame;
14973        assert_eq!(
14974            first.header.ty,
14975            FrameType::HelloAck,
14976            "the first frame a registering module reads must be its HELLO_ACK"
14977        );
14978        assert_eq!(first.header.corr, 7);
14979        let second = module_rx.recv().await.unwrap().frame;
14980        assert_eq!(second.header.ty, FrameType::Request);
14981        assert!(
14982            matches!(
14983                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
14984                ModuleControlRequest::RouteBind { .. }
14985            ),
14986            "the route.bind follows the ack"
14987        );
14988        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
14989
14990        handler
14991            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
14992            .await
14993            .unwrap();
14994        assert!(open.await.unwrap().is_empty());
14995        let _ = client_rx.recv().await.unwrap();
14996    }
14997
14998    #[tokio::test(start_paused = true)]
14999    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
15000        let handler = ControlHandler::with_forwarding(
15001            Arc::new(Registry::default()),
15002            Arc::new(ForwardingTable::default()),
15003        );
15004        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
15005        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
15006        hello_via_sink(
15007            &handler,
15008            &first_ctx,
15009            &mut first_rx,
15010            hello_frame("first", PROTOCOL_VERSION, 1),
15011        )
15012        .await;
15013        hello_via_sink(
15014            &handler,
15015            &second_ctx,
15016            &mut second_rx,
15017            hello_frame("second", PROTOCOL_VERSION, 2),
15018        )
15019        .await;
15020        let root = unique_project_root("second-only");
15021        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
15022        let cloned = handler.clone();
15023        let open = tokio::spawn(async move {
15024            cloned
15025                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
15026                .await
15027                .unwrap()
15028        });
15029        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
15030            .await
15031            .unwrap()
15032            .unwrap();
15033        let first = query_live_roots(&handler, &first_ctx).await;
15034        let second = query_live_roots(&handler, &second_ctx).await;
15035        assert!(
15036            matches!(
15037                first,
15038                ModuleControlResponseToModule::LiveRoots {
15039                    total_bindings: 0,
15040                    ..
15041                }
15042            ),
15043            "cross-module scope must not expose another module's roots"
15044        );
15045        assert!(
15046            matches!(
15047                second,
15048                ModuleControlResponseToModule::LiveRoots {
15049                    total_bindings: 1,
15050                    ..
15051                }
15052            ),
15053            "second module must see its pending route"
15054        );
15055        handler
15056            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
15057            .await
15058            .unwrap();
15059        assert!(open.await.unwrap().is_empty());
15060    }
15061
15062    #[tokio::test(start_paused = true)]
15063    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
15064        let handler = ControlHandler::with_forwarding(
15065            Arc::new(Registry::default()),
15066            Arc::new(ForwardingTable::default()),
15067        );
15068        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
15069        hello_via_sink(
15070            &handler,
15071            &target_ctx,
15072            &mut target_rx,
15073            hello_frame("target", PROTOCOL_VERSION, 1),
15074        )
15075        .await;
15076        let actual = query_live_roots(&handler, &target_ctx).await;
15077        let ModuleControlResponseToModule::LiveRoots {
15078            roots,
15079            unknown_root_bindings,
15080            total_bindings,
15081        } = actual
15082        else {
15083            panic!("expected live roots")
15084        };
15085        assert!(roots.is_empty());
15086        assert_eq!(unknown_root_bindings, 0);
15087        assert_eq!(total_bindings, 0);
15088        assert_eq!(
15089            total_bindings,
15090            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15091        );
15092    }
15093
15094    /// Read the vendored fed corpus rather than hand-building a package.
15095    ///
15096    /// A hand-built object encodes what the test author believed the carrier
15097    /// emits. These vectors are what it actually emits, and one of them exists
15098    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
15099    /// ignores additive unknown fields at the traversal emit terminus."
15100    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
15101        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
15102            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
15103        let text = std::fs::read_to_string(&path)
15104            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
15105        let vectors: Vec<(String, Value)> = text
15106            .lines()
15107            .filter(|line| !line.trim().is_empty())
15108            .map(|line| {
15109                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
15110                let id = entry["corpus_id"]
15111                    .as_str()
15112                    .expect("every vector carries a corpus_id")
15113                    .to_string();
15114                (id, entry["package"].clone())
15115            })
15116            .collect();
15117        // Pin the count: a corpus that silently shrinks would take its coverage
15118        // with it, and a suite reading N-1 vectors reports the same clean pass
15119        // as one reading N.
15120        assert_eq!(
15121            vectors.len(),
15122            3,
15123            "vendored fed corpus changed size; re-sync from subc-federation"
15124        );
15125
15126        // Pin what makes the corpus DISCRIMINATING, not just present.
15127        //
15128        // The relay test below takes its expected value from the corpus, so the
15129        // corpus supplies the test's power to detect a lossy relay rather than
15130        // its correctness. A relay that dropped unrecognised fields would still
15131        // be caught -- but only by a package carrying fields it does not know.
15132        // Shrink every package to the handful of keys any implementation would
15133        // recognise and the test keeps passing over an input that can no longer
15134        // fail, which is the same clean green as a corpus that shrank away.
15135        //
15136        // So assert the precondition rather than duplicating the packages here:
15137        // at least one vector must carry a field beyond the small common set.
15138        // That is one claim to maintain instead of nine, and it fails loudly if
15139        // a re-sync ever flattens the corpus.
15140        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
15141        let richest = vectors
15142            .iter()
15143            .filter_map(|(_, package)| package.as_object())
15144            .map(|object| {
15145                object
15146                    .keys()
15147                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
15148                    .count()
15149            })
15150            .max()
15151            .unwrap_or(0);
15152        assert!(
15153            richest >= 2,
15154            "vendored corpus no longer carries a package with unmodelled fields, \
15155             so the relay test can no longer distinguish a verbatim relay from a lossy one"
15156        );
15157
15158        vectors
15159    }
15160
15161    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
15162    ///
15163    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
15164    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
15165    /// three-key object, so a relay that quietly dropped fields it did not
15166    /// recognise would satisfy it. These vectors carry nine keys including ones
15167    /// this crate has no type for, so a typed relay fails here and only here.
15168    #[tokio::test]
15169    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
15170        for (corpus_id, package) in fed_admission_facts_vectors() {
15171            let registry = Arc::new(Registry::default());
15172            let forwarding = Arc::new(ForwardingTable::default());
15173            let supervisor = SupervisorHandle::new();
15174            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15175            let handler =
15176                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15177                    .with_supervisor(supervisor)
15178                    .with_admission_facts_config(
15179                        Some("fed".to_string()),
15180                        Some(vec!["target".to_string()]),
15181                    );
15182
15183            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
15184            hello_via_sink(
15185                &handler,
15186                &target_ctx,
15187                &mut target_rx,
15188                hello_frame("target", PROTOCOL_VERSION, 1),
15189            )
15190            .await;
15191
15192            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
15193            let route_handler = handler.clone();
15194            let expected = package.clone();
15195            let route_task = tokio::spawn(async move {
15196                route_handler
15197                    .handle_control_frame(
15198                        &client_ctx,
15199                        route_open_frame_with_admission_facts(
15200                            20,
15201                            "target",
15202                            unique_project_root("admission-facts"),
15203                            Some(subc_control::ConsumerIdentity {
15204                                module_id: "fed".to_string(),
15205                                launch_nonce: "fed-nonce".to_string(),
15206                            }),
15207                            Some(package),
15208                        ),
15209                    )
15210                    .await
15211                    .unwrap()
15212            });
15213
15214            let bind_frame = target_rx.recv().await.unwrap();
15215            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15216            let ModuleControlRequest::RouteBind {
15217                admission_facts, ..
15218            } = bind
15219            else {
15220                panic!("{corpus_id}: expected route.bind")
15221            };
15222            assert_eq!(
15223                admission_facts,
15224                Some(expected),
15225                "{corpus_id}: relay must not add, drop or reshape any field"
15226            );
15227
15228            handler
15229                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15230                .await
15231                .unwrap();
15232            route_task.await.unwrap();
15233        }
15234    }
15235
15236    #[tokio::test]
15237    async fn route_bind_never_relays_the_openers_build_class() {
15238        let registry = Arc::new(Registry::default());
15239        let forwarding = Arc::new(ForwardingTable::default());
15240        let handler =
15241            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15242        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
15243        hello_via_sink(
15244            &handler,
15245            &target_ctx,
15246            &mut target_rx,
15247            hello_frame("target", PROTOCOL_VERSION, 1),
15248        )
15249        .await;
15250
15251        let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
15252        identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
15253        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15254            busy_refusals: None,
15255            target: RouteTarget::ToolProvider {
15256                module_id: "target".to_string(),
15257            },
15258            identity,
15259            consumer_identity: None,
15260            consumer_capabilities: None,
15261            role_versions: None,
15262            admission_facts: None,
15263            scope: None,
15264        })
15265        .unwrap();
15266        let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
15267
15268        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
15269        let route_handler = handler.clone();
15270        let route_task = tokio::spawn(async move {
15271            route_handler
15272                .handle_control_frame(&client_ctx, open)
15273                .await
15274                .unwrap()
15275        });
15276        let bind_frame = target_rx.recv().await.unwrap();
15277        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15278        let ModuleControlRequest::RouteBind { identity, .. } = bind else {
15279            panic!("expected route.bind")
15280        };
15281        assert_eq!(
15282            identity.build_class, None,
15283            "route.bind carried the build class"
15284        );
15285        assert!(
15286            !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
15287            "the route.bind bytes still name build_class"
15288        );
15289        handler
15290            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15291            .await
15292            .unwrap();
15293        route_task.await.unwrap();
15294    }
15295
15296    #[tokio::test]
15297    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
15298        let registry = Arc::new(Registry::default());
15299        let forwarding = Arc::new(ForwardingTable::default());
15300        let supervisor = SupervisorHandle::new();
15301        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15302        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
15303        let handler =
15304            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15305                .with_supervisor(supervisor)
15306                .with_admission_facts_config(
15307                    Some("fed".to_string()),
15308                    Some(vec!["target".to_string()]),
15309                );
15310
15311        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
15312        hello_via_sink(
15313            &handler,
15314            &target_ctx,
15315            &mut target_rx,
15316            hello_frame("target", PROTOCOL_VERSION, 1),
15317        )
15318        .await;
15319        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
15320        hello_via_sink(
15321            &handler,
15322            &other_ctx,
15323            &mut other_rx,
15324            hello_frame("other", PROTOCOL_VERSION, 2),
15325        )
15326        .await;
15327
15328        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
15329        let expected_facts = facts.clone();
15330        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
15331        let route_handler = handler.clone();
15332        let route_task = tokio::spawn(async move {
15333            route_handler
15334                .handle_control_frame(
15335                    &client_ctx,
15336                    route_open_frame_with_admission_facts(
15337                        10,
15338                        "target",
15339                        unique_project_root("admission-facts"),
15340                        Some(subc_control::ConsumerIdentity {
15341                            module_id: "fed".to_string(),
15342                            launch_nonce: "fed-nonce".to_string(),
15343                        }),
15344                        Some(facts.clone()),
15345                    ),
15346                )
15347                .await
15348                .unwrap()
15349        });
15350        let bind_frame = target_rx.recv().await.unwrap();
15351        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15352        let ModuleControlRequest::RouteBind {
15353            admission_facts, ..
15354        } = bind
15355        else {
15356            panic!("expected route.bind")
15357        };
15358        assert_eq!(admission_facts, Some(expected_facts));
15359        handler
15360            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15361            .await
15362            .unwrap();
15363        assert!(route_task.await.unwrap().is_empty());
15364        assert!(matches!(
15365            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
15366                .unwrap(),
15367            ClientControlResponse::RouteOpen { .. }
15368        ));
15369
15370        let direct = handler
15371            .handle_control_frame(
15372                &route_ctx(ConnectionId::new(73)).0,
15373                route_open_frame_with_admission_facts(
15374                    11,
15375                    "target",
15376                    unique_project_root("admission-facts"),
15377                    None,
15378                    Some(json!({"x": 1})),
15379                ),
15380            )
15381            .await
15382            .unwrap();
15383        assert_eq!(
15384            parse_error(&direct[0])["code"],
15385            "admission_facts_not_permitted"
15386        );
15387
15388        let different_reserved = handler
15389            .handle_control_frame(
15390                &route_ctx(ConnectionId::new(77)).0,
15391                route_open_frame_with_admission_facts(
15392                    15,
15393                    "target",
15394                    unique_project_root("admission-facts"),
15395                    Some(subc_control::ConsumerIdentity {
15396                        module_id: "other".to_string(),
15397                        launch_nonce: "other-nonce".to_string(),
15398                    }),
15399                    Some(json!({"x": 1})),
15400                ),
15401            )
15402            .await
15403            .unwrap();
15404        assert_eq!(
15405            parse_error(&different_reserved[0])["code"],
15406            "admission_facts_not_permitted"
15407        );
15408
15409        let other_target = handler
15410            .handle_control_frame(
15411                &route_ctx(ConnectionId::new(74)).0,
15412                route_open_frame_with_admission_facts(
15413                    12,
15414                    "other",
15415                    unique_project_root("admission-facts"),
15416                    Some(subc_control::ConsumerIdentity {
15417                        module_id: "fed".to_string(),
15418                        launch_nonce: "fed-nonce".to_string(),
15419                    }),
15420                    Some(json!({"x": 1})),
15421                ),
15422            )
15423            .await
15424            .unwrap();
15425        assert_eq!(
15426            parse_error(&other_target[0])["code"],
15427            "admission_facts_target_not_allowed"
15428        );
15429
15430        let nonexistent = handler
15431            .handle_control_frame(
15432                &route_ctx(ConnectionId::new(75)).0,
15433                route_open_frame_with_admission_facts(
15434                    13,
15435                    "missing",
15436                    unique_project_root("admission-facts"),
15437                    None,
15438                    Some(json!({"x": 1})),
15439                ),
15440            )
15441            .await
15442            .unwrap();
15443        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
15444
15445        let described = handler
15446            .handle_control_frame(
15447                &route_ctx(ConnectionId::new(76)).0,
15448                Frame::build(
15449                    FrameType::Request,
15450                    control_flags(),
15451                    0,
15452                    0,
15453                    14,
15454                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15455                )
15456                .unwrap(),
15457            )
15458            .await
15459            .unwrap();
15460        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15461            serde_json::from_slice(&described[0].body).unwrap()
15462        else {
15463            panic!("expected server.describe response")
15464        };
15465        assert!(capabilities
15466            .iter()
15467            .any(|cap| cap == "admission_facts_relay_v1"));
15468    }
15469
15470    #[tokio::test]
15471    async fn admission_facts_without_configured_carrier_are_rejected() {
15472        let registry = Arc::new(Registry::default());
15473        let forwarding = Arc::new(ForwardingTable::default());
15474        let handler = ControlHandler::with_forwarding(registry, forwarding);
15475        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
15476        hello_via_sink(
15477            &handler,
15478            &target_ctx,
15479            &mut target_rx,
15480            hello_frame("target", PROTOCOL_VERSION, 1),
15481        )
15482        .await;
15483
15484        let responses = handler
15485            .handle_control_frame(
15486                &route_ctx(ConnectionId::new(79)).0,
15487                route_open_frame_with_admission_facts(
15488                    16,
15489                    "target",
15490                    unique_project_root("admission-facts"),
15491                    None,
15492                    Some(json!({"x": 1})),
15493                ),
15494            )
15495            .await
15496            .unwrap();
15497        assert_eq!(
15498            parse_error(&responses[0])["code"],
15499            "admission_facts_not_permitted"
15500        );
15501    }
15502
15503    #[tokio::test]
15504    async fn route_open_relays_consumer_capabilities_verbatim() {
15505        let registry = Arc::new(Registry::default());
15506        let forwarding = Arc::new(ForwardingTable::default());
15507        let handler =
15508            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15509        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
15510        hello_via_sink(
15511            &handler,
15512            &module_ctx,
15513            &mut module_rx,
15514            hello_frame("aft", PROTOCOL_VERSION, 7),
15515        )
15516        .await;
15517
15518        let expected = vec!["elicitation".to_string(), "roots".to_string()];
15519        let expected_for_request = expected.clone();
15520        let project_root = unique_project_root("consumer-capabilities-present");
15521        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
15522        let route_handler = handler.clone();
15523        let route_task = tokio::spawn(async move {
15524            route_handler
15525                .handle_control_frame(
15526                    &client_ctx,
15527                    route_open_frame_with_consumer_capabilities(
15528                        401,
15529                        "aft",
15530                        project_root,
15531                        Some(expected_for_request),
15532                    ),
15533                )
15534                .await
15535                .unwrap()
15536        });
15537        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15538            .await
15539            .unwrap()
15540            .unwrap();
15541        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15542        let ModuleControlRequest::RouteBind {
15543            consumer_capabilities,
15544            ..
15545        } = bind
15546        else {
15547            panic!("expected route.bind request, got {bind:?}");
15548        };
15549        assert_eq!(consumer_capabilities, Some(expected.clone()));
15550
15551        handler
15552            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15553            .await
15554            .unwrap();
15555        let route_response = route_task.await.unwrap();
15556        assert!(route_response.is_empty());
15557        let published = client_rx.recv().await.unwrap();
15558        assert!(matches!(
15559            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15560            ClientControlResponse::RouteOpen { .. }
15561        ));
15562    }
15563
15564    #[tokio::test]
15565    async fn route_open_without_consumer_capabilities_relays_none() {
15566        let registry = Arc::new(Registry::default());
15567        let forwarding = Arc::new(ForwardingTable::default());
15568        let handler =
15569            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15570        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
15571        hello_via_sink(
15572            &handler,
15573            &module_ctx,
15574            &mut module_rx,
15575            hello_frame("aft", PROTOCOL_VERSION, 7),
15576        )
15577        .await;
15578
15579        let project_root = unique_project_root("consumer-capabilities-absent");
15580        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
15581        let route_handler = handler.clone();
15582        let route_task = tokio::spawn(async move {
15583            route_handler
15584                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
15585                .await
15586                .unwrap()
15587        });
15588        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15589            .await
15590            .unwrap()
15591            .unwrap();
15592        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15593        let ModuleControlRequest::RouteBind {
15594            consumer_capabilities,
15595            ..
15596        } = bind
15597        else {
15598            panic!("expected route.bind request, got {bind:?}");
15599        };
15600        assert_eq!(consumer_capabilities, None);
15601
15602        handler
15603            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15604            .await
15605            .unwrap();
15606        let route_response = route_task.await.unwrap();
15607        assert!(route_response.is_empty());
15608        let published = client_rx.recv().await.unwrap();
15609        assert!(matches!(
15610            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15611            ClientControlResponse::RouteOpen { .. }
15612        ));
15613    }
15614
15615    /// Opens a route to a freshly registered `aft` with `sent` as the
15616    /// route.open's role_versions, acks the bind, and returns the role_versions
15617    /// the module's bind carried.
15618    async fn bind_role_versions_for(
15619        sent: Option<BTreeMap<String, String>>,
15620        connection: u64,
15621    ) -> Option<BTreeMap<String, String>> {
15622        let registry = Arc::new(Registry::default());
15623        let forwarding = Arc::new(ForwardingTable::default());
15624        let handler =
15625            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15626        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
15627        hello_via_sink(
15628            &handler,
15629            &module_ctx,
15630            &mut module_rx,
15631            hello_frame("aft", PROTOCOL_VERSION, 7),
15632        )
15633        .await;
15634        let project_root = unique_project_root("role-versions");
15635        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
15636        let route_handler = handler.clone();
15637        let route_task = tokio::spawn(async move {
15638            route_handler
15639                .handle_control_frame(
15640                    &client_ctx,
15641                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
15642                )
15643                .await
15644                .unwrap()
15645        });
15646        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15647            .await
15648            .expect("a well-formed route.open reaches the module as a bind")
15649            .unwrap();
15650        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15651        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
15652            panic!("expected route.bind request, got {bind:?}");
15653        };
15654        handler
15655            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15656            .await
15657            .unwrap();
15658        assert!(route_task.await.unwrap().is_empty());
15659        let published = client_rx.recv().await.unwrap();
15660        assert!(matches!(
15661            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15662            ClientControlResponse::RouteOpen { .. }
15663        ));
15664        role_versions
15665    }
15666
15667    #[tokio::test]
15668    async fn route_open_relays_role_versions_verbatim() {
15669        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
15670        assert_eq!(
15671            bind_role_versions_for(Some(sent.clone()), 141).await,
15672            Some(sent)
15673        );
15674    }
15675
15676    /// An empty map declares nothing, so the provider sees no field rather
15677    /// than an empty object it would have to treat as a second "none".
15678    #[tokio::test]
15679    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
15680        assert_eq!(bind_role_versions_for(None, 143).await, None);
15681        assert_eq!(
15682            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
15683            None
15684        );
15685    }
15686
15687    #[tokio::test]
15688    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
15689        let registry = Arc::new(Registry::default());
15690        let forwarding = Arc::new(ForwardingTable::default());
15691        let handler =
15692            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15693        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
15694        hello_via_sink(
15695            &handler,
15696            &module_ctx,
15697            &mut module_rx,
15698            hello_frame("aft", PROTOCOL_VERSION, 7),
15699        )
15700        .await;
15701
15702        let nine: BTreeMap<String, String> = (0..9)
15703            .map(|index| (format!("role-{index}"), "v1".to_string()))
15704            .collect();
15705        for (label, malformed) in [
15706            (
15707                "invalid role name",
15708                role_versions(&[("Tool_Provider", "v1")]),
15709            ),
15710            ("invalid version", role_versions(&[("tool-provider", "v0")])),
15711            ("nine entries", nine),
15712        ] {
15713            // A refused open answers at once; one that reached the module
15714            // would wait for its bind ack and trip this timeout.
15715            let responses = tokio::time::timeout(
15716                Duration::from_secs(1),
15717                handler.handle_control_frame(
15718                    &route_ctx(ConnectionId::new(148)).0,
15719                    route_open_frame_with_role_versions(
15720                        404,
15721                        "aft",
15722                        unique_project_root("role-versions-malformed"),
15723                        Some(malformed),
15724                    ),
15725                ),
15726            )
15727            .await
15728            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
15729            .unwrap();
15730            assert_eq!(responses.len(), 1, "{label}");
15731            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
15732            let error = parse_error(&responses[0]);
15733            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
15734            assert_eq!(
15735                error["detail"]["field"], "role_versions",
15736                "{label}: {error}"
15737            );
15738            assert!(
15739                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
15740                "{label}: a malformed declaration is terminal"
15741            );
15742            assert!(
15743                module_rx.try_recv().is_err(),
15744                "{label}: the module must never see a bind"
15745            );
15746        }
15747    }
15748
15749    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
15750    /// consumer can tell this daemon forwards the field from one that would
15751    /// drop it.
15752    #[tokio::test]
15753    async fn route_role_versions_capability_is_advertised() {
15754        let handler = ControlHandler::new(Arc::new(Registry::default()));
15755        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
15756        let ack = hello_via_sink(
15757            &handler,
15758            &ctx,
15759            &mut rx,
15760            hello_frame("m", PROTOCOL_VERSION, 1),
15761        )
15762        .await;
15763        let ack = parse_ack(&ack);
15764        assert!(
15765            ack.subc_capabilities
15766                .iter()
15767                .any(|c| c == "route-role-versions/v1"),
15768            "{:?}",
15769            ack.subc_capabilities
15770        );
15771
15772        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15773        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15774        let reply = handler
15775            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
15776            .await
15777            .unwrap()
15778            .pop()
15779            .unwrap();
15780        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15781            serde_json::from_slice(&reply.body).unwrap()
15782        else {
15783            panic!("not a server.describe reply");
15784        };
15785        assert!(
15786            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
15787            "{capabilities:?}"
15788        );
15789    }
15790
15791    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
15792    /// catalog entry without `events` (the module declares none) from a daemon
15793    /// that never reports declared events.
15794    #[tokio::test]
15795    async fn catalog_events_capability_is_advertised() {
15796        let handler = ControlHandler::new(Arc::new(Registry::default()));
15797        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15798        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15799        let reply = handler
15800            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
15801            .await
15802            .unwrap()
15803            .pop()
15804            .unwrap();
15805        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15806            serde_json::from_slice(&reply.body).unwrap()
15807        else {
15808            panic!("not a server.describe reply");
15809        };
15810        assert!(
15811            capabilities
15812                .iter()
15813                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
15814            "{capabilities:?}"
15815        );
15816    }
15817
15818    #[tokio::test]
15819    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
15820        let handler = ControlHandler::new(Arc::new(Registry::default()));
15821        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
15822        let ack = hello_via_sink(
15823            &handler,
15824            &module,
15825            &mut module_rx,
15826            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
15827        )
15828        .await;
15829        let ack = parse_ack(&ack);
15830        assert!(
15831            ack.subc_capabilities
15832                .iter()
15833                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15834            "{:?}",
15835            ack.subc_capabilities
15836        );
15837        assert!(ack
15838            .subc_capabilities
15839            .iter()
15840            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15841
15842        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
15843        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15844        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15845        let reply = handler
15846            .handle_control_frame(&client, frame)
15847            .await
15848            .unwrap()
15849            .pop()
15850            .unwrap();
15851        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15852            serde_json::from_slice(&reply.body).unwrap()
15853        else {
15854            panic!("not a server.describe reply");
15855        };
15856        assert!(
15857            capabilities
15858                .iter()
15859                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15860            "{capabilities:?}"
15861        );
15862        assert!(capabilities
15863            .iter()
15864            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15865    }
15866
15867    #[tokio::test]
15868    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
15869        let handler = ControlHandler::new(Arc::new(Registry::default()));
15870        let (module, mut rx) = route_ctx(ConnectionId::new(156));
15871        let ack = hello_via_sink(
15872            &handler,
15873            &module,
15874            &mut rx,
15875            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
15876        )
15877        .await;
15878        assert!(parse_ack(&ack)
15879            .subc_capabilities
15880            .iter()
15881            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
15882        assert!(parse_ack(&ack)
15883            .subc_capabilities
15884            .iter()
15885            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
15886        let (client, _rx) = route_ctx(ConnectionId::new(157));
15887        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15888        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15889        let reply = handler
15890            .handle_control_frame(&client, frame)
15891            .await
15892            .unwrap()
15893            .pop()
15894            .unwrap();
15895        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15896            serde_json::from_slice(&reply.body).unwrap()
15897        else {
15898            panic!("server.describe reply")
15899        };
15900        assert!(capabilities
15901            .iter()
15902            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
15903        assert!(capabilities
15904            .iter()
15905            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
15906    }
15907
15908    #[tokio::test]
15909    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
15910        let handler = ControlHandler::new(Arc::new(Registry::default()));
15911        let (module, mut rx) = route_ctx(ConnectionId::new(158));
15912        let ack = hello_via_sink(
15913            &handler,
15914            &module,
15915            &mut rx,
15916            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
15917        )
15918        .await;
15919        assert!(parse_ack(&ack)
15920            .subc_capabilities
15921            .iter()
15922            .any(|cap| cap == "scope-workdir/v1"));
15923        let (client, _rx) = route_ctx(ConnectionId::new(159));
15924        let frame = Frame::build(
15925            FrameType::Request,
15926            control_flags(),
15927            0,
15928            0,
15929            5,
15930            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15931        )
15932        .unwrap();
15933        let reply = handler
15934            .handle_control_frame(&client, frame)
15935            .await
15936            .unwrap()
15937            .remove(0);
15938        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15939            serde_json::from_slice(&reply.body).unwrap()
15940        else {
15941            panic!("server.describe reply")
15942        };
15943        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
15944    }
15945
15946    #[tokio::test]
15947    async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
15948        let handler = ControlHandler::new(Arc::new(Registry::default()));
15949        let (module, mut rx) = route_ctx(ConnectionId::new(160));
15950        let ack = hello_via_sink(
15951            &handler,
15952            &module,
15953            &mut rx,
15954            hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
15955        )
15956        .await;
15957        assert!(parse_ack(&ack)
15958            .subc_capabilities
15959            .iter()
15960            .any(|cap| cap == "worker-runs/v1"));
15961        let (client, _rx) = route_ctx(ConnectionId::new(161));
15962        let frame = Frame::build(
15963            FrameType::Request,
15964            control_flags(),
15965            0,
15966            0,
15967            5,
15968            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15969        )
15970        .unwrap();
15971        let reply = handler
15972            .handle_control_frame(&client, frame)
15973            .await
15974            .unwrap()
15975            .remove(0);
15976        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15977            serde_json::from_slice(&reply.body).unwrap()
15978        else {
15979            panic!("server.describe reply")
15980        };
15981        assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
15982    }
15983
15984    #[tokio::test]
15985    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
15986        let handler = ControlHandler::new(Arc::new(Registry::default()));
15987        let (module, mut rx) = route_ctx(ConnectionId::new(154));
15988        let ack = hello_via_sink(
15989            &handler,
15990            &module,
15991            &mut rx,
15992            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
15993        )
15994        .await;
15995        assert!(parse_ack(&ack)
15996            .subc_capabilities
15997            .iter()
15998            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
15999        let (client, _rx) = route_ctx(ConnectionId::new(155));
16000        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16001        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16002        let reply = handler
16003            .handle_control_frame(&client, frame)
16004            .await
16005            .unwrap()
16006            .pop()
16007            .unwrap();
16008        let ClientControlResponse::ServerDescribe { capabilities, .. } =
16009            serde_json::from_slice(&reply.body).unwrap()
16010        else {
16011            panic!("not a server.describe reply")
16012        };
16013        assert!(capabilities
16014            .iter()
16015            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16016    }
16017
16018    #[tokio::test]
16019    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
16020        let registry = Arc::new(Registry::default());
16021        let forwarding = Arc::new(ForwardingTable::default());
16022        let handler =
16023            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16024                .with_health_probe_timeout(Duration::from_secs(5));
16025        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
16026        hello_via_sink(
16027            &handler,
16028            &module_ctx,
16029            &mut module_rx,
16030            non_routable_hello_frame_with_control_ops(
16031                "mcp",
16032                300,
16033                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
16034            ),
16035        )
16036        .await;
16037        assert!(registry
16038            .get_module("mcp")
16039            .unwrap()
16040            .unwrap()
16041            .manifest
16042            .provides
16043            .is_empty());
16044
16045        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
16046        let route_response = handler
16047            .handle_control_frame(
16048                &route_client_ctx,
16049                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
16050            )
16051            .await
16052            .unwrap();
16053        assert_eq!(route_response[0].header.ty, FrameType::Error);
16054        assert_eq!(
16055            parse_error(&route_response[0])["code"],
16056            "target_unavailable"
16057        );
16058        assert!(parse_error(&route_response[0])["message"]
16059            .as_str()
16060            .unwrap()
16061            .contains("does not provide the requested target"));
16062        assert!(module_rx.try_recv().is_err());
16063
16064        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
16065        let health_handler = handler.clone();
16066        let health_task = tokio::spawn(async move {
16067            health_handler
16068                .handle_control_frame(
16069                    &health_client_ctx,
16070                    supervisor_health_probe_frame(302, "mcp"),
16071                )
16072                .await
16073                .unwrap()
16074        });
16075        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
16076            .await
16077            .unwrap()
16078            .unwrap();
16079        assert_eq!(
16080            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
16081            ModuleControlRequest::HealthCheck {}
16082        );
16083        handler
16084            .handle_control_frame(
16085                &module_ctx,
16086                health_response(health_frame.header.corr, HealthStatus::Ok),
16087            )
16088            .await
16089            .unwrap();
16090        let health_response = health_task.await.unwrap();
16091        assert_eq!(health_response[0].header.ty, FrameType::Response);
16092        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
16093            ClientControlResponse::SupervisorHealthProbe {
16094                module_id, status, ..
16095            } => {
16096                assert_eq!(module_id, "mcp");
16097                assert_eq!(status, HealthStatus::Ok);
16098            }
16099            other => panic!("unexpected health response: {other:?}"),
16100        }
16101
16102        // Exercise the forwarding cleanup path directly while leaving the registry
16103        // advertisement in place. If cleanup leaves a stale control sink behind,
16104        // the next probe will enqueue onto it and wait for the long probe timeout
16105        // instead of returning an immediate no-connection error.
16106        forwarding
16107            .cleanup_connection(module_ctx.connection_id)
16108            .unwrap();
16109        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
16110        let cleanup_response = tokio::time::timeout(
16111            Duration::from_millis(200),
16112            handler.handle_control_frame(
16113                &cleanup_probe_ctx,
16114                supervisor_health_probe_frame(303, "mcp"),
16115            ),
16116        )
16117        .await
16118        .expect("probe should fail immediately when the control lane is gone")
16119        .unwrap();
16120        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
16121        assert_eq!(
16122            parse_error(&cleanup_response[0])["code"],
16123            "target_unavailable"
16124        );
16125        assert!(parse_error(&cleanup_response[0])["message"]
16126            .as_str()
16127            .unwrap()
16128            .contains("no module connection"));
16129
16130        handler
16131            .cleanup_connection(module_ctx.connection_id)
16132            .unwrap();
16133    }
16134
16135    #[tokio::test]
16136    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
16137        let registry = Arc::new(Registry::default());
16138        let supervisor_handle = SupervisorHandle::new();
16139        let supervisor =
16140            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
16141                .with_handle(supervisor_handle.clone())
16142                .with_connection_file_path(
16143                    std::env::temp_dir()
16144                        .join(format!("subc-route-open-warming-{}", std::process::id())),
16145                );
16146        let module = supervisor
16147            .supervise_configured(
16148                ModuleSpec {
16149                    carry_containment: false,
16150                    module_id: "warming".to_string(),
16151                    program: fake_aft_stub_path(),
16152                    args: Vec::new(),
16153                    env: Vec::new(),
16154                    reserved: false,
16155                    reserved_prefixes: Vec::new(),
16156                    protocol: ModuleProtocol::Subc,
16157                    overlap: Default::default(),
16158                },
16159                true,
16160            )
16161            .unwrap();
16162        assert_eq!(module.state().unwrap(), ModuleState::Running);
16163
16164        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
16165        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
16166        let response = handler
16167            .handle_control_frame(
16168                &ctx,
16169                route_open_frame(304, "warming", unique_project_root("warming")),
16170            )
16171            .await
16172            .unwrap();
16173        module.stop().await.unwrap();
16174
16175        assert_eq!(response[0].header.ty, FrameType::Error);
16176        let error = parse_error(&response[0]);
16177        assert_eq!(error["code"], "module_warming");
16178        assert!(error["message"]
16179            .as_str()
16180            .unwrap()
16181            .contains("state=running, enabled=true, live=false"));
16182    }
16183
16184    #[test]
16185    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
16186        let handler = ControlHandler::new(Arc::new(Registry::default()));
16187        let capture = EventCapture::default();
16188        let _subscriber =
16189            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16190        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
16191        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
16192        let pending = (0..limit).collect::<Vec<_>>();
16193        let response = handler
16194            .route_open_capacity_refusal(
16195                &ctx,
16196                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
16197                "busy",
16198                pending.len(),
16199                limit,
16200            )
16201            .unwrap();
16202        assert_eq!(parse_error(&response)["code"], "target_unavailable");
16203        let event = capture
16204            .events()
16205            .into_iter()
16206            .find(|event| {
16207                event.target == "control"
16208                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
16209            })
16210            .expect("connection admission refusal event");
16211        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16212        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16213        // Refused before `handle_route_open` parses the request, so the caller
16214        // fields are read from the frame itself.
16215        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
16216        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
16217    }
16218
16219    #[test]
16220    fn route_open_target_cap_logs_admission_reason_and_capacity() {
16221        let handler = ControlHandler::new(Arc::new(Registry::default()));
16222        let capture = EventCapture::default();
16223        let _subscriber =
16224            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16225        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
16226        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
16227        let guards = (0..limit)
16228            .map(|_| {
16229                handler
16230                    .route_bind_concurrency
16231                    .try_admit("busy", limit)
16232                    .unwrap()
16233            })
16234            .collect::<Vec<_>>();
16235        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
16236            Err(in_flight) => in_flight,
16237            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
16238        };
16239        let response = handler
16240            .route_open_target_capacity_refusal(
16241                &ctx,
16242                &RouteOpenCaller::unknown(),
16243                &route_open_frame(397, "busy", unique_project_root("target-cap")),
16244                "busy",
16245                in_flight,
16246            )
16247            .unwrap();
16248        assert_eq!(parse_error(&response)["code"], "target_unavailable");
16249        let event = capture
16250            .events()
16251            .into_iter()
16252            .find(|event| {
16253                event.target == "control"
16254                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
16255            })
16256            .expect("target admission refusal event");
16257        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16258        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16259        drop(guards);
16260    }
16261
16262    #[cfg(unix)]
16263    #[tokio::test(start_paused = true)]
16264    async fn supervisor_list_shows_a_failed_modules_next_retry() {
16265        let registry = Arc::new(Registry::default());
16266        let handle = SupervisorHandle::new();
16267        let supervisor =
16268            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
16269                .with_handle(handle.clone());
16270        let module = supervisor
16271            .spawn(crate::ModuleSpec {
16272                carry_containment: false,
16273                module_id: "retry-list".into(),
16274                program: PathBuf::from("/bin/sh"),
16275                args: vec!["-c".into(), "exit 17".into()],
16276                env: vec![],
16277                reserved: false,
16278                reserved_prefixes: vec![],
16279                protocol: ModuleProtocol::None,
16280                overlap: Default::default(),
16281            })
16282            .unwrap();
16283        let deadline = std::time::Instant::now() + Duration::from_secs(10);
16284        while module.state().unwrap() != crate::ModuleState::Failed {
16285            assert!(
16286                std::time::Instant::now() < deadline,
16287                "waiting for budget failure"
16288            );
16289            tokio::task::yield_now().await;
16290        }
16291        let handler = ControlHandler::new(registry).with_supervisor(handle);
16292        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16293        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16294        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16295        assert_eq!(body["modules"][0]["state"], "failed");
16296        let retry = body["modules"][0]["next_retry_at_ms"]
16297            .as_u64()
16298            .expect("scheduled retry on the wire");
16299        let now = std::time::SystemTime::now()
16300            .duration_since(std::time::UNIX_EPOCH)
16301            .unwrap()
16302            .as_millis() as u64;
16303        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
16304        module.set_enabled(false).await.unwrap();
16305        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16306        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16307        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16308        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
16309    }
16310
16311    #[tokio::test]
16312    async fn slow_step_before_pid_read_does_not_spend_its_budget() {
16313        let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
16314        let supervisor =
16315            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
16316        let module = supervisor
16317            .spawn(crate::ModuleSpec {
16318                carry_containment: false,
16319                module_id: "pid-read-budget".into(),
16320                program: std::env::current_exe().unwrap(),
16321                args: vec![
16322                    "--exact".into(),
16323                    "control::tests::pid_resource_read_fixture".into(),
16324                ],
16325                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16326                    .into_iter()
16327                    .map(|key| (key.into(), dir.display().to_string()))
16328                    .chain(std::iter::once((
16329                        "SUBC_SUBTREE_PID_FIXTURE".into(),
16330                        "1".into(),
16331                    )))
16332                    .collect(),
16333                reserved: false,
16334                reserved_prefixes: Vec::new(),
16335                protocol: ModuleProtocol::None,
16336                overlap: Default::default(),
16337            })
16338            .unwrap();
16339        let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16340        while !matches!(
16341            module.child_resource_usage(),
16342            subc_control::ChildResourceUsage::Measured(_)
16343        ) {
16344            assert!(
16345                tokio::time::Instant::now() < ready_deadline,
16346                "fixture process was not measurable"
16347            );
16348            tokio::time::sleep(Duration::from_millis(10)).await;
16349        }
16350        tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
16351        let reading = {
16352            let _guard = crate::subtree_resources::fault_read_guard().await;
16353            crate::subtree_resources::read_process(module.clone()).await
16354        };
16355        stop_resource_fixture(&module, &dir).await;
16356        assert!(
16357            matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
16358            "{reading:?}"
16359        );
16360    }
16361
16362    #[test]
16363    fn pid_resource_read_fixture() {
16364        if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
16365            let dir = std::path::PathBuf::from(std::env::var_os("XDG_DATA_HOME").unwrap());
16366            let deadline = std::time::Instant::now() + Duration::from_secs(60);
16367            while !dir.join("stop-module").exists() && std::time::Instant::now() < deadline {
16368                std::thread::sleep(Duration::from_millis(5));
16369            }
16370            std::fs::write(dir.join("module-stopped"), "").unwrap();
16371        }
16372    }
16373
16374    async fn stop_resource_fixture(
16375        module: &crate::supervise::SupervisedModule,
16376        dir: &std::path::Path,
16377    ) {
16378        let stopping = module.stop();
16379        tokio::pin!(stopping);
16380        tokio::select! {
16381            result = &mut stopping => result.unwrap(),
16382            () = async {
16383                while module.state().unwrap() != crate::supervise::ModuleState::Draining {
16384                    tokio::time::sleep(Duration::from_millis(5)).await;
16385                }
16386                std::fs::write(dir.join("stop-module"), "").unwrap();
16387            } => stopping.await.unwrap(),
16388        }
16389        #[cfg(windows)]
16390        assert!(
16391            dir.join("module-stopped").exists(),
16392            "the resource fixture must exit cooperatively, not wait for the drain kill"
16393        );
16394        assert_eq!(
16395            module.state().unwrap(),
16396            crate::supervise::ModuleState::Stopped
16397        );
16398    }
16399
16400    #[tokio::test]
16401    async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
16402        let registry = Arc::new(Registry::default());
16403        let handle = SupervisorHandle::new();
16404        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16405            .with_handle(handle.clone());
16406        supervisor
16407            .supervise_configured(
16408                crate::ModuleSpec {
16409                    module_id: "disabled-resource-reading".into(),
16410                    program: std::env::current_exe().unwrap(),
16411                    args: Vec::new(),
16412                    env: Vec::new(),
16413                    reserved: false,
16414                    reserved_prefixes: Vec::new(),
16415                    protocol: ModuleProtocol::None,
16416                    overlap: Default::default(),
16417                    carry_containment: false,
16418                },
16419                false,
16420            )
16421            .unwrap();
16422        let handler = ControlHandler::new(registry).with_supervisor(handle);
16423        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16424        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16425        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16426        assert!(body["modules"][0].get("tree").is_none());
16427        assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
16428    }
16429
16430    #[tokio::test]
16431    async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
16432        let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
16433        let registry = Arc::new(Registry::default());
16434        let handle = SupervisorHandle::new();
16435        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16436            .with_handle(handle.clone());
16437        let module = supervisor
16438            .spawn(crate::ModuleSpec {
16439                carry_containment: false,
16440                module_id: "list-process-priority".into(),
16441                program: std::env::current_exe().unwrap(),
16442                args: vec![
16443                    "--exact".into(),
16444                    "control::tests::pid_resource_read_fixture".into(),
16445                ],
16446                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16447                    .into_iter()
16448                    .map(|key| (key.into(), dir.display().to_string()))
16449                    .chain(std::iter::once((
16450                        "SUBC_SUBTREE_PID_FIXTURE".into(),
16451                        "1".into(),
16452                    )))
16453                    .collect(),
16454                reserved: false,
16455                reserved_prefixes: Vec::new(),
16456                protocol: ModuleProtocol::None,
16457                overlap: Default::default(),
16458            })
16459            .unwrap();
16460        let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16461        loop {
16462            let image = module.running_image_agreement().await;
16463            if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
16464                break;
16465            }
16466            assert!(
16467                tokio::time::Instant::now() < ready_deadline,
16468                "the fixture process never became observable: {image:?}"
16469            );
16470            tokio::time::sleep(Duration::from_millis(10)).await;
16471        }
16472        let guard = crate::subtree_resources::fault_read_guard().await;
16473        let (target, release, _) = crate::subtree_resources::held();
16474        module.set_subtree_target_for_test(target);
16475        let handler = ControlHandler::new(registry).with_supervisor(handle);
16476        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16477        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16478        drop(release);
16479        drop(guard);
16480        stop_resource_fixture(&module, &dir).await;
16481        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16482        assert_eq!(
16483            body["modules"][0]["resources"]["status"], "measured",
16484            "the optional tree timeout must not discard the process reading: {body}"
16485        );
16486        assert!(body["modules"][0].get("tree").is_none());
16487        assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
16488        assert_eq!(
16489            body["modules"][0]["pending_reload"]["image"]["status"], "match",
16490            "the tree timeout must not hide a completed image observation"
16491        );
16492    }
16493
16494    #[cfg(any(target_os = "linux", target_os = "macos"))]
16495    #[tokio::test]
16496    async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
16497        let mut handler = ControlHandler::new(Arc::new(Registry::default()))
16498            .with_daemon_provenance(
16499                std::process::id(),
16500                0,
16501                Some(std::env::current_exe().unwrap()),
16502                None,
16503                None,
16504            );
16505        let (probe, release, began) = ExecutableIdentityProbe::held();
16506        handler.daemon_provenance.probe = probe;
16507        let observe = |handler: ControlHandler, corr| {
16508            tokio::spawn(async move {
16509                let (ctx, _) = route_ctx(ConnectionId::new(corr));
16510                let frame =
16511                    Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
16512                        .unwrap();
16513                handler
16514                    .handle_supervisor_provenance(&ctx, frame, None)
16515                    .await
16516                    .unwrap()
16517            })
16518        };
16519        let explicit = observe(handler.clone(), 114);
16520        began.await.unwrap();
16521        let concurrent = observe(handler.clone(), 115);
16522        // Keep explicit provenance waiting on its first image observation beyond
16523        // the list deadline; release it only after the concurrent list replies.
16524        tokio::time::sleep(Duration::from_millis(300)).await;
16525        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16526        let list = tokio::time::timeout(
16527            Duration::from_secs(1),
16528            handler.handle_supervisor_list(frame),
16529        )
16530        .await;
16531        let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
16532        drop(release);
16533        let replies = explicit.await.unwrap();
16534        let concurrent_replies = concurrent.await.unwrap();
16535        assert!(
16536            explicit_was_pending,
16537            "explicit provenance must await the full observation"
16538        );
16539        let list = list
16540            .expect("a concurrent explicit probe must not stall list")
16541            .unwrap();
16542        let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
16543        assert_eq!(list_body["op"], "supervisor.list");
16544        for replies in [replies, concurrent_replies] {
16545            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16546            assert_eq!(
16547                body["daemon"]["daemon_observed"]["running_image"]["status"],
16548                "match"
16549            );
16550        }
16551    }
16552
16553    #[tokio::test]
16554    async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
16555        let registry = Arc::new(Registry::default());
16556        let handle = SupervisorHandle::new();
16557        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16558            .with_handle(handle.clone());
16559        let module = supervisor
16560            .supervise_configured(
16561                crate::ModuleSpec {
16562                    carry_containment: false,
16563                    module_id: "memory-read".into(),
16564                    program: PathBuf::from("disabled-module"),
16565                    args: Vec::new(),
16566                    env: Vec::new(),
16567                    reserved: false,
16568                    reserved_prefixes: Vec::new(),
16569                    protocol: ModuleProtocol::Subc,
16570                    overlap: Default::default(),
16571                },
16572                true,
16573            )
16574            .unwrap();
16575        let _guard = crate::subtree_resources::fault_read_guard().await;
16576        let (target, release, began) = crate::subtree_resources::held();
16577        // Optional list work may be skipped when its deadline is already spent.
16578        // Establish the stalled fixture before asking the list to observe it.
16579        let stalled_target = target.clone();
16580        let stalled = tokio::task::spawn_blocking(move || {
16581            stalled_target.tree(&subc_os::ProcessTable::default())
16582        });
16583        began.await.unwrap();
16584        module.set_subtree_target_for_test(target);
16585        let handler = ControlHandler::new(registry).with_supervisor(handle);
16586        let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
16587        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16588        let replies = handler
16589            .handle_client_control_request(
16590                &ctx,
16591                frame,
16592                ClientControlRequest::SupervisorList {},
16593                &mut None,
16594            )
16595            .await
16596            .unwrap();
16597        assert!(
16598            replies.is_empty(),
16599            "list reply must be detached from the connection reader"
16600        );
16601        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16602        let replies = handler
16603            .handle_client_control_request(
16604                &ctx,
16605                frame,
16606                ClientControlRequest::SupervisorSpawnSnapshot {},
16607                &mut None,
16608            )
16609            .await
16610            .unwrap();
16611        assert_eq!(
16612            replies.len(),
16613            1,
16614            "another request must complete while the subtree read stalls"
16615        );
16616        let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16617            .await
16618            .unwrap()
16619            .unwrap();
16620        release.send(()).unwrap();
16621        drop(release);
16622        stalled.await.unwrap();
16623        let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
16624        assert!(
16625            body["modules"][0].get("tree").is_none(),
16626            "an unknown tree reading must be omitted after the bounded read expires"
16627        );
16628        // Lists export completed work rather than measuring synchronously.
16629        // Complete a refresh after the fixture is released before asserting
16630        // that a later list contains its measured tree.
16631        handler
16632            .resource_tracker
16633            .read(handler.supervisor.list(), false, "released_tree_fixture")
16634            .await;
16635        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16636        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16637        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16638        assert_eq!(
16639            body["modules"][0]["tree"]["source"], "process_tree",
16640            "the next request must export the completed refresh"
16641        );
16642        assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
16643    }
16644
16645    #[tokio::test]
16646    async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
16647        use subc_control::{
16648            AgentCommandResources, ChildMemoryKind, ChildResourceReading,
16649            ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
16650            TreeReading, TreeSource,
16651        };
16652        let registry = Arc::new(Registry::default());
16653        let handle = SupervisorHandle::new();
16654        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16655            .with_handle(handle.clone());
16656        supervisor
16657            .supervise_configured(
16658                crate::ModuleSpec {
16659                    carry_containment: false,
16660                    module_id: "budget-worker".into(),
16661                    program: PathBuf::from("disabled-module"),
16662                    args: Vec::new(),
16663                    env: Vec::new(),
16664                    reserved: false,
16665                    reserved_prefixes: Vec::new(),
16666                    protocol: ModuleProtocol::Subc,
16667                    overlap: Default::default(),
16668                },
16669                true,
16670            )
16671            .unwrap();
16672        let handler = ControlHandler::new(registry).with_supervisor(handle);
16673        let now = u64::try_from(
16674            std::time::SystemTime::now()
16675                .duration_since(std::time::UNIX_EPOCH)
16676                .unwrap()
16677                .as_millis(),
16678        )
16679        .unwrap();
16680        let measured = now - 1_000;
16681        let hour = CpuLastHour::new(100, 60_000);
16682        let daemon = DaemonResources::new(7, 1, "test")
16683            .with_measured_at_ms(Some(measured))
16684            .with_resources(Some(ChildResourceUsage::Measured(
16685                ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
16686            )))
16687            .with_cpu_last_hour(Some(hour.clone()));
16688        let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
16689            .with_measured_at_ms(Some(measured))
16690            .with_cpu_last_hour(Some(hour.clone()));
16691        let tree = TreeReading::new(
16692            150,
16693            50,
16694            ChildMemoryKind::ResidentSet,
16695            1,
16696            TreeSource::ProcessTree,
16697        )
16698        .with_measured_at_ms(Some(measured));
16699        handler.resource_tracker.seed_committed_for_test(
16700            &handler.supervisor.list(),
16701            crate::resource_totals::Readings {
16702                modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
16703                daemon: Some(daemon),
16704                agent_commands: Some(vec![agents]),
16705                ..Default::default()
16706            },
16707            now,
16708        );
16709        list_diagnostics::scope(async {
16710            tokio::time::sleep(Duration::from_millis(110)).await;
16711            assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
16712            let request =
16713                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16714            let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
16715            let ClientControlResponse::SupervisorList {
16716                modules,
16717                daemon,
16718                agent_commands,
16719                ..
16720            } = serde_json::from_slice(&replies[0].body).unwrap()
16721            else {
16722                panic!("expected a list response")
16723            };
16724            assert!(matches!(
16725                modules[0].resources,
16726                Some(ChildResourceUsage::Unavailable {
16727                    reason: ChildResourceUnavailableReason::Unreadable
16728                })
16729            ));
16730            assert_eq!(
16731                (
16732                    modules[0].tree.as_ref().unwrap().cpu_ms,
16733                    modules[0].tree.as_ref().unwrap().measured_at_ms
16734                ),
16735                (150, Some(measured))
16736            );
16737            let daemon = daemon.unwrap();
16738            let agents = agent_commands.unwrap();
16739            assert_eq!(daemon.measured_at_ms, Some(measured));
16740            assert_eq!(
16741                (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
16742                (1, 50, Some(measured))
16743            );
16744            for hour in [
16745                modules[0].cpu_last_hour.as_ref(),
16746                daemon.cpu_last_hour.as_ref(),
16747                agents[0].cpu_last_hour.as_ref(),
16748            ] {
16749                let hour = hour.unwrap();
16750                assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
16751            }
16752        })
16753        .await;
16754    }
16755
16756    #[tokio::test]
16757    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
16758        let registry = Arc::new(Registry::default());
16759        let handle = SupervisorHandle::new();
16760        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16761            .with_handle(handle.clone());
16762        supervisor
16763            .supervise_configured(
16764                crate::ModuleSpec {
16765                    carry_containment: false,
16766                    module_id: "drift".into(),
16767                    program: PathBuf::from("disabled-module"),
16768                    args: Vec::new(),
16769                    env: Vec::new(),
16770                    reserved: false,
16771                    reserved_prefixes: Vec::new(),
16772                    protocol: ModuleProtocol::Subc,
16773                    overlap: Default::default(),
16774                },
16775                false,
16776            )
16777            .unwrap();
16778        let handler = ControlHandler::new(registry).with_supervisor(handle);
16779        for configured in [None, Some(false), Some(true)] {
16780            handler.capability_evaluator.configure(
16781                configured.map(|enabled| ("drift".to_string(), enabled)),
16782                BTreeMap::new(),
16783            );
16784            let frame =
16785                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16786            let replies = handler
16787                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
16788                .await
16789                .unwrap();
16790            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16791            assert!(body.get("configured_enabled").is_none());
16792            assert_eq!(body["op"], "supervisor.ack");
16793            let frame =
16794                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16795            let replies = handler.handle_supervisor_list(frame).await.unwrap();
16796            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16797            assert_eq!(
16798                body["modules"][0].get("configured_enabled"),
16799                configured.map(Value::Bool).as_ref()
16800            );
16801            assert!(body["modules"][0].get("tree").is_none());
16802            assert!(body["modules"][0].get("subtree").is_none());
16803        }
16804    }
16805
16806    #[tokio::test(flavor = "current_thread")]
16807    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
16808        let handler = ControlHandler::new(Arc::new(Registry::default()));
16809        let capture = EventCapture::default();
16810        let _subscriber =
16811            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16812        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
16813        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16814
16815        let _ = handler
16816            .handle_client_control_request(
16817                &ctx,
16818                frame,
16819                ClientControlRequest::SupervisorSetEnabled {
16820                    module_id: "broca".to_string(),
16821                    enabled: false,
16822                },
16823                &mut None,
16824            )
16825            .await
16826            .unwrap();
16827
16828        let events = capture
16829            .events()
16830            .into_iter()
16831            .filter(|event| {
16832                event.target == "control"
16833                    && event.fields.get("message").map(String::as_str)
16834                        == Some("supervisor request received")
16835            })
16836            .collect::<Vec<_>>();
16837        assert_eq!(events.len(), 1, "one supervisor request log line");
16838        let event = &events[0];
16839        assert_eq!(event.level, tracing::Level::INFO);
16840        assert_eq!(
16841            event.fields.get("op"),
16842            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
16843        );
16844        assert_eq!(
16845            event.fields.get("module_id"),
16846            Some(&"\"broca\"".to_string())
16847        );
16848        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
16849        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
16850        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
16851    }
16852
16853    #[tokio::test(flavor = "current_thread")]
16854    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
16855        let registry = Arc::new(Registry::default());
16856        let module_connection = ConnectionId::new(303);
16857        registry
16858            .register_with_control_ops(
16859                manifest("aft", PROTOCOL_VERSION),
16860                PROTOCOL_VERSION,
16861                module_connection,
16862                module_baseline_control_ops(),
16863            )
16864            .unwrap();
16865        let handler = ControlHandler::new(registry);
16866        let capture = EventCapture::default();
16867        let _subscriber =
16868            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16869        let (ctx, _rx) = route_ctx(module_connection);
16870        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16871
16872        let _ = handler
16873            .handle_client_control_request(
16874                &ctx,
16875                frame,
16876                ClientControlRequest::SupervisorSetEnabled {
16877                    module_id: "broca".to_string(),
16878                    enabled: false,
16879                },
16880                &mut None,
16881            )
16882            .await
16883            .unwrap();
16884
16885        let callers = capture
16886            .events()
16887            .into_iter()
16888            .filter(|event| {
16889                event.fields.get("message").map(String::as_str)
16890                    == Some("supervisor request received")
16891            })
16892            .map(|event| event.fields.get("caller").cloned())
16893            .collect::<Vec<_>>();
16894        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
16895    }
16896
16897    #[tokio::test(flavor = "current_thread")]
16898    async fn supervisor_list_does_not_log_request_received() {
16899        let handler = ControlHandler::new(Arc::new(Registry::default()));
16900        let capture = EventCapture::default();
16901        let _subscriber =
16902            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16903        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
16904        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16905
16906        handler
16907            .handle_client_control_request(
16908                &ctx,
16909                frame,
16910                ClientControlRequest::SupervisorList {},
16911                &mut None,
16912            )
16913            .await
16914            .unwrap();
16915
16916        assert!(capture.events().into_iter().all(|event| {
16917            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
16918        }));
16919    }
16920
16921    #[tokio::test(flavor = "current_thread")]
16922    async fn supervisor_rescan_preview_does_not_log_request_received() {
16923        let handler = ControlHandler::new(Arc::new(Registry::default()));
16924        let capture = EventCapture::default();
16925        let _subscriber =
16926            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16927        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
16928        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16929
16930        handler
16931            .handle_client_control_request(
16932                &ctx,
16933                frame,
16934                ClientControlRequest::SupervisorRescan { preview: true },
16935                &mut None,
16936            )
16937            .await
16938            .unwrap();
16939
16940        assert!(capture.events().into_iter().all(|event| {
16941            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
16942        }));
16943    }
16944
16945    struct ApproveOperator;
16946    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
16947        fn prompt(
16948            &self,
16949            _: &str,
16950            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
16951        ) -> crate::operator_confirm::ProviderResult {
16952            crate::operator_confirm::ProviderResult::Approved
16953        }
16954    }
16955
16956    #[tokio::test]
16957    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
16958        assert_eq!(
16959            ControlHandler::default().privileged_operator_prompt,
16960            PrivilegedOperatorPrompt::Log
16961        );
16962        for (initial, changed, running) in [
16963            ("log", "enforce", PrivilegedOperatorPrompt::Log),
16964            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
16965        ] {
16966            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
16967            let path = dir.join("subc.jsonc");
16968            std::fs::write(
16969                &path,
16970                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
16971            )
16972            .unwrap();
16973            let config = crate::daemon_config::load(&path).unwrap().unwrap();
16974            let registry = Arc::new(Registry::default());
16975            let handle = SupervisorHandle::new();
16976            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
16977                .with_handle(handle.clone());
16978            let handler = ControlHandler::new(registry)
16979                .with_supervisor(handle)
16980                .with_privileged_operator_prompt(config.privileged_operator_prompt)
16981                .with_supervisor_rescan(supervisor, &path, config.port);
16982            handler.forwarding.operator_confirms().configure(
16983                Arc::new(ApproveOperator),
16984                crate::operator_confirm::OperatorLimits::default(),
16985            );
16986            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
16987
16988            for (disk_mode, needs_restart) in [
16989                (Some(initial), false),
16990                (Some(changed), true),
16991                (Some(changed), true),
16992                (None, running == PrivilegedOperatorPrompt::Enforce),
16993                (Some(initial), false),
16994            ] {
16995                let mut doc = json!({"version":1});
16996                if let Some(mode) = disk_mode {
16997                    doc["privileged_operator_prompt"] = json!(mode);
16998                }
16999                std::fs::write(&path, doc.to_string()).unwrap();
17000                for preview in [true, false] {
17001                    let request = ClientControlRequest::SupervisorRescan { preview };
17002                    let frame = Frame::build(
17003                        FrameType::Request,
17004                        control_flags(),
17005                        0,
17006                        0,
17007                        1,
17008                        serde_json::to_vec(&request).unwrap(),
17009                    )
17010                    .unwrap();
17011                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17012                    assert_eq!(frames.len(), 1);
17013                    let ClientControlResponse::SupervisorRescan { result } =
17014                        serde_json::from_slice(&frames[0].body).unwrap()
17015                    else {
17016                        panic!("expected a rescan response");
17017                    };
17018                    let expected = if needs_restart {
17019                        vec!["privileged_operator_prompt"]
17020                    } else {
17021                        Vec::new()
17022                    };
17023                    assert_eq!(result.restart_required, expected);
17024                    assert_eq!(handler.privileged_operator_prompt, running);
17025                    assert_eq!(
17026                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
17027                        running
17028                    );
17029                }
17030            }
17031        }
17032    }
17033
17034    #[tokio::test]
17035    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
17036        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
17037        let path = dir.join("subc.jsonc");
17038        let registry = Arc::new(Registry::default());
17039        let handle = SupervisorHandle::new();
17040        let supervisor =
17041            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
17042        let handler = ControlHandler::new(registry)
17043            .with_supervisor(handle)
17044            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
17045            .with_supervisor_rescan(supervisor, &path, None);
17046        handler.forwarding.operator_confirms().configure(
17047            Arc::new(ApproveOperator),
17048            crate::operator_confirm::OperatorLimits::default(),
17049        );
17050        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
17051        for preview in [true, false] {
17052            std::fs::write(
17053                &path,
17054                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
17055            )
17056            .unwrap();
17057            let request = ClientControlRequest::SupervisorRescan { preview };
17058            let frame = Frame::build(
17059                FrameType::Request,
17060                control_flags(),
17061                0,
17062                0,
17063                1,
17064                serde_json::to_vec(&request).unwrap(),
17065            )
17066            .unwrap();
17067            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17068            assert_eq!(frames.len(), 1);
17069            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
17070            assert_eq!(
17071                handler.privileged_operator_prompt,
17072                PrivilegedOperatorPrompt::Enforce
17073            );
17074        }
17075    }
17076
17077    /// One wire code has several senders, so the refusal line names the check
17078    /// that refused. This drives the shared refusal path for ordinary refusals
17079    /// with an unregistered
17080    /// target and requires the branch label on the event.
17081    #[tokio::test]
17082    async fn route_open_refusal_names_the_check_that_refused() {
17083        let handler = ControlHandler::new(Arc::new(Registry::default()));
17084        let capture = EventCapture::default();
17085        let _subscriber =
17086            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17087        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17088        let response = handler
17089            .handle_control_frame(
17090                &ctx,
17091                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
17092            )
17093            .await
17094            .unwrap();
17095
17096        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17097        let event = capture
17098            .events()
17099            .into_iter()
17100            .find(|event| {
17101                event.target == "control"
17102                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17103            })
17104            .expect("route.open refusal event");
17105        assert_eq!(
17106            event.fields.get("reason"),
17107            Some(&"\"not_registered\"".to_string())
17108        );
17109    }
17110
17111    #[tokio::test]
17112    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
17113        let registry = Arc::new(Registry::default());
17114        let supervisor_handle = SupervisorHandle::new();
17115        let supervisor =
17116            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
17117                .with_handle(supervisor_handle.clone())
17118                .with_connection_file_path(std::env::temp_dir().join(format!(
17119                    "subc-route-open-refusal-info-{}",
17120                    std::process::id()
17121                )));
17122        let module = supervisor
17123            .supervise_configured(
17124                ModuleSpec {
17125                    carry_containment: false,
17126                    module_id: "warming".to_string(),
17127                    program: fake_aft_stub_path(),
17128                    args: Vec::new(),
17129                    env: Vec::new(),
17130                    reserved: false,
17131                    reserved_prefixes: Vec::new(),
17132                    protocol: ModuleProtocol::Subc,
17133                    overlap: Default::default(),
17134                },
17135                true,
17136            )
17137            .unwrap();
17138        assert_eq!(module.state().unwrap(), ModuleState::Running);
17139
17140        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17141        assert!(handler
17142            .counters()
17143            .snapshot()
17144            .get("route_open_refused_by_code")
17145            .is_none());
17146        let capture = EventCapture::default();
17147        let _subscriber =
17148            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17149        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
17150        let response = handler
17151            .handle_control_frame(
17152                &ctx,
17153                route_open_frame(394, "warming", unique_project_root("refusal-info")),
17154            )
17155            .await
17156            .unwrap();
17157        module.stop().await.unwrap();
17158
17159        assert_eq!(parse_error(&response[0])["code"], "module_warming");
17160        let event = capture
17161            .events()
17162            .into_iter()
17163            .find(|event| {
17164                event.target == "control"
17165                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
17166            })
17167            .expect("route.open refusal event");
17168        assert_eq!(
17169            event.fields.get("module_id"),
17170            Some(&"\"warming\"".to_string())
17171        );
17172        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
17173        assert_eq!(
17174            event.fields.get("reason"),
17175            Some(&"\"supervised_not_registered\"".to_string())
17176        );
17177        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
17178        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
17179        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
17180        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
17181        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
17182        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
17183        assert_eq!(
17184            handler.counters().snapshot()["route_open_refused_by_code"],
17185            json!({ "module_warming": 1 })
17186        );
17187    }
17188
17189    const OUTAGE_START: &str = "route.open refusing module: not serving";
17190    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
17191
17192    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
17193        capture
17194            .events()
17195            .into_iter()
17196            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
17197            .collect()
17198    }
17199
17200    fn supervise_stub(
17201        registry: &Arc<Registry>,
17202        module_id: &str,
17203        enabled: bool,
17204    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
17205        let supervisor_handle = SupervisorHandle::new();
17206        let supervisor =
17207            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
17208                .with_handle(supervisor_handle.clone())
17209                .with_connection_file_path(std::env::temp_dir().join(format!(
17210                    "subc-route-outage-{module_id}-{}",
17211                    std::process::id()
17212                )));
17213        let module = supervisor
17214            .supervise_configured(
17215                ModuleSpec {
17216                    carry_containment: false,
17217                    module_id: module_id.to_string(),
17218                    program: fake_aft_stub_path(),
17219                    args: Vec::new(),
17220                    env: Vec::new(),
17221                    reserved: false,
17222                    reserved_prefixes: Vec::new(),
17223                    protocol: ModuleProtocol::Subc,
17224                    overlap: Default::default(),
17225                },
17226                enabled,
17227            )
17228            .unwrap();
17229        (supervisor_handle, module)
17230    }
17231
17232    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
17233        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
17234            module_id: module_id.to_string(),
17235            drain_timeout_ms: Some(50),
17236        })
17237        .unwrap();
17238        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
17239    }
17240
17241    /// Two handlers built over one forwarding table must share one outage
17242    /// tracker; separate trackers would each log their own opening line for
17243    /// the same outage.
17244    #[test]
17245    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
17246        let registry = Arc::new(Registry::default());
17247        let forwarding = Arc::new(ForwardingTable::default());
17248        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17249        let second = ControlHandler::with_forwarding(registry, forwarding);
17250        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
17251    }
17252
17253    /// A client can name any module id it likes. Refusing an unknown one,
17254    /// however often, must not create outage state or outage lines, or the
17255    /// tracker would be a memory sink any client could fill.
17256    #[tokio::test(flavor = "current_thread")]
17257    async fn route_open_unknown_module_refusals_add_no_outage_state() {
17258        let handler = ControlHandler::new(Arc::new(Registry::default()));
17259        let capture = EventCapture::default();
17260        let _subscriber =
17261            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17262        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
17263        for corr in 0..8 {
17264            let response = handler
17265                .handle_control_frame(
17266                    &ctx,
17267                    route_open_frame(
17268                        380 + corr,
17269                        &format!("nobody-{corr}"),
17270                        unique_project_root("outage-unknown"),
17271                    ),
17272                )
17273                .await
17274                .unwrap();
17275            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17276        }
17277
17278        assert_eq!(handler.route_outages.tracked_module_count(), 0);
17279        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
17280        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
17281    }
17282
17283    /// Drives the refusal path end to end: a supervised module that served
17284    /// before and stopped being registered with no instruction to stop is a
17285    /// WARN, and the same module refused after an operator `supervisor.restart`
17286    /// is an INFO.
17287    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17288    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
17289        let registry = Arc::new(Registry::default());
17290        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
17291        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17292        let capture = EventCapture::default();
17293        let _subscriber =
17294            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17295        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
17296        // The stub never registers, so pretend it served once: otherwise every
17297        // refusal would fall in its startup window.
17298        handler.route_outages.record_accepted("outage-restart");
17299
17300        let response = handler
17301            .handle_control_frame(
17302                &ctx,
17303                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
17304            )
17305            .await
17306            .unwrap();
17307        assert_eq!(response[0].header.ty, FrameType::Error);
17308        let starts = outage_lines(&capture, OUTAGE_START);
17309        assert_eq!(starts.len(), 1, "{starts:?}");
17310        assert_eq!(starts[0].level, tracing::Level::WARN);
17311        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
17312        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
17313        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
17314        handler.route_outages.record_accepted("outage-restart");
17315        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
17316
17317        let restart = handler
17318            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
17319            .await
17320            .unwrap();
17321        assert_eq!(
17322            restart[0].header.ty,
17323            FrameType::Response,
17324            "{:?}",
17325            parse_error(&restart[0])
17326        );
17327        handler
17328            .handle_control_frame(
17329                &ctx,
17330                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
17331            )
17332            .await
17333            .unwrap();
17334        module.stop().await.unwrap();
17335
17336        let starts = outage_lines(&capture, OUTAGE_START);
17337        assert_eq!(starts.len(), 2, "{starts:?}");
17338        assert_eq!(starts[1].level, tracing::Level::INFO);
17339        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
17340    }
17341
17342    /// A restart refused before it touched the module (here: the module is
17343    /// disabled) must clear its operator mark, so the next real outage is
17344    /// still reported as a warning.
17345    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17346    async fn failed_operator_restart_leaves_no_operator_mark() {
17347        let registry = Arc::new(Registry::default());
17348        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
17349        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17350        let capture = EventCapture::default();
17351        let _subscriber =
17352            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17353        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
17354        handler.route_outages.record_accepted("outage-disabled");
17355
17356        let restart = handler
17357            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
17358            .await
17359            .unwrap();
17360        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
17361        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
17362
17363        handler
17364            .handle_control_frame(
17365                &ctx,
17366                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
17367            )
17368            .await
17369            .unwrap();
17370        let starts = outage_lines(&capture, OUTAGE_START);
17371        assert_eq!(starts.len(), 1, "{starts:?}");
17372        assert_eq!(starts[0].level, tracing::Level::WARN);
17373    }
17374
17375    #[tokio::test(flavor = "current_thread")]
17376    async fn route_open_unknown_module_escapes_target_module_id() {
17377        let handler = ControlHandler::new(Arc::new(Registry::default()));
17378        let capture = EventCapture::default();
17379        let _subscriber =
17380            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17381        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
17382        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17383        let response = handler
17384            .handle_control_frame(
17385                &ctx,
17386                route_open_frame(
17387                    395,
17388                    hostile_module_id,
17389                    unique_project_root("hostile-target-module-id"),
17390                ),
17391            )
17392            .await
17393            .unwrap();
17394
17395        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17396        let event = capture
17397            .events()
17398            .into_iter()
17399            .find(|event| {
17400                event.target == "control"
17401                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17402            })
17403            .expect("route.open unknown-module refusal event");
17404        let logged = event.fields.get("module_id").expect("module_id field");
17405        assert!(!logged.bytes().any(|byte| byte < 0x20));
17406        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17407    }
17408
17409    /// A route.open refusal is the daemon's own answer, so its ERROR carries
17410    /// the daemon-origin bit on the frame the client receives.
17411    #[tokio::test(flavor = "current_thread")]
17412    async fn route_open_refusal_carries_daemon_origin() {
17413        let handler = ControlHandler::new(Arc::new(Registry::default()));
17414        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17415        let response = handler
17416            .handle_control_frame(
17417                &ctx,
17418                route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
17419            )
17420            .await
17421            .unwrap();
17422
17423        assert_eq!(response.len(), 1);
17424        assert_eq!(response[0].header.ty, FrameType::Error);
17425        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17426        assert!(
17427            response[0].header.flags.is_daemon_origin(),
17428            "route.open refusal must carry daemon origin: {:?}",
17429            response[0].header.flags
17430        );
17431    }
17432
17433    /// The drain `module_reloading` refusal is daemon-synthesized for an
17434    /// endpoint that is reloading, so it carries the daemon-origin bit on the
17435    /// frame the client receives.
17436    #[tokio::test(flavor = "current_thread")]
17437    async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
17438        let registry = Arc::new(Registry::default());
17439        let forwarding = Arc::new(ForwardingTable::default());
17440        let handler =
17441            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17442        let module_connection = ConnectionId::new(95);
17443        let (module_ctx, mut module_rx) = route_ctx(module_connection);
17444        hello_via_sink(
17445            &handler,
17446            &module_ctx,
17447            &mut module_rx,
17448            hello_frame("aft", PROTOCOL_VERSION, 395),
17449        )
17450        .await;
17451        forwarding
17452            .begin_module_drain("aft", RouteCloseReason::Restart)
17453            .unwrap()
17454            .expect("the module endpoint drains");
17455
17456        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
17457        let response = handler
17458            .handle_control_frame(
17459                &client_ctx,
17460                route_open_frame(396, "aft", unique_project_root("drain-origin")),
17461            )
17462            .await
17463            .unwrap();
17464
17465        assert_eq!(response.len(), 1);
17466        assert_eq!(response[0].header.ty, FrameType::Error);
17467        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17468        assert!(
17469            response[0].header.flags.is_daemon_origin(),
17470            "drain module_reloading must carry daemon origin: {:?}",
17471            response[0].header.flags
17472        );
17473    }
17474
17475    /// A module that refuses a bind with a daemon code still cannot mint a
17476    /// daemon refusal: the relayed `module_reloading` body reaches the client
17477    /// with the daemon-origin bit clear.
17478    #[tokio::test(flavor = "current_thread")]
17479    async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
17480        let registry = Arc::new(Registry::default());
17481        let forwarding = Arc::new(ForwardingTable::default());
17482        let handler =
17483            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17484        let module_connection = ConnectionId::new(95);
17485        let (module_ctx, mut module_rx) = route_ctx(module_connection);
17486        hello_via_sink(
17487            &handler,
17488            &module_ctx,
17489            &mut module_rx,
17490            hello_frame("aft", PROTOCOL_VERSION, 395),
17491        )
17492        .await;
17493
17494        let client_connection = ConnectionId::new(96);
17495        let (client_ctx, _client_rx) = route_ctx(client_connection);
17496        let (route_task, bind) = relay_route_open(
17497            &handler,
17498            client_connection,
17499            &client_ctx.egress,
17500            &mut module_rx,
17501            396,
17502            "aft",
17503            "spoofed-reloading",
17504        )
17505        .await;
17506        // The module sets the origin bit on its own rejection too: neither
17507        // the bit nor the daemon code may survive the relay as daemon-owned.
17508        let rejection = Frame::build(
17509            FrameType::Error,
17510            crate::router::daemon_error_flags(),
17511            0,
17512            0,
17513            bind.header.corr,
17514            serde_json::to_vec(&ErrorBody::new(
17515                "module_reloading",
17516                "module refused route.bind",
17517            ))
17518            .unwrap(),
17519        )
17520        .unwrap();
17521        handler
17522            .handle_control_frame(&module_ctx, rejection)
17523            .await
17524            .unwrap();
17525
17526        let response = route_task.await.unwrap();
17527        assert_eq!(response.len(), 1);
17528        assert_eq!(response[0].header.ty, FrameType::Error);
17529        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17530        assert!(
17531            !response[0].header.flags.is_daemon_origin(),
17532            "a module's own rejection must not carry daemon origin: {:?}",
17533            response[0].header.flags
17534        );
17535    }
17536
17537    #[tokio::test(flavor = "current_thread")]
17538    async fn route_open_module_rejection_uses_daemon_counter_key() {
17539        let registry = Arc::new(Registry::default());
17540        let forwarding = Arc::new(ForwardingTable::default());
17541        let handler =
17542            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17543        let module_connection = ConnectionId::new(95);
17544        let (module_ctx, mut module_rx) = route_ctx(module_connection);
17545        hello_via_sink(
17546            &handler,
17547            &module_ctx,
17548            &mut module_rx,
17549            hello_frame("aft", PROTOCOL_VERSION, 395),
17550        )
17551        .await;
17552
17553        let client_connection = ConnectionId::new(96);
17554        let (client_ctx, _client_rx) = route_ctx(client_connection);
17555        let capture = EventCapture::default();
17556        // tracing-core decides once per callsite whether its events are enabled,
17557        // and caches the answer. If a parallel test with no subscriber reaches
17558        // this callsite first, the cached answer is "disabled" and this test's
17559        // capture sees nothing. Keeping a second dispatch alive makes tracing-core
17560        // ask every live subscriber instead, while only this test's default
17561        // receives the events.
17562        static INTEREST_ANCHOR: std::sync::OnceLock<tracing::Dispatch> = std::sync::OnceLock::new();
17563        INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
17564        let _subscriber =
17565            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17566        let (route_task, bind) = relay_route_open(
17567            &handler,
17568            client_connection,
17569            &client_ctx.egress,
17570            &mut module_rx,
17571            396,
17572            "aft",
17573            "hostile-module-code",
17574        )
17575        .await;
17576        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
17577        let rejection = Frame::build(
17578            FrameType::Error,
17579            control_flags(),
17580            0,
17581            0,
17582            bind.header.corr,
17583            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
17584        )
17585        .unwrap();
17586        handler
17587            .handle_control_frame(&module_ctx, rejection)
17588            .await
17589            .unwrap();
17590
17591        let response = route_task.await.unwrap();
17592        assert_eq!(parse_error(&response[0])["code"], hostile_code);
17593        let counters = handler.counters().snapshot();
17594        assert_eq!(
17595            counters["route_open_refused_by_code"],
17596            json!({ "module_rejected": 1 })
17597        );
17598        assert!(counters["route_open_refused_by_code"]
17599            .get(hostile_code)
17600            .is_none());
17601
17602        let event = capture
17603            .events()
17604            .into_iter()
17605            .find(|event| {
17606                event.target == "control"
17607                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
17608            })
17609            .expect("route.open module-rejection refusal event");
17610        let logged = event.fields.get("module_code").expect("module_code field");
17611        assert!(!logged.bytes().any(|byte| byte < 0x20));
17612        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17613    }
17614
17615    #[tokio::test]
17616    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
17617        let registry = Arc::new(Registry::default());
17618        let supervisor_handle = SupervisorHandle::new();
17619        let missing_program = std::env::temp_dir().join(format!(
17620            "subc-route-open-missing-program-{}",
17621            std::process::id()
17622        ));
17623        let supervisor =
17624            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
17625                .with_handle(supervisor_handle.clone());
17626        let module = supervisor
17627            .supervise_configured(
17628                ModuleSpec {
17629                    carry_containment: false,
17630                    module_id: "failed".to_string(),
17631                    program: missing_program,
17632                    args: Vec::new(),
17633                    env: Vec::new(),
17634                    reserved: false,
17635                    reserved_prefixes: Vec::new(),
17636                    protocol: ModuleProtocol::Subc,
17637                    overlap: Default::default(),
17638                },
17639                true,
17640            )
17641            .unwrap();
17642        assert_eq!(module.state().unwrap(), ModuleState::Failed);
17643
17644        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17645        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
17646        let response = handler
17647            .handle_control_frame(
17648                &ctx,
17649                route_open_frame(305, "failed", unique_project_root("failed")),
17650            )
17651            .await
17652            .unwrap();
17653
17654        assert_eq!(response[0].header.ty, FrameType::Error);
17655        let error = parse_error(&response[0]);
17656        assert_eq!(error["code"], "target_unavailable");
17657        assert!(error["message"]
17658            .as_str()
17659            .unwrap()
17660            .contains("state=failed, enabled=true, live=false"));
17661    }
17662
17663    #[tokio::test]
17664    async fn route_open_role_mismatch_remains_target_unavailable() {
17665        let registry = Arc::new(Registry::default());
17666        let handler = ControlHandler::new(Arc::clone(&registry));
17667        handler
17668            .handle_control(
17669                ConnectionId::new(41),
17670                non_routable_hello_frame_with_control_ops("health-only", 306, None),
17671            )
17672            .unwrap();
17673
17674        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
17675        let response = handler
17676            .handle_control_frame(
17677                &ctx,
17678                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
17679            )
17680            .await
17681            .unwrap();
17682
17683        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17684        assert!(parse_error(&response[0])["message"]
17685            .as_str()
17686            .unwrap()
17687            .contains("does not provide the requested target"));
17688    }
17689
17690    #[tokio::test]
17691    async fn route_open_inactive_registration_remains_target_unavailable() {
17692        let registry = Arc::new(Registry::default());
17693        let handler = ControlHandler::new(Arc::clone(&registry));
17694        handler
17695            .handle_control(
17696                ConnectionId::new(43),
17697                hello_frame("inactive", PROTOCOL_VERSION, 308),
17698            )
17699            .unwrap();
17700        assert!(registry
17701            .set_module_state_for_test("inactive", ChannelState::Closed)
17702            .unwrap());
17703
17704        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
17705        let response = handler
17706            .handle_control_frame(
17707                &ctx,
17708                route_open_frame(309, "inactive", unique_project_root("inactive")),
17709            )
17710            .await
17711            .unwrap();
17712
17713        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17714        assert!(parse_error(&response[0])["message"]
17715            .as_str()
17716            .unwrap()
17717            .contains("is not active"));
17718    }
17719
17720    #[tokio::test]
17721    async fn late_health_reply_is_recorded_through_the_module_response_path() {
17722        let registry = Arc::new(Registry::default());
17723        let forwarding = Arc::new(ForwardingTable::default());
17724        let supervisor_handle = SupervisorHandle::new();
17725        let supervisor =
17726            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
17727                .with_forwarding(Arc::clone(&forwarding))
17728                .with_handle(supervisor_handle.clone());
17729        let module = supervisor
17730            .supervise_configured(
17731                crate::ModuleSpec {
17732                    carry_containment: false,
17733                    module_id: "late-health-response".to_string(),
17734                    program: PathBuf::from("disabled-module"),
17735                    args: Vec::new(),
17736                    env: Vec::new(),
17737                    reserved: false,
17738                    reserved_prefixes: Vec::new(),
17739                    protocol: ModuleProtocol::Subc,
17740                    overlap: Default::default(),
17741                },
17742                false,
17743            )
17744            .unwrap();
17745        let handler =
17746            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17747                .with_supervisor(supervisor_handle);
17748        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
17749        handler
17750            .handle_control_frame(
17751                &module_ctx,
17752                hello_frame_with_control_ops(
17753                    "late-health-response",
17754                    PROTOCOL_VERSION,
17755                    7,
17756                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17757                ),
17758            )
17759            .await
17760            .unwrap();
17761        let probe_started_at = Instant::now() - Duration::from_millis(80);
17762        let pending = forwarding
17763            .begin_health_probe_rpc_for(
17764                "late-health-response",
17765                MODULE_CONTROL_OP_HEALTH_CHECK,
17766                probe_started_at,
17767                Instant::now() - Duration::from_millis(1),
17768            )
17769            .unwrap();
17770        assert!(forwarding
17771            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
17772            .unwrap());
17773
17774        let responses = handler
17775            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
17776            .await
17777            .unwrap();
17778
17779        assert!(responses.is_empty());
17780        let health = module.status().unwrap().health;
17781        assert_eq!(health.late_answer_count, 1);
17782        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
17783    }
17784
17785    #[tokio::test]
17786    async fn health_probe_timeout_and_module_death_are_typed() {
17787        let registry = Arc::new(Registry::default());
17788        let forwarding = Arc::new(ForwardingTable::default());
17789        let handler =
17790            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17791                .with_health_probe_timeout(Duration::from_millis(50));
17792        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
17793        hello_via_sink(
17794            &handler,
17795            &module_ctx,
17796            &mut module_rx,
17797            hello_frame_with_control_ops(
17798                "aft",
17799                PROTOCOL_VERSION,
17800                7,
17801                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17802            ),
17803        )
17804        .await;
17805
17806        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
17807        let responses = handler
17808            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
17809            .await
17810            .unwrap();
17811        assert_eq!(responses[0].header.ty, FrameType::Error);
17812        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
17813        let _ = module_rx.try_recv();
17814
17815        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
17816        let health_handler = handler.clone();
17817        let death_task = tokio::spawn(async move {
17818            health_handler
17819                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
17820                .await
17821                .unwrap()
17822        });
17823        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
17824            .await
17825            .unwrap()
17826            .unwrap();
17827        handler
17828            .cleanup_connection(module_ctx.connection_id)
17829            .unwrap();
17830        let responses = death_task.await.unwrap();
17831        assert_eq!(responses[0].header.ty, FrameType::Error);
17832        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
17833    }
17834
17835    #[test]
17836    fn hello_requires_exact_protocol_version() {
17837        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
17838            let registry = Arc::new(Registry::default());
17839            let handler = ControlHandler::new(Arc::clone(&registry));
17840            let responses = handler
17841                .handle_control(
17842                    ConnectionId::new(connection),
17843                    hello_frame("aft", offered, 9),
17844                )
17845                .unwrap();
17846
17847            assert_eq!(responses.len(), 1);
17848            assert_eq!(responses[0].header.ty, FrameType::Error);
17849            let error = parse_error(&responses[0]);
17850            assert_eq!(error["code"], "version_unsupported");
17851            assert!(registry.get_module("aft").unwrap().is_none());
17852            assert_eq!(registry.active_registration_count().unwrap(), 0);
17853        }
17854    }
17855
17856    #[test]
17857    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
17858        let registry = Arc::new(Registry::default());
17859        let forwarding = Arc::new(ForwardingTable::default());
17860        let handler =
17861            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17862        let module_connection = ConnectionId::new(301);
17863        let registration = registry
17864            .register_with_control_ops(
17865                manifest("aft-push", PROTOCOL_VERSION),
17866                PROTOCOL_VERSION,
17867                module_connection,
17868                module_baseline_control_ops(),
17869            )
17870            .unwrap();
17871        let (module_tx, _module_rx) = mpsc::channel(8);
17872        let endpoint = forwarding
17873            .register_module_connection(
17874                module_connection,
17875                "aft-push".to_string(),
17876                PROTOCOL_VERSION,
17877                manifest_concurrency(&registration.manifest),
17878                FrameSink::new(module_tx),
17879            )
17880            .unwrap();
17881
17882        // A push op this version does not know is ignored (forward-compat), not errored.
17883        let unknown = Frame::build(
17884            FrameType::Push,
17885            control_flags(),
17886            0,
17887            0,
17888            5,
17889            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
17890        )
17891        .unwrap();
17892        let out = handler.handle_status_update(endpoint, unknown).unwrap();
17893        assert!(
17894            out.is_empty(),
17895            "unknown push op must be ignored, got {out:?}"
17896        );
17897
17898        // A malformed body for a KNOWN op is a real error worth surfacing.
17899        let malformed = Frame::build(
17900            FrameType::Push,
17901            control_flags(),
17902            0,
17903            0,
17904            6,
17905            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
17906        )
17907        .unwrap();
17908        let out = handler.handle_status_update(endpoint, malformed).unwrap();
17909        assert_eq!(out.len(), 1);
17910        assert_eq!(out[0].header.ty, FrameType::Error);
17911        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
17912    }
17913
17914    #[test]
17915    fn hello_rejected_when_connection_already_owns_client_routes() {
17916        let registry = Arc::new(Registry::default());
17917        let forwarding = Arc::new(ForwardingTable::default());
17918        let handler =
17919            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17920        // Commits a client route on connection 202 (bound to a module on conn 101).
17921        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
17922        let client_connection = ConnectionId::new(202);
17923
17924        // That same connection now tries to register as a module: rejected, so one
17925        // connection never holds both client-route and module-endpoint state.
17926        let responses = handler
17927            .handle_control(
17928                client_connection,
17929                hello_frame("aft-second", PROTOCOL_VERSION, 9),
17930            )
17931            .unwrap();
17932        assert_eq!(responses[0].header.ty, FrameType::Error);
17933        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
17934        assert!(registry.get_module("aft-second").unwrap().is_none());
17935    }
17936
17937    #[tokio::test]
17938    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
17939        let registry = Arc::new(Registry::default());
17940        let forwarding = Arc::new(ForwardingTable::default());
17941        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
17942        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
17943        hello_via_sink(
17944            &handler,
17945            &module_ctx,
17946            &mut module_rx,
17947            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
17948        )
17949        .await;
17950        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
17951        let pending = forwarding
17952            .begin_route_bind_relay_for_test(
17953                client_ctx.connection_id,
17954                client_ctx.egress.clone(),
17955                2,
17956                "alpha",
17957            )
17958            .unwrap();
17959        forwarding
17960            .complete_pending_relay(
17961                module_ctx.connection_id,
17962                pending.corr,
17963                RouteBindRelayOutcome::Accepted,
17964            )
17965            .unwrap();
17966        client_rx.try_recv().unwrap();
17967        for module_id in ["beta", "alpha"] {
17968            let replies = handler
17969                .handle_control_frame(
17970                    &module_ctx,
17971                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
17972                )
17973                .await
17974                .unwrap();
17975            assert_eq!(replies.len(), 1, "second HELLO must be refused");
17976            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
17977        }
17978        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
17979        assert!(registry.get_module("beta").unwrap().is_none());
17980        assert!(matches!(
17981            forwarding
17982                .lookup_data_route(
17983                    client_ctx.connection_id,
17984                    pending.client_channel,
17985                    pending.client_epoch,
17986                )
17987                .unwrap(),
17988            DataRoute::Client(DataRouteState::Bound(_))
17989        ));
17990        assert!(handler
17991            .hello_launch_nonces
17992            .lock()
17993            .unwrap()
17994            .presented(module_ctx.connection_id, Some("alpha-nonce")));
17995        assert!(module_rx.try_recv().is_err());
17996    }
17997
17998    #[test]
17999    fn reserved_module_hello_requires_matching_launch_nonce() {
18000        let registry = Arc::new(Registry::default());
18001        let supervisor = SupervisorHandle::new();
18002        // The supervisor recorded the nonce it injected when it spawned the reserved
18003        // module; the HELLO verifier checks against the same shared handle.
18004        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
18005        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
18006
18007        // A HELLO with NO nonce is rejected.
18008        let no_nonce = handler
18009            .handle_control(
18010                ConnectionId::new(1),
18011                hello_frame("vault", PROTOCOL_VERSION, 1),
18012            )
18013            .unwrap();
18014        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
18015        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
18016        assert!(registry.get_module("vault").unwrap().is_none());
18017
18018        // A HELLO with the WRONG nonce is rejected.
18019        let wrong = handler
18020            .handle_control(
18021                ConnectionId::new(2),
18022                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
18023            )
18024            .unwrap();
18025        assert_eq!(wrong[0].header.ty, FrameType::Error);
18026        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
18027        assert!(registry.get_module("vault").unwrap().is_none());
18028
18029        // A HELLO with the CORRECT nonce registers.
18030        let ok = handler
18031            .handle_control(
18032                ConnectionId::new(3),
18033                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
18034            )
18035            .unwrap();
18036        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
18037        assert!(registry.get_module("vault").unwrap().is_some());
18038    }
18039
18040    #[test]
18041    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
18042        let registry = Arc::new(Registry::default());
18043        let supervisor = SupervisorHandle::new();
18044        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18045        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18046        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
18047
18048        let squat = handler
18049            .handle_control(
18050                ConnectionId::new(1),
18051                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
18052            )
18053            .unwrap();
18054        assert_eq!(squat[0].header.ty, FrameType::Error);
18055        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
18056        assert!(parse_error(&squat[0])["message"]
18057            .as_str()
18058            .unwrap()
18059            .contains("fed:"));
18060
18061        let accepted_peer = handler
18062            .handle_control(
18063                ConnectionId::new(2),
18064                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
18065            )
18066            .unwrap();
18067        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
18068
18069        let accepted_short = handler
18070            .handle_control(
18071                ConnectionId::new(3),
18072                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
18073            )
18074            .unwrap();
18075        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
18076
18077        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
18078            let response = handler
18079                .handle_control(
18080                    ConnectionId::new(conn),
18081                    hello_frame(module_id, PROTOCOL_VERSION, conn),
18082                )
18083                .unwrap();
18084            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
18085        }
18086    }
18087
18088    #[test]
18089    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
18090        let registry = Arc::new(Registry::default());
18091        let supervisor = SupervisorHandle::new();
18092        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18093        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18094        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
18095        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
18096
18097        let owner_nonce = handler
18098            .handle_control(
18099                ConnectionId::new(1),
18100                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
18101            )
18102            .unwrap();
18103        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
18104        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
18105        assert!(registry.get_module("fed:special").unwrap().is_none());
18106
18107        let exact_nonce = handler
18108            .handle_control(
18109                ConnectionId::new(2),
18110                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
18111            )
18112            .unwrap();
18113        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
18114        assert!(registry.get_module("fed:special").unwrap().is_some());
18115    }
18116
18117    #[test]
18118    fn non_reserved_module_ignores_launch_nonce() {
18119        let registry = Arc::new(Registry::default());
18120        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
18121        // registration succeeds whether a spawned process echoes a nonce or not.
18122        let handler = ControlHandler::new(Arc::clone(&registry));
18123        let no_nonce = handler
18124            .handle_control(
18125                ConnectionId::new(1),
18126                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
18127            )
18128            .unwrap();
18129        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
18130        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
18131
18132        let echoed_nonce = handler
18133            .handle_control(
18134                ConnectionId::new(2),
18135                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
18136            )
18137            .unwrap();
18138        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
18139        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
18140    }
18141
18142    #[test]
18143    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
18144        let handler = ControlHandler::default();
18145        let conn = ConnectionId::new(1);
18146        let malformed = Frame::build(
18147            FrameType::Hello,
18148            control_flags(),
18149            0,
18150            0,
18151            3,
18152            b"{not json".to_vec(),
18153        )
18154        .unwrap();
18155
18156        let error = handler.handle_control(conn, malformed).unwrap();
18157        assert_eq!(error[0].header.ty, FrameType::Error);
18158        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
18159
18160        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18161        let pong = handler.handle_control(conn, ping).unwrap();
18162        assert_eq!(pong[0].header.ty, FrameType::Pong);
18163        assert_eq!(pong[0].header.corr, 4);
18164    }
18165
18166    #[test]
18167    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
18168        let registry = Arc::new(Registry::default());
18169        let handler = ControlHandler::new(Arc::clone(&registry));
18170
18171        handler
18172            .handle_control(
18173                ConnectionId::new(1),
18174                hello_frame("aft", PROTOCOL_VERSION, 1),
18175            )
18176            .unwrap();
18177        let duplicate = handler
18178            .handle_control(
18179                ConnectionId::new(2),
18180                hello_frame("aft", PROTOCOL_VERSION, 2),
18181            )
18182            .unwrap();
18183
18184        assert_eq!(duplicate[0].header.ty, FrameType::Error);
18185        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
18186        let registration = registry.get_module("aft").unwrap().unwrap();
18187        assert_eq!(registration.connection_id, ConnectionId::new(1));
18188    }
18189
18190    #[test]
18191    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
18192        let registry = Arc::new(Registry::default());
18193        let forwarding = Arc::new(ForwardingTable::default());
18194        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
18195        let handler =
18196            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
18197                .with_process_liveness(process_liveness);
18198        let (ctx, route_channel, route_epoch) =
18199            bind_liveness_route(&registry, &forwarding, "aft-dead");
18200        let responses = handler
18201            .handle_route_poll(
18202                &ctx,
18203                route_poll_frame(41, PollKind::Liveness, route_channel),
18204                route_channel,
18205                route_epoch,
18206                PollKind::Liveness,
18207            )
18208            .unwrap();
18209
18210        assert_eq!(responses.len(), 1);
18211        assert_eq!(responses[0].header.ty, FrameType::Response);
18212        assert_route_poll_liveness(&responses[0], false);
18213    }
18214
18215    #[test]
18216    fn liveness_poll_without_process_source_uses_bound_route() {
18217        let registry = Arc::new(Registry::default());
18218        let forwarding = Arc::new(ForwardingTable::default());
18219        let handler =
18220            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
18221        let (ctx, route_channel, route_epoch) =
18222            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
18223        let responses = handler
18224            .handle_route_poll(
18225                &ctx,
18226                route_poll_frame(42, PollKind::Liveness, route_channel),
18227                route_channel,
18228                route_epoch,
18229                PollKind::Liveness,
18230            )
18231            .unwrap();
18232
18233        assert_route_poll_liveness(&responses[0], true);
18234    }
18235
18236    #[test]
18237    fn liveness_poll_untracked_process_source_uses_bound_route() {
18238        let registry = Arc::new(Registry::default());
18239        let forwarding = Arc::new(ForwardingTable::default());
18240        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
18241        let handler =
18242            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
18243                .with_process_liveness(process_liveness);
18244        let (ctx, route_channel, route_epoch) =
18245            bind_liveness_route(&registry, &forwarding, "aft-untracked");
18246        let responses = handler
18247            .handle_route_poll(
18248                &ctx,
18249                route_poll_frame(43, PollKind::Liveness, route_channel),
18250                route_channel,
18251                route_epoch,
18252                PollKind::Liveness,
18253            )
18254            .unwrap();
18255
18256        assert_route_poll_liveness(&responses[0], true);
18257    }
18258
18259    #[tokio::test]
18260    async fn unknown_op_returns_unknown_control_op() {
18261        let handler = ControlHandler::default();
18262        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
18263        let request = Frame::build(
18264            FrameType::Request,
18265            control_flags(),
18266            0,
18267            0,
18268            55,
18269            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
18270        )
18271        .unwrap();
18272
18273        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18274
18275        assert_eq!(response.len(), 1);
18276        assert_eq!(response[0].header.ty, FrameType::Error);
18277        assert_eq!(response[0].header.corr, 55);
18278        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
18279    }
18280
18281    #[tokio::test]
18282    async fn supervisor_provenance_rejects_unknown_exact_module() {
18283        let handler = ControlHandler::default();
18284        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
18285        let request = Frame::build(
18286            FrameType::Request,
18287            control_flags(),
18288            0,
18289            0,
18290            57,
18291            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
18292        )
18293        .unwrap();
18294
18295        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18296
18297        assert_eq!(response.len(), 1);
18298        assert_eq!(response[0].header.ty, FrameType::Error);
18299        assert_eq!(response[0].header.corr, 57);
18300        let error = parse_error(&response[0]);
18301        assert_eq!(error["code"], "unknown_module");
18302        assert_eq!(error["message"], "module_id 'missing' is not supervised");
18303    }
18304
18305    #[test]
18306    fn provenance_probe_override_keeps_handler_tests_deterministic() {
18307        let expected = subc_control::RunningImageAgreement::Unavailable {
18308            reason: subc_control::RunningImageUnavailableReason::HashFailed,
18309        };
18310        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
18311        assert_eq!(handler.provenance_probe_override, Some(expected));
18312    }
18313
18314    #[test]
18315    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
18316        let verdict = reload_verdict(
18317            std::path::Path::new("/bin/new"),
18318            Some(std::path::Path::new("/bin/old")),
18319            subc_control::RunningImageAgreement::Unavailable {
18320                reason: subc_control::RunningImageUnavailableReason::HashFailed,
18321            },
18322        );
18323        assert!(matches!(
18324            verdict.path,
18325            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
18326                if configured == std::path::Path::new("/bin/new")
18327                    && spawned_from == std::path::Path::new("/bin/old")
18328        ));
18329    }
18330
18331    #[test]
18332    fn launch_spec_without_running_digest_is_unavailable_not_match() {
18333        use subc_control::{
18334            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
18335        };
18336        let configured = crate::supervise::ModuleSpec {
18337            carry_containment: false,
18338            module_id: "legacy".into(),
18339            program: "/module".into(),
18340            args: Vec::new(),
18341            env: Vec::new(),
18342            reserved: false,
18343            reserved_prefixes: Vec::new(),
18344            protocol: subc_control::ModuleProtocol::Subc,
18345            overlap: Default::default(),
18346        };
18347        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
18348        assert_eq!(
18349            launch_spec_agreement(&configured, false, Some(&digest)),
18350            Agreement::Unavailable {
18351                reason: Reason::NotRunning
18352            }
18353        );
18354        assert_eq!(
18355            launch_spec_agreement(&configured, true, None),
18356            Agreement::Unavailable {
18357                reason: Reason::SpawnedSpecUnavailable
18358            }
18359        );
18360        assert_eq!(
18361            launch_spec_agreement(&configured, true, Some(&digest)),
18362            Agreement::Match
18363        );
18364    }
18365
18366    #[cfg(windows)]
18367    #[test]
18368    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
18369        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
18370        let configured = dir.join("module.exe");
18371        std::fs::write(&configured, b"executable path fixture").unwrap();
18372        let spawned = std::fs::canonicalize(&configured).unwrap();
18373        assert_ne!(
18374            configured, spawned,
18375            "fixture must exercise different path spellings"
18376        );
18377        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
18378        let lower_case = configured.to_string_lossy().to_lowercase();
18379        for spelling in [
18380            &configured,
18381            Path::new(&forward_slashes),
18382            Path::new(&lower_case),
18383        ] {
18384            let verdict = reload_verdict(
18385                spelling,
18386                Some(&spawned),
18387                subc_control::RunningImageAgreement::Unavailable {
18388                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18389                },
18390            );
18391            assert_eq!(
18392                verdict.path,
18393                ReloadPathAgreement::Match,
18394                "configured {spelling:?}, spawned {spawned:?}"
18395            );
18396        }
18397    }
18398
18399    #[cfg(windows)]
18400    #[test]
18401    fn windows_reload_path_keeps_different_existing_program_pending() {
18402        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
18403        let old = dir.join("old.exe");
18404        let new = dir.join("new.exe");
18405        std::fs::write(&old, b"same executable content").unwrap();
18406        std::fs::copy(&old, &new).unwrap();
18407        let spawned = std::fs::canonicalize(&old).unwrap();
18408        let image = subc_control::RunningImageAgreement::Unavailable {
18409            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18410        };
18411        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
18412        assert!(
18413            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
18414            if configured == new && spawned_from == spawned)
18415        );
18416        let missing = dir.join("missing.exe");
18417        assert!(matches!(
18418            reload_verdict(&missing, Some(&spawned), image).path,
18419            ReloadPathAgreement::Mismatch { .. }
18420        ));
18421    }
18422
18423    #[test]
18424    fn reload_verdict_detects_replaced_image_at_same_path() {
18425        let image = subc_control::RunningImageAgreement::Mismatch {
18426            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
18427                digest: "old".into(),
18428            },
18429            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
18430                digest: "new".into(),
18431            },
18432        };
18433        let verdict = reload_verdict(
18434            std::path::Path::new("/bin/same"),
18435            Some(std::path::Path::new("/bin/same")),
18436            image.clone(),
18437        );
18438        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18439        assert_eq!(verdict.image, image);
18440    }
18441
18442    #[test]
18443    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
18444        let image = subc_control::RunningImageAgreement::Unavailable {
18445            reason: subc_control::RunningImageUnavailableReason::NotRunning,
18446        };
18447        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
18448        assert_eq!(
18449            verdict.path,
18450            subc_control::ReloadPathAgreement::Unavailable {
18451                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
18452            }
18453        );
18454        assert_eq!(verdict.image, image);
18455
18456        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
18457            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18458        };
18459        let verdict = reload_verdict(
18460            std::path::Path::new("/bin/same"),
18461            Some(std::path::Path::new("/bin/same")),
18462            unconfirmed.clone(),
18463        );
18464        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18465        assert_eq!(verdict.image, unconfirmed);
18466    }
18467
18468    #[test]
18469    fn reload_verdict_preserves_each_image_unavailability_reason() {
18470        use subc_control::RunningImageUnavailableReason as Reason;
18471
18472        for reason in [
18473            Reason::NotRunning,
18474            Reason::UnsupportedPlatform,
18475            Reason::RunningExecutableUnreadable,
18476            Reason::SpawnedPathUnreadable,
18477            Reason::HashFailed,
18478            Reason::ProcessIdentityUnconfirmed,
18479            Reason::Unknown("future_probe_reason".to_string()),
18480        ] {
18481            let image = subc_control::RunningImageAgreement::Unavailable {
18482                reason: reason.clone(),
18483            };
18484            let verdict = reload_verdict(
18485                std::path::Path::new("/bin/same"),
18486                Some(std::path::Path::new("/bin/same")),
18487                image.clone(),
18488            );
18489            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18490            assert_eq!(verdict.image, image, "{reason:?}");
18491        }
18492    }
18493
18494    #[tokio::test]
18495    async fn malformed_control_bodies_return_invalid_control_body() {
18496        let handler = ControlHandler::default();
18497        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
18498
18499        for (corr, body) in [
18500            (56, br#"{"route_channel":1}"#.as_slice()),
18501            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
18502            (
18503                58,
18504                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
18505            ),
18506        ] {
18507            let request = Frame::build(
18508                FrameType::Request,
18509                control_flags(),
18510                0,
18511                0,
18512                corr,
18513                body.to_vec(),
18514            )
18515            .unwrap();
18516            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18517
18518            assert_eq!(response.len(), 1);
18519            assert_eq!(response[0].header.ty, FrameType::Error);
18520            assert_eq!(response[0].header.corr, corr);
18521            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
18522        }
18523    }
18524
18525    #[tokio::test]
18526    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
18527        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18528        let registry = Arc::new(Registry::default());
18529        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
18530        let router = Router::with_control_handler(Arc::clone(&control));
18531        let connection = router.begin_connection();
18532        let (ctx, mut rx) = route_ctx(connection.id());
18533
18534        router
18535            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
18536            .await
18537            .unwrap();
18538        let response = rx.recv().await.unwrap();
18539        let ack = parse_ack(&response);
18540        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18541        let channel = 1;
18542
18543        let goodbye =
18544            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
18545        router.route_for_connection(&ctx, goodbye).await.unwrap();
18546        assert!(rx.try_recv().is_err());
18547        assert!(registry.get_module("aft").unwrap().is_none());
18548
18549        router
18550            .route_for_connection(&ctx, channel_request(channel, 13))
18551            .await
18552            .unwrap();
18553        let error_frame = rx.recv().await.unwrap();
18554        assert_eq!(error_frame.header.ty, FrameType::Error);
18555        assert_eq!(error_frame.header.channel, channel);
18556        let captured = crate::router::test_log::captured_logs(&logs);
18557        assert_eq!(
18558            captured
18559                .lines()
18560                .filter(|line| {
18561                    line.contains("module_id=aft")
18562                        && line.contains("reason=explicit_goodbye")
18563                        && line.contains("module registration ended")
18564                })
18565                .count(),
18566            1,
18567            "unexpected GOODBYE registry log: {captured}"
18568        );
18569    }
18570
18571    #[tokio::test]
18572    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
18573        let registry = Arc::new(Registry::default());
18574        let handler = ControlHandler::new(registry).with_capability_config(
18575            [("prov".to_string(), true), ("cons".to_string(), true)],
18576            BTreeMap::new(),
18577        );
18578        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
18579        register_capability_manifest(
18580            &handler,
18581            &provider_ctx,
18582            &mut provider_rx,
18583            capability_manifest("prov", &["thing/v1"], &[]),
18584            1,
18585        )
18586        .await;
18587        let mut consumer = capability_manifest("cons", &[], &[]);
18588        consumer.capabilities.as_mut().unwrap().requires.push(
18589            subc_protocol::manifest::CapabilityRequirement {
18590                capability: "thing/v1".to_string(),
18591                need: subc_protocol::manifest::CapabilityNeed::Required,
18592            },
18593        );
18594        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
18595        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
18596        assert_eq!(
18597            handler.capability_evaluator.verdict("cons", "thing/v1"),
18598            Some(CapabilityVerdict::Provided)
18599        );
18600        let (mut client_rx, _) = open_route_for_capability_test(
18601            &handler,
18602            &provider_ctx,
18603            &mut provider_rx,
18604            703,
18605            3,
18606            "prov",
18607            None,
18608        )
18609        .await;
18610        let goodbye =
18611            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18612        handler
18613            .handle_control_frame(&provider_ctx, goodbye)
18614            .await
18615            .unwrap();
18616        assert_eq!(
18617            handler.capability_evaluator.verdict("cons", "thing/v1"),
18618            Some(CapabilityVerdict::NeverProvided)
18619        );
18620        let closed = client_rx
18621            .try_recv()
18622            .expect("GOODBYE pushes route.closed before route GOODBYE");
18623        assert!(
18624            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
18625            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
18626        );
18627        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
18628        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
18629    }
18630
18631    struct RouteClosedFixture {
18632        handler: ControlHandler,
18633        #[cfg(unix)]
18634        supervisor: Supervisor,
18635        module_ctx: RouteCtx,
18636        #[cfg(unix)]
18637        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18638        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18639        route: Arc<crate::forwarding::RouteBinding>,
18640    }
18641
18642    async fn route_closed_fixture() -> RouteClosedFixture {
18643        let registry = Arc::new(Registry::default());
18644        let handler = ControlHandler::new(Arc::clone(&registry));
18645        #[cfg(unix)]
18646        let supervisor = Supervisor::new(registry, RestartPolicy::default())
18647            .with_forwarding(Arc::clone(&handler.forwarding));
18648        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
18649        register_capability_manifest(
18650            &handler,
18651            &module_ctx,
18652            &mut module_rx,
18653            capability_manifest("prov", &[], &[]),
18654            1,
18655        )
18656        .await;
18657        let (client_rx, bind) = open_route_for_capability_test(
18658            &handler,
18659            &module_ctx,
18660            &mut module_rx,
18661            702,
18662            2,
18663            "prov",
18664            None,
18665        )
18666        .await;
18667        let ModuleControlRequest::RouteBind {
18668            route_channel: channel,
18669            epoch: route_epoch,
18670            ..
18671        } = bind
18672        else {
18673            panic!("expected route.bind");
18674        };
18675        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
18676            handler
18677                .forwarding
18678                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
18679                .unwrap()
18680        else {
18681            panic!("the acknowledged route must be bound");
18682        };
18683        RouteClosedFixture {
18684            handler,
18685            #[cfg(unix)]
18686            supervisor,
18687            module_ctx,
18688            #[cfg(unix)]
18689            module_rx,
18690            client_rx,
18691            route,
18692        }
18693    }
18694
18695    impl RouteClosedFixture {
18696        #[cfg(unix)]
18697        async fn cut(&mut self) {
18698            let notices = async {
18699                let mut draining = self.module_rx.recv().await.unwrap();
18700                assert!(matches!(
18701                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
18702                        &draining.body
18703                    )
18704                    .unwrap(),
18705                    subc_protocol::session::ModuleControlCommand::Draining {
18706                        reason: RouteCloseReason::Disable,
18707                        ..
18708                    }
18709                ));
18710                draining.flushed.take().unwrap().send(()).unwrap();
18711                let mut closing = self.client_rx.recv().await.unwrap();
18712                assert!(matches!(
18713                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
18714                    ClientControlPush::RouteClosing {
18715                        reason: RouteCloseReason::Restart,
18716                        ..
18717                    }
18718                ));
18719                closing.flushed.take().unwrap().send(()).unwrap();
18720            };
18721            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
18722            result.unwrap();
18723        }
18724
18725        fn disconnect(&mut self) -> ClientControlPush {
18726            self.handler
18727                .cleanup_connection(self.module_ctx.connection_id)
18728                .unwrap();
18729            let closed = self.client_rx.try_recv().unwrap();
18730            assert_eq!(closed.header.ty, FrameType::Push);
18731            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
18732            assert!(matches!(
18733                &push,
18734                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
18735                    if channels == &[self.route.client_channel]
18736            ));
18737            assert_eq!(
18738                self.client_rx.try_recv().unwrap().header.ty,
18739                FrameType::Goodbye
18740            );
18741            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
18742            push
18743        }
18744    }
18745
18746    #[cfg(unix)]
18747    #[tokio::test(start_paused = true)]
18748    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
18749        let mut fixture = route_closed_fixture().await;
18750        fixture.cut().await;
18751        assert!(matches!(
18752            fixture.disconnect(),
18753            ClientControlPush::RouteClosed {
18754                reason: RouteCloseReason::Restart,
18755                drained: true,
18756                ..
18757            }
18758        ));
18759    }
18760
18761    #[cfg(unix)]
18762    #[tokio::test(start_paused = true)]
18763    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
18764        let mut fixture = route_closed_fixture().await;
18765        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
18766        let started = tokio::time::Instant::now();
18767        fixture.cut().await;
18768        assert!(started.elapsed() >= Duration::from_secs(2));
18769        assert_eq!(fixture.route.flow.in_flight(), 1);
18770        assert!(matches!(
18771            fixture.disconnect(),
18772            ClientControlPush::RouteClosed {
18773                reason: RouteCloseReason::Restart,
18774                drained: false,
18775                ..
18776            }
18777        ));
18778    }
18779
18780    #[tokio::test]
18781    async fn module_connection_crash_reports_crash_and_not_drained() {
18782        let mut fixture = route_closed_fixture().await;
18783        // The server calls cleanup_connection on module EOF. Without a planned
18784        // daemon shutdown, that disconnect must remain a crash with no prior
18785        // route.closing notice, even when the route has no in-flight calls.
18786        assert!(matches!(
18787            fixture.disconnect(),
18788            ClientControlPush::RouteClosed {
18789                reason: RouteCloseReason::Crash,
18790                drained: false,
18791                ..
18792            }
18793        ));
18794    }
18795
18796    #[tokio::test]
18797    async fn dropping_router_connection_releases_registration() {
18798        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18799        let registry = Arc::new(Registry::default());
18800        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
18801        let router = Router::with_control_handler(Arc::clone(&control));
18802        let connection = router.begin_connection();
18803        let connection_id = connection.id();
18804        let (ctx, mut rx) = route_ctx(connection_id);
18805
18806        router
18807            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
18808            .await
18809            .unwrap();
18810        let response = rx.recv().await.unwrap();
18811        let ack = parse_ack(&response);
18812        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18813        assert!(registry.get_module("aft").unwrap().is_some());
18814
18815        drop(connection);
18816
18817        assert!(registry.get_module("aft").unwrap().is_none());
18818        assert_eq!(registry.active_registration_count().unwrap(), 0);
18819
18820        control
18821            .cleanup_connection(ConnectionId::new(u64::MAX))
18822            .unwrap();
18823        let captured = crate::router::test_log::captured_logs(&logs);
18824        println!("captured registry lifecycle logs:\n{captured}");
18825        let events: Vec<_> = captured
18826            .lines()
18827            .filter(|line| {
18828                line.contains("module registered module_id=aft ")
18829                    || line.contains("module registration ended")
18830            })
18831            .collect();
18832        assert_eq!(
18833            events.len(),
18834            2,
18835            "unexpected registry lifecycle logs: {captured}"
18836        );
18837        assert!(events[0].contains("module registered module_id=aft "));
18838        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
18839        assert!(events[1].contains(&format!(
18840            "module_id=aft connection_id={}",
18841            connection_id.get()
18842        )));
18843        assert!(events[1].contains("reason=connection_closed"));
18844        assert!(events[1].contains("module registration ended"));
18845        assert_eq!(
18846            captured
18847                .lines()
18848                .filter(|line| line.contains("module registered module_id=aft "))
18849                .count(),
18850            1,
18851            "legacy registration admission line must appear once: {captured}"
18852        );
18853    }
18854
18855    #[tokio::test]
18856    async fn hello_registration_keeps_legacy_module_registered_line_once() {
18857        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18858        let registry = Arc::new(Registry::default());
18859        let control = ControlHandler::new(Arc::clone(&registry));
18860        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
18861        hello_via_sink(
18862            &control,
18863            &ctx,
18864            &mut rx,
18865            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
18866        )
18867        .await;
18868
18869        let captured = crate::router::test_log::captured_logs(&logs);
18870        let admissions: Vec<_> = captured
18871            .lines()
18872            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
18873            .collect();
18874        assert_eq!(
18875            admissions.len(),
18876            1,
18877            "legacy admission line must remain exactly once: {captured}"
18878        );
18879        assert!(admissions[0].contains("routable_provider=true"));
18880        assert!(admissions[0].contains("connection_id=777"));
18881    }
18882
18883    fn capability_manifest(
18884        module_id: &str,
18885        provides: &[&str],
18886        must_never_reach: &[&str],
18887    ) -> ModuleManifest {
18888        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
18889        manifest.capabilities = Some(CapabilityDeclarations {
18890            provides: provides
18891                .iter()
18892                .map(|capability| (*capability).to_string())
18893                .collect(),
18894            requires: Vec::new(),
18895            must_never_reach: must_never_reach
18896                .iter()
18897                .map(|capability| (*capability).to_string())
18898                .collect(),
18899        });
18900        manifest
18901    }
18902
18903    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
18904        Frame::build(
18905            FrameType::Hello,
18906            control_flags(),
18907            0,
18908            0,
18909            corr,
18910            serde_json::to_vec(&ModuleHelloBody {
18911                protocol_ver: manifest.protocol_ver,
18912                manifest,
18913                control_ops: None,
18914                launch_nonce: None,
18915            })
18916            .expect("capability test HELLO serializes"),
18917        )
18918        .expect("capability test HELLO frame builds")
18919    }
18920
18921    fn catalog_update_with_capabilities_frame(
18922        corr: u64,
18923        capabilities: CapabilityDeclarations,
18924    ) -> Frame {
18925        Frame::build(
18926            FrameType::Request,
18927            control_flags(),
18928            0,
18929            0,
18930            corr,
18931            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
18932                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
18933                capabilities: Some(capabilities),
18934                ready: None,
18935            })
18936            .expect("capability catalog.update serializes"),
18937        )
18938        .expect("capability catalog.update frame builds")
18939    }
18940
18941    async fn register_capability_manifest(
18942        handler: &ControlHandler,
18943        ctx: &RouteCtx,
18944        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
18945        manifest: ModuleManifest,
18946        corr: u64,
18947    ) {
18948        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
18949    }
18950
18951    async fn open_route_for_capability_test(
18952        handler: &ControlHandler,
18953        target_ctx: &RouteCtx,
18954        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
18955        client_connection_id: u64,
18956        corr: u64,
18957        target_module_id: &str,
18958        consumer_identity: Option<ConsumerIdentity>,
18959    ) -> (
18960        mpsc::Receiver<crate::router::OutboundFrame>,
18961        ModuleControlRequest,
18962    ) {
18963        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
18964        let route_handler = handler.clone();
18965        let target_module_id = target_module_id.to_string();
18966        let route_task = tokio::spawn(async move {
18967            route_handler
18968                .handle_control_frame(
18969                    &client_ctx,
18970                    route_open_frame_with_admission_facts(
18971                        corr,
18972                        &target_module_id,
18973                        unique_project_root("admission-facts"),
18974                        consumer_identity,
18975                        None,
18976                    ),
18977                )
18978                .await
18979                .expect("capability test route.open succeeds")
18980        });
18981        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
18982            .await
18983            .expect("capability test route.open must reach route.bind")
18984            .expect("target control receiver stays open");
18985        let bind_request: ModuleControlRequest =
18986            serde_json::from_slice(&bind.body).expect("route.bind decodes");
18987        handler
18988            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
18989            .await
18990            .expect("capability test route.bind ACK succeeds");
18991        assert!(route_task.await.expect("route.open task joins").is_empty());
18992        let opened = client_rx
18993            .recv()
18994            .await
18995            .expect("successful route.open publishes a response");
18996        assert!(matches!(
18997            serde_json::from_slice::<ClientControlResponse>(&opened.body),
18998            Ok(ClientControlResponse::RouteOpen { .. })
18999        ));
19000        (client_rx, bind_request)
19001    }
19002
19003    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
19004        assert_eq!(frame.header.ty, FrameType::Push);
19005        assert_eq!(frame.header.channel, 0);
19006        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
19007            .expect("route.closed control push decodes");
19008        let ClientControlPush::RouteClosed { channels, .. } = &push else {
19009            panic!("expected route.closed");
19010        };
19011        assert_eq!(channels.len(), 1, "exactly one violating route closed");
19012        let closed_channel = channels[0];
19013        let channels = channels.clone();
19014        assert_eq!(
19015            push,
19016            ClientControlPush::RouteClosed {
19017                module_id: target_module_id.to_string(),
19018                channels,
19019                reason: RouteCloseReason::CapabilityDenied,
19020                drained: false,
19021                abandoned: 0,
19022                excluded_subscriptions: 0,
19023                terminal: Some(false),
19024            }
19025        );
19026        closed_channel
19027    }
19028
19029    #[tokio::test]
19030    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
19031        let registry = Arc::new(Registry::default());
19032        let forwarding = Arc::new(ForwardingTable::default());
19033        let supervisor = SupervisorHandle::new();
19034        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19035        let handler =
19036            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19037                .with_supervisor(supervisor);
19038        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
19039        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
19040        register_capability_manifest(
19041            &handler,
19042            &target_ctx,
19043            &mut target_rx,
19044            capability_manifest("target", &["credentials-provider/v1"], &[]),
19045            1,
19046        )
19047        .await;
19048        register_capability_manifest(
19049            &handler,
19050            &opener_ctx,
19051            &mut opener_rx,
19052            capability_manifest("opener", &[], &["credentials-provider/v1"]),
19053            2,
19054        )
19055        .await;
19056
19057        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
19058        let replies = handler
19059            .handle_control_frame(
19060                &client_ctx,
19061                route_open_frame_with_admission_facts(
19062                    3,
19063                    "target",
19064                    unique_project_root("admission-facts"),
19065                    Some(ConsumerIdentity {
19066                        module_id: "opener".to_string(),
19067                        launch_nonce: "opener-nonce".to_string(),
19068                    }),
19069                    None,
19070                ),
19071            )
19072            .await
19073            .expect("denied route.open returns a typed frame");
19074        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19075        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19076        assert!(
19077            target_rx.try_recv().is_err(),
19078            "forbidden route.open must not relay route.bind"
19079        );
19080    }
19081
19082    #[tokio::test]
19083    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
19084        let registry = Arc::new(Registry::default());
19085        let forwarding = Arc::new(ForwardingTable::default());
19086        let supervisor = SupervisorHandle::new();
19087        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19088        let handler =
19089            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19090                .with_supervisor(supervisor);
19091        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
19092        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
19093        register_capability_manifest(
19094            &handler,
19095            &target_ctx,
19096            &mut target_rx,
19097            capability_manifest("target", &["credentials-provider/v1"], &[]),
19098            1,
19099        )
19100        .await;
19101        register_capability_manifest(
19102            &handler,
19103            &old_opener_ctx,
19104            &mut old_opener_rx,
19105            capability_manifest("opener", &[], &[]),
19106            2,
19107        )
19108        .await;
19109        let (mut client_rx, _) = open_route_for_capability_test(
19110            &handler,
19111            &target_ctx,
19112            &mut target_rx,
19113            712,
19114            3,
19115            "target",
19116            Some(ConsumerIdentity {
19117                module_id: "opener".to_string(),
19118                launch_nonce: "opener-nonce".to_string(),
19119            }),
19120        )
19121        .await;
19122        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19123
19124        handler
19125            .cleanup_connection(old_opener_ctx.connection_id)
19126            .expect("old opener registration cleans up");
19127        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
19128        register_capability_manifest(
19129            &handler,
19130            &new_opener_ctx,
19131            &mut new_opener_rx,
19132            capability_manifest("opener", &[], &["credentials-provider/v1"]),
19133            4,
19134        )
19135        .await;
19136
19137        let closed_channel = assert_capability_denied_push(
19138            client_rx
19139                .try_recv()
19140                .expect("HELLO deny addition must emit route.closed")
19141                .frame,
19142            "target",
19143        );
19144        let goodbye = client_rx
19145            .try_recv()
19146            .expect("HELLO deny addition must send the client a route GOODBYE")
19147            .frame;
19148        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19149        assert_eq!(goodbye.header.channel, closed_channel);
19150        assert_ne!(goodbye.header.channel, 0);
19151        assert_ne!(goodbye.header.epoch, 0);
19152        assert!(client_rx.try_recv().is_err());
19153        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19154        assert!(matches!(
19155            target_rx.try_recv(),
19156            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19157        ));
19158    }
19159
19160    #[tokio::test]
19161    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
19162        let registry = Arc::new(Registry::default());
19163        let forwarding = Arc::new(ForwardingTable::default());
19164        let supervisor = SupervisorHandle::new();
19165        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19166        let handler =
19167            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19168                .with_supervisor(supervisor);
19169        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
19170        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
19171        register_capability_manifest(
19172            &handler,
19173            &target_ctx,
19174            &mut target_rx,
19175            capability_manifest("target", &[], &[]),
19176            1,
19177        )
19178        .await;
19179        register_capability_manifest(
19180            &handler,
19181            &opener_ctx,
19182            &mut opener_rx,
19183            capability_manifest("opener", &[], &["credentials-provider/v1"]),
19184            2,
19185        )
19186        .await;
19187        let (mut client_rx, _) = open_route_for_capability_test(
19188            &handler,
19189            &target_ctx,
19190            &mut target_rx,
19191            722,
19192            3,
19193            "target",
19194            Some(ConsumerIdentity {
19195                module_id: "opener".to_string(),
19196                launch_nonce: "opener-nonce".to_string(),
19197            }),
19198        )
19199        .await;
19200        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19201
19202        let replies = handler
19203            .handle_control_frame(
19204                &target_ctx,
19205                catalog_update_with_capabilities_frame(
19206                    4,
19207                    CapabilityDeclarations {
19208                        provides: vec!["credentials-provider/v1".to_string()],
19209                        requires: Vec::new(),
19210                        must_never_reach: Vec::new(),
19211                    },
19212                ),
19213            )
19214            .await
19215            .expect("claim catalog.update succeeds");
19216        assert!(matches!(
19217            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
19218            Ok(ModuleControlResponseToModule::CatalogUpdate {})
19219        ));
19220        let closed_channel = assert_capability_denied_push(
19221            client_rx
19222                .try_recv()
19223                .expect("claim addition must emit route.closed")
19224                .frame,
19225            "target",
19226        );
19227        let goodbye = client_rx
19228            .try_recv()
19229            .expect("claim addition must send the client a route GOODBYE")
19230            .frame;
19231        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19232        assert_eq!(goodbye.header.channel, closed_channel);
19233        assert_ne!(goodbye.header.channel, 0);
19234        assert_ne!(goodbye.header.epoch, 0);
19235        assert!(client_rx.try_recv().is_err());
19236        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19237        assert!(matches!(
19238            target_rx.try_recv(),
19239            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19240        ));
19241    }
19242
19243    #[tokio::test]
19244    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
19245        let registry = Arc::new(Registry::default());
19246        let forwarding = Arc::new(ForwardingTable::default());
19247        let supervisor = SupervisorHandle::new();
19248        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19249        let handler =
19250            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19251                .with_supervisor(supervisor);
19252        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
19253        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
19254        register_capability_manifest(
19255            &handler,
19256            &target_ctx,
19257            &mut target_rx,
19258            capability_manifest("target", &["credentials-provider/v1"], &[]),
19259            1,
19260        )
19261        .await;
19262        register_capability_manifest(
19263            &handler,
19264            &opener_ctx,
19265            &mut opener_rx,
19266            capability_manifest("opener", &[], &[]),
19267            2,
19268        )
19269        .await;
19270        let (mut client_rx, _) = open_route_for_capability_test(
19271            &handler,
19272            &target_ctx,
19273            &mut target_rx,
19274            732,
19275            3,
19276            "target",
19277            Some(ConsumerIdentity {
19278                module_id: "opener".to_string(),
19279                launch_nonce: "opener-nonce".to_string(),
19280            }),
19281        )
19282        .await;
19283        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19284
19285        handler
19286            .handle_control_frame(
19287                &target_ctx,
19288                catalog_update_with_capabilities_frame(
19289                    4,
19290                    CapabilityDeclarations {
19291                        provides: Vec::new(),
19292                        requires: Vec::new(),
19293                        must_never_reach: Vec::new(),
19294                    },
19295                ),
19296            )
19297            .await
19298            .expect("claim removal catalog.update succeeds");
19299        assert_eq!(
19300            forwarding.active_binding_count().unwrap(),
19301            1,
19302            "removing an attested target claim must leave the route census unchanged"
19303        );
19304        assert!(
19305            client_rx.try_recv().is_err(),
19306            "claim removal must not emit route.closed or a client route GOODBYE"
19307        );
19308        assert!(
19309            target_rx.try_recv().is_err(),
19310            "claim removal must not send the target a route GOODBYE"
19311        );
19312    }
19313
19314    /// A direct client may open a route to a denied capability provider; this
19315    /// policy applies only to attested supervised module origins, not to direct clients.
19316    #[tokio::test]
19317    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
19318        let registry = Arc::new(Registry::default());
19319        let forwarding = Arc::new(ForwardingTable::default());
19320        let supervisor = SupervisorHandle::new();
19321        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19322        let handler =
19323            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19324                .with_supervisor(supervisor);
19325        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
19326        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
19327        register_capability_manifest(
19328            &handler,
19329            &target_ctx,
19330            &mut target_rx,
19331            capability_manifest("target", &["credentials-provider/v1"], &[]),
19332            1,
19333        )
19334        .await;
19335        register_capability_manifest(
19336            &handler,
19337            &opener_ctx,
19338            &mut opener_rx,
19339            capability_manifest("opener", &[], &["credentials-provider/v1"]),
19340            2,
19341        )
19342        .await;
19343
19344        let (_client_rx, bind) = open_route_for_capability_test(
19345            &handler,
19346            &target_ctx,
19347            &mut target_rx,
19348            742,
19349            3,
19350            "target",
19351            None,
19352        )
19353        .await;
19354        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
19355            panic!("direct scope-honesty route must bind");
19356        };
19357        assert_eq!(principal, Some(Principal::Direct));
19358        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19359    }
19360
19361    /// A module that denies a capability receives no self-route exemption when it
19362    /// also attestedly provides that capability.
19363    #[tokio::test]
19364    async fn must_never_reach_self_route_is_capability_forbidden() {
19365        let registry = Arc::new(Registry::default());
19366        let forwarding = Arc::new(ForwardingTable::default());
19367        let supervisor = SupervisorHandle::new();
19368        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
19369        let handler =
19370            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19371                .with_supervisor(supervisor);
19372        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
19373        register_capability_manifest(
19374            &handler,
19375            &self_ctx,
19376            &mut self_rx,
19377            capability_manifest(
19378                "self-provider",
19379                &["credentials-provider/v1"],
19380                &["credentials-provider/v1"],
19381            ),
19382            1,
19383        )
19384        .await;
19385
19386        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
19387        let replies = handler
19388            .handle_control_frame(
19389                &client_ctx,
19390                route_open_frame_with_admission_facts(
19391                    2,
19392                    "self-provider",
19393                    unique_project_root("admission-facts"),
19394                    Some(ConsumerIdentity {
19395                        module_id: "self-provider".to_string(),
19396                        launch_nonce: "self-nonce".to_string(),
19397                    }),
19398                    None,
19399                ),
19400            )
19401            .await
19402            .expect("self-route refusal returns a typed frame");
19403        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19404        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19405        assert!(
19406            self_rx.try_recv().is_err(),
19407            "self denial must not relay route.bind"
19408        );
19409    }
19410
19411    #[test]
19412    fn unsupported_channel_zero_frame_returns_error() {
19413        let handler = ControlHandler::default();
19414        let request = Frame::build(
19415            FrameType::Request,
19416            control_flags(),
19417            0,
19418            0,
19419            21,
19420            b"opaque".to_vec(),
19421        )
19422        .unwrap();
19423
19424        let response = handler
19425            .handle_control(ConnectionId::new(1), request)
19426            .unwrap();
19427
19428        assert_eq!(response[0].header.ty, FrameType::Error);
19429        assert_eq!(
19430            parse_error(&response[0])["code"],
19431            "unsupported_control_frame"
19432        );
19433    }
19434
19435    /// Blue/green swap at the control-plane boundary. The supervisor that opens
19436    /// a swap is not wired yet, so the candidate is registered here directly
19437    /// into the registry and forwarding candidate slots, the way the swap's
19438    /// HELLO admission will.
19439    mod swap {
19440        use super::*;
19441
19442        const INCUMBENT: ConnectionId = ConnectionId::new(30);
19443        const CANDIDATE: ConnectionId = ConnectionId::new(40);
19444
19445        struct Swap {
19446            registry: Arc<Registry>,
19447            forwarding: Arc<ForwardingTable>,
19448            handler: ControlHandler,
19449            incumbent_ctx: RouteCtx,
19450            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19451            candidate_ctx: RouteCtx,
19452            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19453        }
19454
19455        async fn swap_with_incumbent() -> Swap {
19456            let registry = Arc::new(Registry::default());
19457            let forwarding = Arc::new(ForwardingTable::default());
19458            let handler =
19459                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
19460            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
19461            hello_via_sink(
19462                &handler,
19463                &incumbent_ctx,
19464                &mut incumbent_rx,
19465                hello_frame("aft", PROTOCOL_VERSION, 7),
19466            )
19467            .await;
19468            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
19469            Swap {
19470                registry,
19471                forwarding,
19472                handler,
19473                incumbent_ctx,
19474                incumbent_rx,
19475                candidate_ctx,
19476                candidate_rx,
19477            }
19478        }
19479
19480        fn register_candidate(swap: &Swap, ready: Option<bool>) {
19481            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
19482            candidate_manifest.ready = ready;
19483            let registration = swap
19484                .registry
19485                .register_candidate_with_control_ops(
19486                    candidate_manifest,
19487                    PROTOCOL_VERSION,
19488                    CANDIDATE,
19489                    module_baseline_control_ops(),
19490                )
19491                .unwrap();
19492            swap.forwarding
19493                .register_candidate_module_connection(
19494                    CANDIDATE,
19495                    "aft".to_string(),
19496                    PROTOCOL_VERSION,
19497                    manifest_concurrency(&registration.manifest),
19498                    swap.candidate_ctx.egress.clone(),
19499                )
19500                .unwrap();
19501        }
19502
19503        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
19504            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
19505            swap.registry.promote_candidate("aft").unwrap().unwrap();
19506            cutover.incumbent.unwrap()
19507        }
19508
19509        /// A swap candidate's HELLO reserves its candidate registry entry
19510        /// before installing its candidate forwarding endpoint. The swap
19511        /// supervisor promotes a ready candidate as soon as it reads it as
19512        /// registered, and promotion cuts forwarding over to the candidate
19513        /// endpoint, so the lookup the supervisor reads
19514        /// (`Registry::published_registration`) must not return the candidate
19515        /// until that endpoint exists. Otherwise the cutover finds no candidate
19516        /// endpoint and the swap fails as if the candidate had disconnected.
19517        #[tokio::test]
19518        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
19519            let mut swap = swap_with_incumbent().await;
19520            let hold = Arc::new(std::sync::Barrier::new(2));
19521            let handler = swap
19522                .handler
19523                .clone()
19524                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
19525            handler
19526                .supervisor
19527                .open_swap("aft", "candidate-nonce".to_string());
19528            let hello = {
19529                let handler = handler.clone();
19530                let candidate_ctx = swap.candidate_ctx.clone();
19531                std::thread::spawn(move || {
19532                    handler
19533                        .handle_hello(
19534                            candidate_ctx.connection_id,
19535                            Some(candidate_ctx.egress.clone()),
19536                            hello_frame_with_nonce(
19537                                "aft",
19538                                PROTOCOL_VERSION,
19539                                8,
19540                                Some("candidate-nonce"),
19541                            ),
19542                        )
19543                        .unwrap()
19544                })
19545            };
19546
19547            hold.wait();
19548            // Parked: the candidate slot is claimed but not published, and
19549            // forwarding has no candidate endpoint.
19550            assert!(swap
19551                .registry
19552                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
19553                .unwrap()
19554                .is_some());
19555            assert!(swap
19556                .registry
19557                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19558                .unwrap()
19559                .is_none());
19560            assert!(swap
19561                .forwarding
19562                .module_endpoint_for_connection(CANDIDATE)
19563                .unwrap()
19564                .is_none());
19565
19566            hold.wait();
19567            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
19568            assert_eq!(
19569                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19570                FrameType::HelloAck
19571            );
19572            let published = swap
19573                .registry
19574                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19575                .unwrap()
19576                .expect("candidate is published once its HELLO completes");
19577            assert_eq!(published.connection_id, CANDIDATE);
19578            assert!(swap
19579                .forwarding
19580                .module_endpoint_for_connection(CANDIDATE)
19581                .unwrap()
19582                .is_some());
19583        }
19584
19585        #[tokio::test]
19586        async fn candidate_lookup_after_hello_ack_waits_for_publication() {
19587            let mut swap = swap_with_incumbent().await;
19588            let hold = Arc::new(std::sync::Barrier::new(2));
19589            swap.handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
19590            swap.handler
19591                .supervisor
19592                .open_swap("aft", "candidate-nonce".to_string());
19593            let hello = {
19594                let handler = swap.handler.clone();
19595                let candidate_ctx = swap.candidate_ctx.clone();
19596                std::thread::spawn(move || {
19597                    handler
19598                        .handle_hello(
19599                            CANDIDATE,
19600                            Some(candidate_ctx.egress),
19601                            hello_frame_with_nonce(
19602                                "aft",
19603                                PROTOCOL_VERSION,
19604                                8,
19605                                Some("candidate-nonce"),
19606                            ),
19607                        )
19608                        .unwrap()
19609                })
19610            };
19611            hold.wait();
19612            assert_eq!(
19613                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19614                FrameType::HelloAck
19615            );
19616            let reader_registry = Arc::clone(&swap.registry);
19617            let published = lookup_during_hello_publication(&swap.registry, &hold, move || {
19618                reader_registry
19619                    .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19620                    .unwrap()
19621            });
19622            assert!(hello.join().unwrap().is_empty());
19623            assert_eq!(
19624                published
19625                    .expect("ACK makes candidate publication visible")
19626                    .connection_id,
19627                CANDIDATE
19628            );
19629            assert_eq!(
19630                swap.registry.generation().unwrap(),
19631                1,
19632                "candidate is not a second catalog entry"
19633            );
19634        }
19635
19636        #[tokio::test]
19637        async fn audit4_ungated_removed_candidate_publication_rolls_back_admission() {
19638            let mut swap = swap_with_incumbent().await;
19639            let hold = Arc::new(std::sync::Barrier::new(2));
19640            let handler = swap
19641                .handler
19642                .clone()
19643                .with_hello_registry_to_forwarding_hold(hold.clone());
19644            handler
19645                .supervisor
19646                .open_swap("aft", "candidate-nonce".into());
19647            let hello = {
19648                let handler = handler.clone();
19649                let ctx = swap.candidate_ctx.clone();
19650                std::thread::spawn(move || {
19651                    handler
19652                        .handle_hello(
19653                            CANDIDATE,
19654                            Some(ctx.egress),
19655                            hello_frame_with_nonce(
19656                                "aft",
19657                                PROTOCOL_VERSION,
19658                                8,
19659                                Some("candidate-nonce"),
19660                            ),
19661                        )
19662                        .unwrap()
19663                })
19664            };
19665            hold.wait();
19666            assert_eq!(
19667                swap.registry
19668                    .deregister_connection(CANDIDATE)
19669                    .unwrap()
19670                    .len(),
19671                1
19672            );
19673            hold.wait();
19674            assert!(hello.join().unwrap().is_empty());
19675            assert_eq!(
19676                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19677                FrameType::HelloAck
19678            );
19679            assert!(swap.registry.get_candidate("aft").unwrap().is_none());
19680            assert_eq!(
19681                handler
19682                    .supervisor
19683                    .swap_hello_admission("aft", Some("candidate-nonce")),
19684                SwapHelloAdmission::Candidate,
19685                "failed publication must roll back candidate_admitted for an ungated nonce swap"
19686            );
19687            handler.supervisor.close_swap("aft");
19688        }
19689
19690        fn keyed_total(counters: &Value, key: &str) -> u64 {
19691            counters[key]
19692                .as_object()
19693                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
19694                .unwrap_or(0)
19695        }
19696
19697        #[tokio::test]
19698        async fn replacement_logs_old_end_and_new_admission_once() {
19699            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
19700            let mut swap = swap_with_incumbent().await;
19701            swap.handler
19702                .supervisor
19703                .open_swap("aft", "candidate-nonce".to_string());
19704            hello_via_sink(
19705                &swap.handler,
19706                &swap.candidate_ctx,
19707                &mut swap.candidate_rx,
19708                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
19709            )
19710            .await;
19711            cutover(&swap);
19712            swap.handler.cleanup_connection(INCUMBENT).unwrap();
19713
19714            let captured = crate::router::test_log::captured_logs(&logs);
19715            println!("captured replacement registry logs:\n{captured}");
19716            let new_admissions: Vec<_> = captured
19717                .lines()
19718                .filter(|line| {
19719                    line.contains("connection_id=40")
19720                        && line.contains("swap candidate registered; not routable until cutover")
19721                })
19722                .collect();
19723            assert_eq!(
19724                new_admissions.len(),
19725                1,
19726                "unexpected admission logs: {captured}"
19727            );
19728            assert!(new_admissions[0].contains("module_id=aft"));
19729            assert!(new_admissions[0].contains("connection_id=40"));
19730            assert!(
19731                new_admissions[0].contains("swap candidate registered; not routable until cutover")
19732            );
19733            assert!(new_admissions[0].contains("ready=true"));
19734
19735            let old_admissions: Vec<_> = captured
19736                .lines()
19737                .filter(|line| {
19738                    line.contains("module registered module_id=aft ")
19739                        && line.contains("connection_id=30")
19740                })
19741                .collect();
19742            assert_eq!(
19743                old_admissions.len(),
19744                1,
19745                "unexpected incumbent admission: {captured}"
19746            );
19747
19748            let old_ends: Vec<_> = captured
19749                .lines()
19750                .filter(|line| {
19751                    line.contains("connection_id=30") && line.contains("module registration ended")
19752                })
19753                .collect();
19754            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
19755            assert!(old_ends[0].contains("module_id=aft"));
19756            assert!(old_ends[0].contains("connection_id=30"));
19757            assert!(old_ends[0].contains("reason=replaced"));
19758            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
19759            assert_eq!(
19760                captured
19761                    .lines()
19762                    .filter(|line| line.contains("module registration promoted"))
19763                    .count(),
19764                1,
19765                "unexpected promotion logs: {captured}"
19766            );
19767        }
19768
19769        /// An ack from the incumbent for a bind it was sent before cutover,
19770        /// arriving before the incumbent is drained. The incumbent is the live
19771        /// connection carrying every other client's routes, so the ack must
19772        /// not end it: the waiting client is told to retry, the reservation is
19773        /// given back, and the incumbent is told to drop just that binding.
19774        #[tokio::test]
19775        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
19776            let mut swap = swap_with_incumbent().await;
19777            let handler = swap.handler.clone();
19778
19779            // A co-tenant route, bound on the incumbent before the swap.
19780            let cotenant = ConnectionId::new(31);
19781            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
19782            let (cotenant_task, cotenant_bind) = relay_route_open(
19783                &handler,
19784                cotenant,
19785                &cotenant_ctx.egress,
19786                &mut swap.incumbent_rx,
19787                100,
19788                "aft",
19789                "swap-cotenant",
19790            )
19791            .await;
19792            handler
19793                .handle_control_frame(
19794                    &swap.incumbent_ctx,
19795                    route_bind_ack(cotenant_bind.header.corr),
19796                )
19797                .await
19798                .unwrap();
19799            assert!(cotenant_task.await.unwrap().is_empty());
19800            let (cotenant_channel, cotenant_epoch) =
19801                published_route(&cotenant_rx.recv().await.unwrap());
19802
19803            // A second route.open, relayed to the incumbent and not yet acked.
19804            let caller = ConnectionId::new(32);
19805            let (caller_ctx, mut caller_rx) = route_ctx(caller);
19806            let (caller_task, caller_bind) = relay_route_open(
19807                &handler,
19808                caller,
19809                &caller_ctx.egress,
19810                &mut swap.incumbent_rx,
19811                101,
19812                "aft",
19813                "swap-caller",
19814            )
19815            .await;
19816            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
19817
19818            register_candidate(&swap, None);
19819            cutover(&swap);
19820
19821            // The incumbent acks after promotion and before any drain.
19822            let ack = handler
19823                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
19824                .await;
19825            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
19826            if module_loop_error.is_some() {
19827                // What the connection loop does with an untranslated router
19828                // error: end the connection, releasing every route on it.
19829                handler.cleanup_connection(INCUMBENT).unwrap();
19830            }
19831
19832            // 1. The incumbent's other routes survive.
19833            assert!(
19834                cotenant_rx.try_recv().is_err(),
19835                "the co-tenant route on the incumbent was torn down by one late ack: \
19836                 {module_loop_error:?}"
19837            );
19838            assert!(matches!(
19839                swap.forwarding
19840                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
19841                    .unwrap(),
19842                DataRoute::Client(DataRouteState::Bound(_))
19843            ));
19844            assert_eq!(module_loop_error, None);
19845            assert!(swap
19846                .registry
19847                .get_module_by_connection(INCUMBENT)
19848                .unwrap()
19849                .is_some());
19850
19851            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
19852            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
19853                .await
19854                .expect("the incumbent is told to drop the abandoned binding")
19855                .unwrap()
19856                .frame;
19857            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19858            assert_eq!(goodbye.header.channel, abandoned_channel);
19859            assert_eq!(goodbye.header.epoch, abandoned_epoch);
19860            assert!(swap.incumbent_rx.try_recv().is_err());
19861
19862            // 3. The waiting client gets a retryable refusal and no route.
19863            let response = caller_task.await.unwrap();
19864            assert_eq!(response.len(), 1);
19865            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
19866            assert!(caller_rx.try_recv().is_err());
19867
19868            // 4. The reservation pair is given back, and the pending bind
19869            //    settled exactly once: one accepted open (the co-tenant) and one
19870            //    refused open (the caller), nothing counted twice.
19871            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
19872            let counters = handler.counters().snapshot();
19873            assert_eq!(
19874                keyed_total(&counters, "route_open_accepted_by_principal"),
19875                1
19876            );
19877            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
19878            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
19879        }
19880
19881        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
19882        /// id would resolve to the promoted candidate and every new route.open
19883        /// would be refused as reloading, leaving neither process routable.
19884        #[tokio::test]
19885        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
19886            let mut swap = swap_with_incumbent().await;
19887            register_candidate(&swap, None);
19888            let incumbent = cutover(&swap);
19889            swap.forwarding
19890                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
19891                .unwrap()
19892                .expect("the incumbent is still registered");
19893
19894            let client = ConnectionId::new(33);
19895            let (client_ctx, mut client_rx) = route_ctx(client);
19896            let route_handler = swap.handler.clone();
19897            let open_ctx = RouteCtx {
19898                connection_id: client,
19899                egress: client_ctx.egress.clone(),
19900            };
19901            let mut route_task = tokio::spawn(async move {
19902                route_handler
19903                    .handle_control_frame(
19904                        &open_ctx,
19905                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
19906                    )
19907                    .await
19908                    .unwrap()
19909            });
19910            let bind = tokio::select! {
19911                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
19912                response = &mut route_task => {
19913                    let response = response.unwrap();
19914                    panic!(
19915                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
19916                        parse_error(&response[0])["code"]
19917                    );
19918                }
19919            };
19920            swap.handler
19921                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
19922                .await
19923                .unwrap();
19924            assert!(route_task.await.unwrap().is_empty());
19925            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
19926            match swap
19927                .forwarding
19928                .lookup_data_route(client, channel, epoch)
19929                .unwrap()
19930            {
19931                DataRoute::Client(DataRouteState::Bound(route)) => {
19932                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
19933                }
19934                other => panic!("expected a bound route on the candidate, got {other:?}"),
19935            }
19936            assert!(swap.incumbent_rx.try_recv().is_err());
19937        }
19938
19939        /// A candidate declares itself ready with `catalog.update` on its own
19940        /// connection. If the connection-keyed registry lookups searched only the
19941        /// active slot, this would answer `not_registered` and the candidate
19942        /// would never become ready.
19943        #[tokio::test]
19944        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
19945            let swap = swap_with_incumbent().await;
19946            register_candidate(&swap, Some(false));
19947            let update = Frame::build(
19948                FrameType::Request,
19949                control_flags(),
19950                0,
19951                0,
19952                55,
19953                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
19954                    provides: manifest("aft", PROTOCOL_VERSION).provides,
19955                    capabilities: None,
19956                    ready: Some(true),
19957                })
19958                .unwrap(),
19959            )
19960            .unwrap();
19961
19962            let replies = swap
19963                .handler
19964                .handle_control_frame(&swap.candidate_ctx, update)
19965                .await
19966                .unwrap();
19967
19968            assert_eq!(replies.len(), 1);
19969            assert_eq!(
19970                replies[0].header.ty,
19971                FrameType::Response,
19972                "candidate catalog.update was refused: {:?}",
19973                serde_json::from_slice::<Value>(&replies[0].body).ok()
19974            );
19975            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
19976            assert_eq!(
19977                swap.registry
19978                    .get_module("aft")
19979                    .unwrap()
19980                    .unwrap()
19981                    .connection_id,
19982                INCUMBENT
19983            );
19984        }
19985    }
19986
19987    /// The HELLO gate while the supervisor has a swap open: only the nonce it
19988    /// minted for the candidate admits a second process, into the candidate
19989    /// slot, and that check runs ahead of the reserved-module gate.
19990    mod swap_admission {
19991        use super::*;
19992
19993        const INCUMBENT_NONCE: &str = "incumbent-nonce";
19994        const CANDIDATE_NONCE: &str = "candidate-nonce";
19995
19996        fn handler_with_incumbent(
19997            module_id: &str,
19998            reserved: bool,
19999        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
20000            let registry = Arc::new(Registry::default());
20001            let supervisor = SupervisorHandle::new();
20002            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
20003            if reserved {
20004                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
20005            }
20006            let handler =
20007                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
20008            let incumbent = handler
20009                .handle_control(
20010                    ConnectionId::new(1),
20011                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
20012                )
20013                .unwrap();
20014            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
20015            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
20016            (registry, supervisor, handler)
20017        }
20018
20019        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
20020        /// admits every nonce, so while a swap is open the swap gate is the only
20021        /// thing between a key-holder and the candidate slot. A nonce the
20022        /// supervisor did not mint, or none at all, is refused, and neither the
20023        /// incumbent's registration nor the candidate slot moves.
20024        #[test]
20025        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
20026            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
20027
20028            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
20029                let replies = handler
20030                    .handle_control(
20031                        ConnectionId::new(connection),
20032                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
20033                    )
20034                    .unwrap();
20035                assert_eq!(replies[0].header.ty, FrameType::Error);
20036                assert_eq!(
20037                    parse_error(&replies[0])["code"],
20038                    "swap_token_invalid",
20039                    "nonce {nonce:?}"
20040                );
20041            }
20042            assert!(registry.get_candidate("aft").unwrap().is_none());
20043            assert_eq!(
20044                registry.get_module("aft").unwrap().unwrap().connection_id,
20045                ConnectionId::new(1)
20046            );
20047
20048            // Control: the minted token is admitted, into the candidate slot,
20049            // and only once.
20050            let admitted = handler
20051                .handle_control(
20052                    ConnectionId::new(4),
20053                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
20054                )
20055                .unwrap();
20056            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
20057            assert_eq!(
20058                registry
20059                    .get_candidate("aft")
20060                    .unwrap()
20061                    .unwrap()
20062                    .connection_id,
20063                ConnectionId::new(4)
20064            );
20065            assert_eq!(
20066                registry.get_module("aft").unwrap().unwrap().connection_id,
20067                ConnectionId::new(1),
20068                "the candidate must not take the active slot"
20069            );
20070            let replayed = handler
20071                .handle_control(
20072                    ConnectionId::new(5),
20073                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
20074                )
20075                .unwrap();
20076            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
20077
20078            // The case only this gate covers: the incumbent has died mid-swap,
20079            // so its duplicate refusal is gone too, and without the gate a
20080            // key-holder would take the id's ACTIVE slot.
20081            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
20082            let squatter = handler
20083                .handle_control(
20084                    ConnectionId::new(6),
20085                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
20086                )
20087                .unwrap();
20088            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
20089            assert!(
20090                registry.get_module("aft").unwrap().is_none(),
20091                "a squatter took the active slot of an id being swapped"
20092            );
20093        }
20094
20095        /// Design mutation arm (iii). A reserved module's candidate presents a
20096        /// nonce the reserved gate has never seen (that gate holds the
20097        /// incumbent's), so the swap gate must run first or the candidate is
20098        /// refused `reserved_module` and a reserved module can never be swapped.
20099        #[test]
20100        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
20101            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
20102
20103            let replies = handler
20104                .handle_control(
20105                    ConnectionId::new(2),
20106                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20107                )
20108                .unwrap();
20109
20110            assert_eq!(
20111                replies[0].header.ty,
20112                FrameType::HelloAck,
20113                "reserved candidate refused: {:?}",
20114                serde_json::from_slice::<Value>(&replies[0].body).ok()
20115            );
20116            assert_eq!(
20117                registry
20118                    .get_candidate("vault")
20119                    .unwrap()
20120                    .unwrap()
20121                    .connection_id,
20122                ConnectionId::new(2)
20123            );
20124        }
20125
20126        /// With no swap open the gate is inert: the incumbent's reserved gate
20127        /// and duplicate refusal behave exactly as before.
20128        #[test]
20129        fn without_an_open_swap_the_ordinary_gates_decide() {
20130            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
20131            supervisor.close_swap("vault");
20132
20133            let candidate = handler
20134                .handle_control(
20135                    ConnectionId::new(2),
20136                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20137                )
20138                .unwrap();
20139            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
20140            let duplicate = handler
20141                .handle_control(
20142                    ConnectionId::new(3),
20143                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
20144                )
20145                .unwrap();
20146            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
20147            assert!(registry.get_candidate("vault").unwrap().is_none());
20148        }
20149    }
20150
20151    /// `scope.sync` and `scope.describe` through the real control handler: who
20152    /// may sync is decided by the registration and launch nonce of the module
20153    /// connection, never by the request body.
20154    mod scopes {
20155        mod close_reasons {
20156            include!("route_close_reason_tests.rs");
20157        }
20158        mod idle_tests {
20159            use super::*;
20160            include!("scope_idle_tests.rs");
20161        }
20162        mod route_open_timings {
20163            use super::*;
20164            include!("route_open_timing_tests.rs");
20165        }
20166        mod route_open_refusal_log {
20167            use super::*;
20168            include!("route_open_refusal_log_tests.rs");
20169        }
20170        mod activity {
20171            include!("activity_scope_tests.rs");
20172        }
20173        mod delta {
20174            include!("scope_handler_delta_tests.rs");
20175        }
20176        mod call_modes {
20177            include!("scope_call_mode_tests.rs");
20178        }
20179        mod federated {
20180            include!("scope_federated_tests.rs");
20181        }
20182        mod authorize {
20183            include!("route_authorize_tests.rs");
20184        }
20185        #[tokio::test]
20186        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
20187            let rig = rig().await;
20188            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
20189                "s".into(),
20190                1,
20191                "tool".into(),
20192            ))
20193            .unwrap();
20194            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
20195            let (direct, _rx) = wide_ctx(999);
20196            for (ctx, expected) in [
20197                (&rig.owner, FrameType::Response),
20198                (&direct, FrameType::Error),
20199            ] {
20200                let frame =
20201                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
20202                        .unwrap();
20203                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
20204                assert_eq!(replies[0].header.ty, expected);
20205                if expected == FrameType::Response {
20206                    let reply: subc_control::ScopeServedByReply =
20207                        serde_json::from_slice(&replies[0].body).unwrap();
20208                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
20209                } else {
20210                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
20211                }
20212            }
20213            let reply = call(
20214                &rig.handler,
20215                &rig.owner,
20216                &ModuleControlRequestFromModule::ScopeDescribe {
20217                    owner: Principal::Reserved {
20218                        module_id: OWNER.into(),
20219                    },
20220                    scope_ref: "s".into(),
20221                },
20222            )
20223            .await;
20224            assert!(matches!(
20225                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
20226                ModuleControlResponseToModule::ScopeDescribe { .. }
20227            ));
20228        }
20229
20230        use subc_protocol::scope::{
20231            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
20232            ScopeStatus,
20233        };
20234
20235        use super::*;
20236
20237        const OWNER: &str = "prefrontal-core";
20238
20239        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
20240            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
20241        }
20242
20243        async fn call(
20244            handler: &ControlHandler,
20245            ctx: &RouteCtx,
20246            request: &ModuleControlRequestFromModule,
20247        ) -> Frame {
20248            let body = serde_json::to_vec(request).unwrap();
20249            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
20250            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
20251            assert_eq!(replies.len(), 1, "{replies:?}");
20252            replies.pop().unwrap()
20253        }
20254
20255        async fn sync(
20256            handler: &ControlHandler,
20257            ctx: &RouteCtx,
20258            generation: u64,
20259            scopes: Vec<ScopeRecord>,
20260        ) -> Result<ModuleControlResponseToModule, String> {
20261            let reply = call(
20262                handler,
20263                ctx,
20264                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
20265            )
20266            .await;
20267            match reply.header.ty {
20268                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
20269                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
20270            }
20271        }
20272
20273        async fn describe(
20274            handler: &ControlHandler,
20275            ctx: &RouteCtx,
20276            owner: &str,
20277            scope_ref: &str,
20278        ) -> ModuleControlResponseToModule {
20279            let reply = call(
20280                handler,
20281                ctx,
20282                &ModuleControlRequestFromModule::ScopeDescribe {
20283                    owner: Principal::Reserved {
20284                        module_id: owner.to_string(),
20285                    },
20286                    scope_ref: scope_ref.to_string(),
20287                },
20288            )
20289            .await;
20290            assert_eq!(
20291                reply.header.ty,
20292                FrameType::Response,
20293                "{:?}",
20294                parse_error(&reply)
20295            );
20296            serde_json::from_slice(&reply.body).unwrap()
20297        }
20298
20299        #[tokio::test]
20300        async fn scope_describe_never_echoes_bind_proof_or_hash() {
20301            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
20302            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
20303            let mut rig = rig().await;
20304            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
20305                .await;
20306            let _bound = rig
20307                .bound(
20308                    Some(AFT),
20309                    PLEXUS,
20310                    Some(
20311                        rig.selector("s", Some(1))
20312                            .with_bind_proof(Some(PROOF.into())),
20313                    ),
20314                )
20315                .await;
20316            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
20317            let output = serde_json::to_string(&reply).unwrap();
20318            assert!(output.contains("\"status\":\"live\""));
20319            assert!(!output.contains(PROOF));
20320            assert!(!output.contains(HASH));
20321        }
20322
20323        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
20324        async fn module(
20325            handler: &ControlHandler,
20326            connection: u64,
20327            module_id: &str,
20328            nonce: Option<&str>,
20329        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20330            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
20331            hello_via_sink(
20332                handler,
20333                &ctx,
20334                &mut rx,
20335                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
20336            )
20337            .await;
20338            (ctx, rx)
20339        }
20340
20341        /// `direct` and every other client connection has no registration, so
20342        /// it can neither sync nor own a scope.
20343        #[tokio::test]
20344        async fn a_client_connection_cannot_sync_or_describe() {
20345            let handler = ControlHandler::new(Arc::new(Registry::default()));
20346            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
20347            for request in [
20348                ModuleControlRequestFromModule::ScopeSync {
20349                    generation: 1,
20350                    scopes: vec![head("s", 1)],
20351                },
20352                ModuleControlRequestFromModule::ScopeDescribe {
20353                    owner: Principal::Direct,
20354                    scope_ref: "s".to_string(),
20355                },
20356            ] {
20357                let reply = call(&handler, &ctx, &request).await;
20358                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
20359            }
20360            assert!(
20361                !handler
20362                    .scopes
20363                    .read()
20364                    .unwrap()
20365                    .describe(
20366                        &Principal::Reserved {
20367                            module_id: OWNER.to_string()
20368                        },
20369                        "s"
20370                    )
20371                    .owner_synced
20372            );
20373        }
20374
20375        /// A module the supervisor did not spawn registers without a launch
20376        /// nonce, so it is never an owner's current launch.
20377        #[tokio::test]
20378        async fn a_module_without_a_supervised_launch_cannot_sync() {
20379            let handler = ControlHandler::new(Arc::new(Registry::default()));
20380            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
20381            assert_eq!(
20382                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
20383                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20384            );
20385        }
20386
20387        #[tokio::test]
20388        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
20389            let supervisor = SupervisorHandle::new();
20390            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20391            let handler = ControlHandler::new(Arc::new(Registry::default()))
20392                .with_supervisor(supervisor.clone());
20393            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20394            sync(&handler, &incumbent, 1, vec![head("s", 1)])
20395                .await
20396                .expect("the current launch syncs");
20397
20398            // A swap candidate registers with the swap token and is refused
20399            // while the incumbent keeps syncing.
20400            supervisor.open_swap(OWNER, "n2".to_string());
20401            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
20402            assert_eq!(
20403                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20404                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20405            );
20406            sync(&handler, &incumbent, 2, vec![head("s", 1)])
20407                .await
20408                .expect("the serving owner syncs during the swap");
20409
20410            // The swap fails and is rolled back. The candidate never held sync
20411            // authority, and still cannot sync.
20412            supervisor.close_swap(OWNER);
20413            assert_eq!(
20414                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20415                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20416            );
20417            sync(&handler, &incumbent, 3, vec![head("s", 1)])
20418                .await
20419                .expect("the serving owner syncs after the rollback");
20420            handler.cleanup_connection(candidate.connection_id).unwrap();
20421
20422            // A swap that cuts over. Promotion records the candidate's nonce as
20423            // the module's spawn nonce, which is what `set_spawn_nonce` does
20424            // here; the promoted connection then takes authority at any
20425            // generation and the superseded incumbent is refused.
20426            supervisor.open_swap(OWNER, "n3".to_string());
20427            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
20428            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
20429            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
20430                .await
20431                .expect("the promoted launch takes authority");
20432            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
20433                panic!("unexpected reply {reply:?}");
20434            };
20435            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
20436            assert_eq!(
20437                sync(&handler, &incumbent, 4, Vec::new()).await,
20438                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20439            );
20440        }
20441
20442        /// Authority dies with its connection: the cleanup path releases it,
20443        /// so the owner's next connection takes it at any generation.
20444        #[tokio::test]
20445        async fn closing_the_authority_connection_frees_sync_authority() {
20446            let supervisor = SupervisorHandle::new();
20447            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20448            let handler = ControlHandler::new(Arc::new(Registry::default()))
20449                .with_supervisor(supervisor.clone());
20450            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20451            sync(&handler, &first, 10, vec![head("s", 1)])
20452                .await
20453                .unwrap();
20454            handler.cleanup_connection(first.connection_id).unwrap();
20455
20456            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
20457            sync(&handler, &second, 1, vec![head("s", 1)])
20458                .await
20459                .expect("the next connection takes the released authority");
20460        }
20461
20462        #[tokio::test]
20463        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
20464            let supervisor = SupervisorHandle::new();
20465            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20466            let handler =
20467                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
20468            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
20469            sync(&handler, &first, 10, vec![head("s", 1)])
20470                .await
20471                .unwrap();
20472            handler
20473                .handle_control_frame(
20474                    &first,
20475                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
20476                )
20477                .await
20478                .unwrap();
20479            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
20480            sync(&handler, &second, 1, vec![head("s", 1)])
20481                .await
20482                .expect("GOODBYE releases authority even if the old socket remains open");
20483        }
20484
20485        #[tokio::test]
20486        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
20487            let registry = Arc::new(Registry::default());
20488            let supervisor_handle = SupervisorHandle::new();
20489            let supervisor =
20490                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
20491                    .with_handle(supervisor_handle.clone())
20492                    .with_daemon_incarnation("incarnation-7".to_string());
20493            // Configured with enabled: false, so the supervisor lists the
20494            // module without spawning a process for it.
20495            supervisor
20496                .supervise_configured(
20497                    ModuleSpec {
20498                        carry_containment: false,
20499                        module_id: OWNER.to_string(),
20500                        program: PathBuf::from("/nonexistent/prefrontal-core"),
20501                        args: Vec::new(),
20502                        env: Vec::new(),
20503                        reserved: false,
20504                        reserved_prefixes: Vec::new(),
20505                        protocol: ModuleProtocol::Subc,
20506                        overlap: Default::default(),
20507                    },
20508                    false,
20509                )
20510                .unwrap();
20511            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
20512            let handler =
20513                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
20514            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
20515
20516            // Configured but not yet synced: a reader waits for the owner.
20517            let ModuleControlResponseToModule::ScopeDescribe {
20518                status,
20519                daemon_incarnation,
20520                owner_synced,
20521                owner_configured,
20522                scope,
20523                ..
20524            } = describe(&handler, &reader, OWNER, "s").await
20525            else {
20526                panic!("not a describe reply");
20527            };
20528            assert_eq!(status, ScopeStatus::NotLive);
20529            assert_eq!(daemon_incarnation, "incarnation-7");
20530            assert!(!owner_synced);
20531            assert!(owner_configured);
20532            assert!(scope.is_none());
20533
20534            // Not a supervised module: the owner will never sync, and a reader
20535            // refuses rather than waits.
20536            let ModuleControlResponseToModule::ScopeDescribe {
20537                status,
20538                owner_configured,
20539                ..
20540            } = describe(&handler, &reader, "ghost", "s").await
20541            else {
20542                panic!("not a describe reply");
20543            };
20544            assert_eq!(status, ScopeStatus::NotLive);
20545            assert!(!owner_configured);
20546
20547            // Live, with the stamp fields and the computed owner_authorized.
20548            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
20549            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
20550            let ModuleControlResponseToModule::ScopeDescribe {
20551                status,
20552                scope_epoch,
20553                owner_synced,
20554                scope,
20555                ..
20556            } = describe(&handler, &reader, OWNER, "s").await
20557            else {
20558                panic!("not a describe reply");
20559            };
20560            assert_eq!(status, ScopeStatus::Live);
20561            assert_eq!(scope_epoch, Some(4));
20562            assert!(owner_synced);
20563            let stamp = scope.expect("a live scope carries its stamp");
20564            assert!(
20565                stamp.owner_authorized,
20566                "prefrontal-core is the default authority"
20567            );
20568            assert_eq!(stamp.kind, ScopeKind::Head);
20569        }
20570
20571        #[tokio::test]
20572        async fn scope_authority_owners_decides_owner_authorized() {
20573            let supervisor = SupervisorHandle::new();
20574            supervisor.set_spawn_nonce("broca", "b1".to_string());
20575            let handler = ControlHandler::new(Arc::new(Registry::default()))
20576                .with_supervisor(supervisor)
20577                .with_scope_authority_owners(vec!["broca".to_string()]);
20578            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
20579            let mut gated = head("s", 1);
20580            gated.attributes.agent_id = Some("agent".to_string());
20581            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
20582            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
20583                describe(&handler, &broca, "broca", "s").await
20584            else {
20585                panic!("not a describe reply");
20586            };
20587            assert!(scope.unwrap().owner_authorized);
20588        }
20589
20590        /// With route admission, the stamp, the commit re-check and drains in
20591        /// place, the feature is advertised: the module ops in HELLO_ACK, and
20592        /// `scopes/v1` in HELLO_ACK and `server.describe`.
20593        #[tokio::test]
20594        async fn scope_ops_and_the_scopes_capability_are_advertised() {
20595            let handler = ControlHandler::new(Arc::new(Registry::default()));
20596            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
20597            let ack = hello_via_sink(
20598                &handler,
20599                &ctx,
20600                &mut rx,
20601                hello_frame("m", PROTOCOL_VERSION, 1),
20602            )
20603            .await;
20604            let ack = parse_ack(&ack);
20605            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
20606                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
20607            }
20608            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
20609
20610            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
20611            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
20612            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
20613            let reply = handler
20614                .handle_control_frame(&client, frame)
20615                .await
20616                .unwrap()
20617                .pop()
20618                .unwrap();
20619            let ClientControlResponse::ServerDescribe { capabilities, .. } =
20620                serde_json::from_slice(&reply.body).unwrap()
20621            else {
20622                panic!("not a server.describe reply");
20623            };
20624            assert!(
20625                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
20626                "{capabilities:?}"
20627            );
20628        }
20629
20630        // ---- route admission, stamps, commit re-check and drains ----------
20631
20632        const PLEXUS: &str = "plexus";
20633        const OTHER: &str = "other";
20634        const AFT: &str = "aft";
20635        const BROCA: &str = "broca";
20636        const MAGIC: &str = "magic-context";
20637
20638        fn nonce(module_id: &str) -> String {
20639            format!("nonce-{module_id}")
20640        }
20641
20642        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20643            let (tx, rx) = mpsc::channel(64);
20644            (
20645                RouteCtx {
20646                    connection_id: ConnectionId::new(connection),
20647                    egress: FrameSink::new(tx),
20648                },
20649                rx,
20650            )
20651        }
20652
20653        /// A daemon with a configured owner (prefrontal-core) registered on its
20654        /// own module connection, two routable targets (plexus, other), and
20655        /// launch nonces minted for the modules that open routes as carriers.
20656        struct Rig {
20657            handler: ControlHandler,
20658            forwarding: Arc<ForwardingTable>,
20659            owner: RouteCtx,
20660            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20661            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
20662            generation: u64,
20663            next_connection: u64,
20664            _supervisor: Supervisor,
20665        }
20666
20667        async fn rig() -> Rig {
20668            rig_with_flow_support(true).await
20669        }
20670
20671        async fn rig_with_flow_support(flow_support: bool) -> Rig {
20672            rig_with_scope_capabilities(flow_support, false).await
20673        }
20674
20675        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
20676            let mut provides = Vec::new();
20677            if flow_support {
20678                provides.push("flow-scopes/v1");
20679            }
20680            if run_support {
20681                provides.push("agent-run-scopes/v1");
20682            }
20683            rig_with_target_capabilities(&provides).await
20684        }
20685
20686        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
20687            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
20688            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
20689            // Decoder versions alone cannot admit scoped calls: only a capability
20690            // declaration promises the required run or flow behaviour.
20691            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
20692            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
20693            Frame::build(
20694                FrameType::Hello,
20695                control_flags(),
20696                0,
20697                0,
20698                connection,
20699                serde_json::to_vec(&body).unwrap(),
20700            )
20701            .unwrap()
20702        }
20703
20704        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
20705            let registry = Arc::new(Registry::default());
20706            let forwarding = Arc::new(ForwardingTable::default());
20707            let supervisor_handle = SupervisorHandle::new();
20708            let supervisor =
20709                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
20710                    .with_handle(supervisor_handle.clone());
20711            supervisor
20712                .supervise_configured(
20713                    ModuleSpec {
20714                        carry_containment: false,
20715                        module_id: OWNER.to_string(),
20716                        program: PathBuf::from("/nonexistent/prefrontal-core"),
20717                        args: Vec::new(),
20718                        env: Vec::new(),
20719                        reserved: false,
20720                        reserved_prefixes: Vec::new(),
20721                        protocol: ModuleProtocol::Subc,
20722                        overlap: Default::default(),
20723                    },
20724                    false,
20725                )
20726                .unwrap();
20727            for module_id in [OWNER, AFT, BROCA, MAGIC] {
20728                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
20729            }
20730            let handler =
20731                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
20732                    .with_supervisor(supervisor_handle);
20733            let (owner, mut owner_rx) = wide_ctx(1);
20734            hello_via_sink(
20735                &handler,
20736                &owner,
20737                &mut owner_rx,
20738                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
20739            )
20740            .await;
20741            let mut modules = BTreeMap::new();
20742            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
20743                let (ctx, mut rx) = wide_ctx(connection);
20744                let hello = hello_with_capabilities(module_id, connection, provides);
20745                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
20746                modules.insert(module_id.to_string(), (ctx, rx));
20747            }
20748            Rig {
20749                handler,
20750                forwarding,
20751                owner,
20752                _owner_rx: owner_rx,
20753                modules,
20754                generation: 0,
20755                next_connection: 100,
20756                _supervisor: supervisor,
20757            }
20758        }
20759
20760        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
20761            ScopeCarrier::new(Principal::Reserved {
20762                module_id: module_id.to_string(),
20763            })
20764            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
20765        }
20766
20767        /// The scope most tests open under: aft carries to any module, broca
20768        /// only to plexus and other, and the owner delegates as agent-1.
20769        fn session(scope_epoch: u64) -> ScopeRecord {
20770            let mut record = head("s", scope_epoch);
20771            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
20772            record.attributes.agent_id = Some("agent-1".to_string());
20773            record.attributes.delegates = true;
20774            record
20775        }
20776
20777        impl Rig {
20778            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
20779                self.generation += 1;
20780                sync(&self.handler, &self.owner, self.generation, scopes)
20781                    .await
20782                    .expect("the owner's sync is accepted");
20783            }
20784
20785            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
20786                ScopeSelector::new(
20787                    Principal::Reserved {
20788                        module_id: OWNER.to_string(),
20789                    },
20790                    scope_ref.to_string(),
20791                    scope_epoch,
20792                )
20793            }
20794
20795            fn open_frame(
20796                &mut self,
20797                opener: Option<&str>,
20798                target: &str,
20799                scope: Option<ScopeSelector>,
20800            ) -> (
20801                RouteCtx,
20802                mpsc::Receiver<crate::router::OutboundFrame>,
20803                Frame,
20804            ) {
20805                self.next_connection += 1;
20806                let (ctx, rx) = wide_ctx(self.next_connection);
20807                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
20808                    busy_refusals: None,
20809                    target: RouteTarget::ToolProvider {
20810                        module_id: target.to_string(),
20811                    },
20812                    identity: BindIdentity::new("unit".to_string(), "session".to_string()),
20813                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
20814                        module_id: module_id.to_string(),
20815                        launch_nonce: nonce(module_id),
20816                    }),
20817                    consumer_capabilities: None,
20818                    role_versions: None,
20819                    admission_facts: None,
20820                    scope,
20821                })
20822                .unwrap();
20823                let frame = Frame::build(
20824                    FrameType::Request,
20825                    control_flags(),
20826                    0,
20827                    0,
20828                    self.next_connection,
20829                    body,
20830                )
20831                .unwrap();
20832                (ctx, rx, frame)
20833            }
20834
20835            /// Open and expect a refusal before anything is relayed.
20836            async fn refused(
20837                &mut self,
20838                opener: Option<&str>,
20839                target: &str,
20840                scope: Option<ScopeSelector>,
20841            ) -> String {
20842                self.refusal_body(opener, target, scope).await["code"]
20843                    .as_str()
20844                    .unwrap()
20845                    .to_string()
20846            }
20847
20848            async fn refusal_body(
20849                &mut self,
20850                opener: Option<&str>,
20851                target: &str,
20852                scope: Option<ScopeSelector>,
20853            ) -> Value {
20854                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
20855                let replies = tokio::time::timeout(
20856                    Duration::from_secs(2),
20857                    self.handler.handle_control_frame(&ctx, frame),
20858                )
20859                .await
20860                .expect("the open must be refused before waiting for a bind ack")
20861                .unwrap();
20862                assert_eq!(replies.len(), 1, "{replies:?}");
20863                assert_eq!(replies[0].header.ty, FrameType::Error);
20864                let (_, module_rx) = self.modules.get_mut(target).unwrap();
20865                assert!(
20866                    module_rx.try_recv().is_err(),
20867                    "a refused open relays nothing"
20868                );
20869                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
20870                parse_error(&replies[0])
20871            }
20872
20873            /// Start an open and return its task and the bind the target got.
20874            async fn relayed(
20875                &mut self,
20876                opener: Option<&str>,
20877                target: &str,
20878                scope: Option<ScopeSelector>,
20879            ) -> Relayed {
20880                self.relayed_with_legacy_root(opener, target, scope, None)
20881                    .await
20882            }
20883
20884            async fn relayed_with_legacy_root(
20885                &mut self,
20886                opener: Option<&str>,
20887                target: &str,
20888                scope: Option<ScopeSelector>,
20889                legacy_root: Option<&std::path::Path>,
20890            ) -> Relayed {
20891                let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
20892                if let Some(root) = legacy_root {
20893                    let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
20894                    body["identity"]["project_root"] = serde_json::json!(root);
20895                    frame.body = serde_json::to_vec(&body).unwrap();
20896                }
20897                let handler = self.handler.clone();
20898                let task_ctx = ctx.clone();
20899                let task = tokio::spawn(async move {
20900                    handler
20901                        .handle_control_frame(&task_ctx, frame)
20902                        .await
20903                        .unwrap()
20904                });
20905                let (_, module_rx) = self.modules.get_mut(target).unwrap();
20906                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
20907                    .await
20908                    .expect("the target receives the relayed route.bind")
20909                    .unwrap()
20910                    .frame;
20911                Relayed {
20912                    target: target.to_string(),
20913                    client: ctx,
20914                    client_rx: rx,
20915                    task,
20916                    bind,
20917                }
20918            }
20919
20920            async fn ack(&self, relayed: &Relayed) {
20921                let (module, _) = &self.modules[&relayed.target];
20922                self.handler
20923                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
20924                    .await
20925                    .unwrap();
20926            }
20927
20928            /// Open, ack and return the bound route.
20929            async fn bound(
20930                &mut self,
20931                opener: Option<&str>,
20932                target: &str,
20933                scope: Option<ScopeSelector>,
20934            ) -> Bound {
20935                let relayed = self.relayed(opener, target, scope).await;
20936                self.commit_relayed(relayed).await
20937            }
20938
20939            async fn bound_with_legacy_root(
20940                &mut self,
20941                opener: Option<&str>,
20942                target: &str,
20943                scope: Option<ScopeSelector>,
20944                root: &std::path::Path,
20945            ) -> Bound {
20946                let relayed = self
20947                    .relayed_with_legacy_root(opener, target, scope, Some(root))
20948                    .await;
20949                self.commit_relayed(relayed).await
20950            }
20951
20952            async fn commit_relayed(&self, relayed: Relayed) -> Bound {
20953                self.ack(&relayed).await;
20954                let Relayed {
20955                    target,
20956                    client,
20957                    mut client_rx,
20958                    task,
20959                    bind,
20960                } = relayed;
20961                assert!(
20962                    task.await.unwrap().is_empty(),
20963                    "the open is answered by commit"
20964                );
20965                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
20966                Bound {
20967                    target,
20968                    client,
20969                    client_rx,
20970                    channel,
20971                    epoch,
20972                    bind,
20973                }
20974            }
20975
20976            fn live(&self, route: &Bound) -> bool {
20977                matches!(
20978                    self.forwarding
20979                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
20980                        .unwrap(),
20981                    DataRoute::Client(DataRouteState::Bound(_))
20982                )
20983            }
20984        }
20985
20986        struct Relayed {
20987            target: String,
20988            client: RouteCtx,
20989            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20990            task: tokio::task::JoinHandle<Vec<Frame>>,
20991            bind: Frame,
20992        }
20993
20994        struct Bound {
20995            target: String,
20996            client: RouteCtx,
20997            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20998            channel: u16,
20999            epoch: u32,
21000            bind: Frame,
21001        }
21002
21003        impl Bound {
21004            /// The reason of the `route.closed` this client was sent, after
21005            /// checking it also got a GOODBYE on exactly this route.
21006            fn closed_reason(&mut self) -> RouteCloseReason {
21007                let mut reason = None;
21008                let mut goodbye = false;
21009                while let Ok(outbound) = self.client_rx.try_recv() {
21010                    let frame = outbound.frame;
21011                    match frame.header.ty {
21012                        FrameType::Goodbye => {
21013                            assert_eq!(
21014                                (frame.header.channel, frame.header.epoch),
21015                                (self.channel, self.epoch)
21016                            );
21017                            goodbye = true;
21018                        }
21019                        FrameType::Push => {
21020                            let ClientControlPush::RouteClosed {
21021                                reason: r,
21022                                module_id,
21023                                ..
21024                            } = serde_json::from_slice(&frame.body).unwrap()
21025                            else {
21026                                panic!("unexpected push");
21027                            };
21028                            assert_eq!(module_id, self.target);
21029                            reason = Some(r);
21030                        }
21031                        other => panic!("unexpected frame {other:?}"),
21032                    }
21033                }
21034                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
21035                reason.expect("the client is told why the route closed")
21036            }
21037
21038            fn untouched(&mut self) -> bool {
21039                self.client_rx.try_recv().is_err()
21040            }
21041
21042            fn stamp(&self) -> Option<ScopeStamp> {
21043                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
21044                    ModuleControlRequest::RouteBind { scope, .. } => scope,
21045                    other => panic!("expected a route.bind, got {other:?}"),
21046                }
21047            }
21048        }
21049
21050        #[tokio::test]
21051        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
21052        ) {
21053            let mut rig = rig().await;
21054            let mut record = session(1);
21055            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
21056            record.child_owners = vec![Principal::Reserved {
21057                module_id: MAGIC.to_string(),
21058            }];
21059            rig.sync(vec![record]).await;
21060            let scope = || Some(rig_selector("s", Some(1)));
21061
21062            // Admitted: the owner, a bare carrier to any module, a targeted
21063            // carrier to its listed module.
21064            rig.bound(Some(OWNER), PLEXUS, scope()).await;
21065            rig.bound(Some(AFT), OTHER, scope()).await;
21066            rig.bound(Some(BROCA), PLEXUS, scope()).await;
21067
21068            // Refused scope_not_carrier: a targeted carrier to an unlisted
21069            // module, a module that is not listed at all (a child owner is not
21070            // a carrier), and a direct key-holder.
21071            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
21072                assert_eq!(
21073                    rig.refused(opener, target, scope()).await,
21074                    error_codes::SCOPE_NOT_CARRIER,
21075                    "{opener:?} -> {target}"
21076                );
21077            }
21078        }
21079
21080        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
21081            ScopeSelector::new(
21082                Principal::Reserved {
21083                    module_id: OWNER.to_string(),
21084                },
21085                scope_ref.to_string(),
21086                scope_epoch,
21087            )
21088        }
21089
21090        #[tokio::test]
21091        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
21092            let mut rig = rig().await;
21093            rig.sync(vec![session(1)]).await;
21094            for opener in [OWNER, AFT] {
21095                assert_eq!(
21096                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
21097                        .await,
21098                    error_codes::SCOPE_EPOCH_REQUIRED,
21099                    "{opener}"
21100                );
21101            }
21102        }
21103
21104        #[tokio::test]
21105        async fn admission_separates_not_synced_not_live_and_ended() {
21106            let mut rig = rig().await;
21107            // Before the configured owner's first sync: retryable.
21108            let code = rig
21109                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21110                .await;
21111            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
21112            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
21113
21114            // An owner that is not configured will never sync: terminal.
21115            let ghost = ScopeSelector::new(
21116                Principal::Reserved {
21117                    module_id: "ghost".to_string(),
21118                },
21119                "s".to_string(),
21120                Some(1),
21121            );
21122            assert_eq!(
21123                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
21124                error_codes::SCOPE_NOT_LIVE
21125            );
21126
21127            rig.sync(vec![session(2)]).await;
21128            assert_eq!(
21129                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
21130                    .await,
21131                error_codes::SCOPE_NOT_LIVE
21132            );
21133            for epoch in [1, 3] {
21134                assert_eq!(
21135                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
21136                        .await,
21137                    error_codes::SCOPE_ENDED,
21138                    "epoch {epoch}"
21139                );
21140            }
21141            // Control: the live epoch is admitted.
21142            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
21143                .await;
21144        }
21145
21146        #[tokio::test]
21147        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
21148            let mut rig = rig().await;
21149            rig.sync(vec![session(1)]).await;
21150            let route = rig
21151                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21152                .await;
21153            let stamp = route.stamp().expect("a scoped bind carries the stamp");
21154            assert_eq!(stamp.scope_ref, "s");
21155            assert_eq!(stamp.scope_epoch, 1);
21156            assert_eq!(stamp.kind, ScopeKind::Head);
21157            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
21158            assert!(stamp.attributes.delegates);
21159            assert!(stamp.owner_authorized);
21160            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21161            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
21162
21163            // broca owns a scope of its own on its own module connection; it is
21164            // not in scope_authority_owners, so its stamp is not authorized.
21165            let (broca, mut broca_rx) = wide_ctx(50);
21166            hello_via_sink(
21167                &rig.handler,
21168                &broca,
21169                &mut broca_rx,
21170                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
21171            )
21172            .await;
21173            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
21174                .await
21175                .unwrap();
21176            let own = ScopeSelector::new(
21177                Principal::Reserved {
21178                    module_id: BROCA.to_string(),
21179                },
21180                "b".to_string(),
21181                Some(1),
21182            );
21183            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
21184            assert!(!route.stamp().unwrap().owner_authorized);
21185        }
21186
21187        #[tokio::test]
21188        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
21189            let mut rig = rig().await;
21190            let mut record = head("s", 1);
21191            record.carriers = vec![carrier(AFT, None)];
21192            let flow_id = "Flow:run-7/step_2!~";
21193            record.attributes.flow_id = Some(flow_id.to_string());
21194            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
21195                .await
21196                .unwrap();
21197            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
21198                panic!("not a sync reply");
21199            };
21200            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
21201            let route = rig
21202                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21203                .await;
21204            assert!(rig.live(&route), "the stamped bind committed");
21205            let stamp = route.stamp().expect("a flow scope carries a stamp");
21206            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
21207            assert_eq!(stamp.attributes.agent_id, None);
21208            assert!(!stamp.attributes.delegates);
21209            assert!(stamp.owner_authorized);
21210            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21211            assert_eq!(unscoped.stamp(), None);
21212        }
21213
21214        #[tokio::test]
21215        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
21216            let mut rig = rig_with_flow_support(false).await;
21217            let mut record = session(1);
21218            record.attributes.flow_id = Some("flow:7".to_string());
21219            rig.sync(vec![record]).await;
21220            for opener in [OWNER, AFT] {
21221                let body = rig
21222                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21223                    .await;
21224                assert_eq!(body["code"], "target_flow_unsupported");
21225                let message = body["message"].as_str().unwrap();
21226                for required in [PLEXUS, "flow-scopes/v1"] {
21227                    assert!(message.contains(required), "{message}");
21228                }
21229            }
21230        }
21231
21232        #[tokio::test]
21233        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
21234            let mut rig = rig_with_flow_support(true).await;
21235            let mut record = session(1);
21236            record.attributes.flow_id = Some("flow:7".to_string());
21237            rig.sync(vec![record]).await;
21238            for opener in [OWNER, AFT] {
21239                let route = rig
21240                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21241                    .await;
21242                assert!(rig.live(&route));
21243                assert_eq!(
21244                    route.stamp().unwrap().attributes.flow_id.as_deref(),
21245                    Some("flow:7")
21246                );
21247            }
21248        }
21249
21250        #[tokio::test]
21251        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
21252            use std::future::Future;
21253
21254            let mut rig = rig_with_flow_support(true).await;
21255            let mut record = session(1);
21256            record.attributes.flow_id = Some("flow:7".to_string());
21257            rig.sync(vec![record]).await;
21258            let (client, mut client_rx, frame) =
21259                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
21260            // Hold the route-open response permit so admission sees the first
21261            // target but relay reservation cannot capture an endpoint yet.
21262            for _ in 0..64 {
21263                client.egress.try_send(route_bind_ack(1)).unwrap();
21264            }
21265            let handler = rig.handler.clone();
21266            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21267            std::future::poll_fn(|cx| {
21268                assert!(open.as_mut().poll(cx).is_pending());
21269                std::task::Poll::Ready(())
21270            })
21271            .await;
21272            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21273
21274            let old_connection = rig.modules[PLEXUS].0.connection_id;
21275            rig.handler.cleanup_connection(old_connection).unwrap();
21276            let (replacement, mut replacement_rx) = wide_ctx(200);
21277            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
21278            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
21279            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
21280            let hello = Frame::build(
21281                FrameType::Hello,
21282                control_flags(),
21283                0,
21284                0,
21285                200,
21286                serde_json::to_vec(&body).unwrap(),
21287            )
21288            .unwrap();
21289            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
21290
21291            client_rx.try_recv().unwrap();
21292            let replies = tokio::time::timeout(Duration::from_secs(2), open)
21293                .await
21294                .expect("the replacement is refused without waiting for a bind ack")
21295                .unwrap();
21296            assert_eq!(replies.len(), 1);
21297            let body = parse_error(&replies[0]);
21298            assert_eq!(body["code"], "target_flow_unsupported");
21299            for required in [PLEXUS, "flow-scopes/v1"] {
21300                assert!(body["message"].as_str().unwrap().contains(required));
21301            }
21302            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
21303            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21304            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21305            assert!(rig
21306                .handler
21307                .registry
21308                .get_module_by_connection(replacement.connection_id)
21309                .unwrap()
21310                .is_some());
21311        }
21312
21313        #[tokio::test]
21314        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
21315            use std::future::Future;
21316            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
21317            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21318            const NEXT_HASH: &str =
21319                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21320            let mut rig = rig().await;
21321            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
21322            record.carriers.push(ScopeCarrier::new(Principal::Direct));
21323            rig.sync(vec![record.clone()]).await;
21324            let (client, mut rx, frame) = rig.open_frame(
21325                None,
21326                PLEXUS,
21327                Some(
21328                    rig.selector("s", Some(1))
21329                        .with_bind_proof(Some(PROOF.into())),
21330                ),
21331            );
21332            // Hold egress reservation while the scope owner changes its bind-proof
21333            // hash, so a proof valid at admission must be checked again at relay.
21334            for _ in 0..64 {
21335                client.egress.try_send(route_bind_ack(1)).unwrap();
21336            }
21337            let handler = rig.handler.clone();
21338            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21339            std::future::poll_fn(|cx| {
21340                assert!(open.as_mut().poll(cx).is_pending());
21341                std::task::Poll::Ready(())
21342            })
21343            .await;
21344            record.bind_proof_sha256 = Some(NEXT_HASH.into());
21345            rig.sync(vec![record]).await;
21346            rx.try_recv().unwrap();
21347            let replies = tokio::time::timeout(Duration::from_secs(2), open)
21348                .await
21349                .unwrap()
21350                .unwrap();
21351            assert_eq!(
21352                parse_error(&replies[0])["code"],
21353                error_codes::SCOPE_BIND_PROOF_MISMATCH
21354            );
21355            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
21356            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21357            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21358        }
21359
21360        #[tokio::test]
21361        async fn scope_bind_proof_reserved_openers_ignore_proof() {
21362            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21363            let mut rig = rig().await;
21364            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21365                .await;
21366            for opener in [OWNER, AFT] {
21367                for proof in [None, Some("malformed+proof=")] {
21368                    let route = rig
21369                        .bound(
21370                            Some(opener),
21371                            PLEXUS,
21372                            Some(
21373                                rig.selector("s", Some(1))
21374                                    .with_bind_proof(proof.map(str::to_string)),
21375                            ),
21376                        )
21377                        .await;
21378                    assert!(rig.live(&route));
21379                }
21380            }
21381        }
21382
21383        #[tokio::test]
21384        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
21385            use std::future::Future;
21386            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21387            const NEXT_HASH: &str =
21388                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21389            let mut rig = rig().await;
21390            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21391                .await;
21392            let (client, mut rx, frame) =
21393                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
21394            for _ in 0..64 {
21395                client.egress.try_send(route_bind_ack(1)).unwrap();
21396            }
21397            let handler = rig.handler.clone();
21398            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21399            std::future::poll_fn(|cx| {
21400                assert!(open.as_mut().poll(cx).is_pending());
21401                std::task::Poll::Ready(())
21402            })
21403            .await;
21404            rig.sync(vec![
21405                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
21406            ])
21407            .await;
21408            rx.try_recv().unwrap();
21409            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21410            let bind = std::future::poll_fn(|cx| {
21411                assert!(open.as_mut().poll(cx).is_pending());
21412                module_rx.poll_recv(cx)
21413            })
21414            .await
21415            .unwrap()
21416            .frame;
21417            handler
21418                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
21419                .await
21420                .unwrap();
21421            let replies = tokio::time::timeout(Duration::from_secs(2), open)
21422                .await
21423                .unwrap()
21424                .unwrap();
21425            assert_eq!(replies.len(), 1);
21426            // Proof rechecking allowed the relay. Independent tag settlement
21427            // still asks the opener to retry after any same-epoch content change.
21428            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
21429            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21430            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
21431            let route = rig
21432                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
21433                .await;
21434            assert!(rig.live(&route));
21435        }
21436
21437        #[tokio::test]
21438        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
21439        {
21440            let mut rig = rig_with_flow_support(false).await;
21441            rig.sync(vec![session(1)]).await;
21442            let route = rig
21443                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21444                .await;
21445            assert!(rig.live(&route));
21446            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
21447            let mut record = session(1);
21448            record.attributes.flow_id = Some("flow:7".to_string());
21449            rig.sync(vec![record]).await;
21450            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
21451            assert!(rig.live(&unscoped));
21452            assert_eq!(unscoped.stamp(), None);
21453        }
21454
21455        #[tokio::test]
21456        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
21457            let mut rig = rig().await;
21458            let mut record = session(1);
21459            record.attributes.flow_id = Some("flow:7".to_string());
21460            rig.sync(vec![record.clone()]).await;
21461            let before = rig.forwarding.published_scope_tag(OWNER, "s");
21462            let mut owner_route = rig
21463                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21464                .await;
21465            let mut carrier_route = rig
21466                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
21467                .await;
21468            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21469            rig.sync(vec![record.clone()]).await;
21470            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21471            assert!(rig.live(&owner_route) && owner_route.untouched());
21472            assert!(rig.live(&carrier_route) && carrier_route.untouched());
21473
21474            record.attributes.flow_id = Some("flow:8".to_string());
21475            rig.sync(vec![record]).await;
21476            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
21477            let before = before.unwrap();
21478            assert_eq!(after.scope_epoch, before.scope_epoch);
21479            assert!(after.version > before.version);
21480            for route in [&mut owner_route, &mut carrier_route] {
21481                assert!(!rig.live(route));
21482                assert_eq!(
21483                    route.closed_reason(),
21484                    RouteCloseReason::ScopeDelegationChanged
21485                );
21486            }
21487            assert!(rig.live(&unscoped) && unscoped.untouched());
21488            // Each provider also receives a GOODBYE for its drained route;
21489            // consume it before expecting the next route.bind on that sink.
21490            for target in [PLEXUS, OTHER] {
21491                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
21492                let goodbye = module_rx
21493                    .try_recv()
21494                    .expect("the provider sees the drain")
21495                    .frame;
21496                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21497                assert!(module_rx.try_recv().is_err());
21498            }
21499            let rebound = rig
21500                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21501                .await;
21502            assert_eq!(
21503                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
21504                Some("flow:8")
21505            );
21506        }
21507
21508        /// The owner's sync lands between admission and the module's ack. The
21509        /// open is refused by name, the module's other routes stay up, and the
21510        /// reserved pair is released. Changed content is retryable; an ended
21511        /// scope is not.
21512        #[tokio::test]
21513        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
21514            let mut rig = rig().await;
21515            rig.sync(vec![session(1)]).await;
21516            let mut cotenant = rig
21517                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21518                .await;
21519
21520            let mut changed = session(1);
21521            changed.child_owners.push(Principal::Reserved {
21522                module_id: MAGIC.to_string(),
21523            });
21524            let mut ended = None;
21525            for (code, next) in [
21526                (error_codes::SCOPE_CHANGED, vec![changed]),
21527                (error_codes::SCOPE_ENDED, Vec::new()),
21528            ] {
21529                let relayed = rig
21530                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21531                    .await;
21532                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
21533                ended = Some(next.is_empty());
21534                rig.sync(next).await;
21535                rig.ack(&relayed).await;
21536                let replies = relayed.task.await.unwrap();
21537                assert_eq!(replies.len(), 1, "{replies:?}");
21538                assert_eq!(parse_error(&replies[0])["code"], code);
21539                assert_eq!(
21540                    subc_protocol::error_codes::is_retryable_route_open(code),
21541                    code == error_codes::SCOPE_CHANGED
21542                );
21543                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21544                // The module is told to drop just the binding it created.
21545                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21546                // Collected, because ending the scope also closes the co-tenant
21547                // route, whose GOODBYE comes first.
21548                let mut goodbyes = Vec::new();
21549                while let Ok(outbound) = plexus_rx.try_recv() {
21550                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
21551                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
21552                }
21553                assert!(
21554                    goodbyes.contains(&(bind_channel, bind_epoch)),
21555                    "{goodbyes:?}"
21556                );
21557                assert!(rig
21558                    .handler
21559                    .registry
21560                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
21561                    .unwrap()
21562                    .is_some());
21563            }
21564            assert_eq!(ended, Some(true));
21565            // The co-tenant stayed up through the change, and closed only when
21566            // the scope ended, by the drain rule rather than by the commit.
21567            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
21568        }
21569
21570        /// Each row of the drain table on one set of routes: the owner's, a
21571        /// bare carrier's, and a targeted carrier's to each of its targets.
21572        #[tokio::test]
21573        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
21574            struct Case {
21575                name: &'static str,
21576                change: fn(&mut ScopeRecord),
21577                /// Closed routes by index: owner->plexus, aft->plexus,
21578                /// broca->plexus, broca->other.
21579                closed: [Option<RouteCloseReason>; 4],
21580            }
21581            use RouteCloseReason::*;
21582            let cases = [
21583                Case {
21584                    name: "a carrier entry removed",
21585                    change: |r| {
21586                        r.carriers.retain(|c| {
21587                            c.principal
21588                                != Principal::Reserved {
21589                                    module_id: AFT.to_string(),
21590                                }
21591                        })
21592                    },
21593                    closed: [None, Some(ScopeCarrierRemoved), None, None],
21594                },
21595                Case {
21596                    name: "a target removed from a carrier",
21597                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
21598                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
21599                },
21600                Case {
21601                    name: "a bare carrier narrowed to targets",
21602                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
21603                    closed: [None, Some(ScopeCarrierRemoved), None, None],
21604                },
21605                Case {
21606                    name: "delegates turned off",
21607                    change: |r| r.attributes.delegates = false,
21608                    closed: [Some(ScopeDelegationChanged); 4],
21609                },
21610                Case {
21611                    name: "agent_id changed",
21612                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
21613                    closed: [Some(ScopeDelegationChanged); 4],
21614                },
21615                Case {
21616                    name: "a carrier added, child owners changed, the record re-sent",
21617                    change: |r| {
21618                        r.carriers.push(carrier(MAGIC, None));
21619                        r.child_owners.push(Principal::Reserved {
21620                            module_id: MAGIC.to_string(),
21621                        });
21622                    },
21623                    closed: [None; 4],
21624                },
21625                Case {
21626                    name: "a target added",
21627                    change: |r| {
21628                        r.carriers[1]
21629                            .targets
21630                            .as_mut()
21631                            .unwrap()
21632                            .push("third".to_string())
21633                    },
21634                    closed: [None; 4],
21635                },
21636                Case {
21637                    name: "delegates turned on",
21638                    change: |r| r.attributes.delegates = true,
21639                    closed: [None; 4],
21640                },
21641            ];
21642            for case in cases {
21643                let mut rig = rig().await;
21644                rig.sync(vec![session(1)]).await;
21645                let scope = || Some(rig_selector("s", Some(1)));
21646                let mut routes = [
21647                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
21648                    rig.bound(Some(AFT), PLEXUS, scope()).await,
21649                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
21650                    rig.bound(Some(BROCA), OTHER, scope()).await,
21651                ];
21652                let mut record = session(1);
21653                (case.change)(&mut record);
21654                rig.sync(vec![record]).await;
21655                for (index, expected) in case.closed.iter().enumerate() {
21656                    let route = &mut routes[index];
21657                    match expected {
21658                        Some(reason) => {
21659                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
21660                            assert_eq!(
21661                                route.closed_reason(),
21662                                *reason,
21663                                "{}: route {index}",
21664                                case.name
21665                            );
21666                        }
21667                        None => {
21668                            assert!(rig.live(route), "{}: route {index} closed", case.name);
21669                            assert!(
21670                                route.untouched(),
21671                                "{}: route {index} was told something",
21672                                case.name
21673                            );
21674                        }
21675                    }
21676                }
21677            }
21678        }
21679
21680        #[tokio::test]
21681        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
21682            // Removed, and replaced by a higher epoch.
21683            for next in [Vec::new(), vec![session(2)]] {
21684                let mut rig = rig().await;
21685                rig.sync(vec![session(1)]).await;
21686                let mut route = rig
21687                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21688                    .await;
21689                rig.sync(next).await;
21690                assert!(!rig.live(&route));
21691                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
21692            }
21693
21694            // A child whose parent ends: its routes close as parent-ended, the
21695            // child stays live, and routes under the parent close as ended.
21696            let mut rig = rig().await;
21697            let mut child = session(1);
21698            child.scope_ref = "child".to_string();
21699            child.kind = ScopeKind::Worker;
21700            child.parent = Some(ScopeParent::new(
21701                Principal::Reserved {
21702                    module_id: OWNER.to_string(),
21703                },
21704                "s".to_string(),
21705                1,
21706            ));
21707            rig.sync(vec![session(1), child.clone()]).await;
21708            let mut child_route = rig
21709                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
21710                .await;
21711            assert_eq!(
21712                child_route.stamp().unwrap().parent_state,
21713                Some(ParentState::Linked)
21714            );
21715            rig.sync(vec![child]).await;
21716            assert!(!rig.live(&child_route));
21717            assert_eq!(
21718                child_route.closed_reason(),
21719                RouteCloseReason::ScopeParentEnded
21720            );
21721        }
21722
21723        #[tokio::test]
21724        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
21725        ) {
21726            let mut rig = rig().await;
21727            rig.sync(vec![session(1)]).await;
21728            let mut route = rig
21729                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21730                .await;
21731            let before = rig.forwarding.published_scope_tag(OWNER, "s");
21732            rig.sync(vec![session(1)]).await;
21733            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21734            assert!(rig.live(&route) && route.untouched());
21735
21736            // A call in flight on the route when another carrier is added. A
21737            // forwarded REQUEST holds one credit on the route's flow until the
21738            // module answers; the router takes it exactly like this.
21739            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
21740                .forwarding
21741                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21742                .unwrap()
21743            else {
21744                panic!("the route is bound");
21745            };
21746            binding.flow.acquire_tagged(9, false).await.unwrap();
21747            let mut widened = session(1);
21748            widened.carriers.push(carrier(MAGIC, None));
21749            rig.sync(vec![widened]).await;
21750            assert!(rig.live(&route) && route.untouched());
21751            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21752            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
21753            // The call's credit is still held on an open flow, so its answer
21754            // will be delivered: closing the route would have closed the flow.
21755            assert_eq!(binding.flow.in_flight(), 1);
21756            binding
21757                .flow
21758                .acquire_tagged(10, false)
21759                .await
21760                .expect("the flow is still open");
21761        }
21762
21763        /// A swap's superseded endpoint keeps its routes until drained; ending
21764        /// the scope closes them there too.
21765        #[tokio::test]
21766        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
21767            let mut rig = rig().await;
21768            rig.sync(vec![session(1)]).await;
21769            let mut on_incumbent = rig
21770                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21771                .await;
21772
21773            // Swap plexus: register a candidate and cut over, leaving the
21774            // incumbent superseded with the route still on it.
21775            let (candidate, _candidate_rx) = wide_ctx(9);
21776            let registration = rig
21777                .handler
21778                .registry
21779                .register_candidate_with_control_ops(
21780                    manifest(PLEXUS, PROTOCOL_VERSION),
21781                    PROTOCOL_VERSION,
21782                    candidate.connection_id,
21783                    module_baseline_control_ops(),
21784                )
21785                .unwrap();
21786            rig.forwarding
21787                .register_candidate_module_connection(
21788                    candidate.connection_id,
21789                    PLEXUS.to_string(),
21790                    PROTOCOL_VERSION,
21791                    manifest_concurrency(&registration.manifest),
21792                    candidate.egress.clone(),
21793                )
21794                .unwrap();
21795            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
21796            rig.handler
21797                .registry
21798                .promote_candidate(PLEXUS)
21799                .unwrap()
21800                .unwrap();
21801            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
21802
21803            rig.sync(Vec::new()).await;
21804            assert!(!rig.live(&on_incumbent));
21805            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
21806            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21807            let goodbye = incumbent_rx
21808                .try_recv()
21809                .expect("the superseded endpoint is told")
21810                .frame;
21811            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21812        }
21813    }
21814}
21815
21816#[cfg(test)]
21817mod concurrency_default_exposure_tests {
21818    use super::*;
21819
21820    fn hello_body(role_json: &str) -> Vec<u8> {
21821        format!(
21822            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
21823        )
21824        .into_bytes()
21825    }
21826
21827    fn manifest_from(body: &[u8]) -> ModuleManifest {
21828        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
21829        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
21830            .expect("manifest parses")
21831    }
21832
21833    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
21834
21835    #[test]
21836    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
21837        let body = hello_body(&format!(
21838            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
21839        ));
21840        let manifest = manifest_from(&body);
21841        // Precondition: serde really resolved it to the default, so the typed
21842        // manifest alone cannot answer the question this probe exists for.
21843        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21844        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
21845    }
21846
21847    #[test]
21848    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
21849        let body = hello_body(&format!(
21850            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
21851        ));
21852        let manifest = manifest_from(&body);
21853        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21854        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21855    }
21856
21857    #[test]
21858    fn non_management_roles_are_never_reported() {
21859        let body = hello_body(
21860            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
21861        );
21862        let manifest = manifest_from(&body);
21863        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21864    }
21865}