Skip to main content

subc_daemon/
control.rs

1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6    fmt,
7    net::SocketAddr,
8    path::{Path, PathBuf},
9    sync::{Arc, Mutex, RwLock},
10    time::{Duration, Instant as StdInstant},
11};
12
13use crate::scopes::admission::denied_capability;
14use serde::{Deserialize, Serialize};
15use subc_control::{
16    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
17    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
18    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
19    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
20    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
21    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
22    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
23};
24use subc_protocol::{
25    call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
26    error_codes,
27    manifest::{
28        validate_hello_capability_grammar, validate_hello_confirm_operations,
29        validate_hello_display_name, validate_hello_event_declarations,
30        validate_hello_flow_operations, validate_hello_operation_deferral,
31        validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
32        validate_hello_self_signal_declarations, validate_role_declarations_value,
33        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
34        ProviderRole,
35    },
36    peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
37    provider_declarations::ProviderDeclarations,
38    scope::{
39        ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
40        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
41        SCOPE_SYNC_OP,
42    },
43    session::{
44        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
45        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
46        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
47        MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS, ROLE_VERSIONS_FIELD,
48    },
49    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
50    Priority, RouteTarget, PROTOCOL_VERSION,
51};
52use tokio::time::{timeout_at, Instant};
53use tracing::{debug, info, instrument::WithSubscriber, warn};
54
55use crate::{
56    caller_identity::CallerIdentity,
57    capability_requirements::{
58        log_duplicate_claim_events, log_manifest_change, log_requirement_events,
59        CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
60        RequirementStatus, RuntimeModule,
61    },
62    child_roster::DaemonShutdownFlag,
63    daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
64    forwarding::{
65        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
66        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
67        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
68    },
69    observability::{
70        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
71    },
72    privileged_prompt::{prompt_text, PrivilegedOperation},
73    provenance::{
74        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
75    },
76    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
77    router::{daemon_error_flags, RouteCtx, RouterError},
78    scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
79    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
80    stderr_tail::{CaptureState, TailEntry},
81    supervise::{
82        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
83        SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
84    },
85    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
86};
87
88/// Lowest envelope version this subc build will negotiate.
89///
90/// Module HELLO negotiation is exact: peers must use the daemon's locked
91/// protocol version. Older and newer peers receive `version_unsupported` and
92/// are not registered.
93pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
94
95const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
96const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
97const CAP_PING_PONG: &str = "ping_pong_v1";
98const CAP_SESSION_ATTACH: &str = "session_attach_v1";
99const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
100
101const SUBC_CONTROL_OPS: &[&str] = &[
102    "claim.mint",
103    "claim.present",
104    ops::DAEMON_SHUTDOWN_INTENT,
105    ops::ACTIVITY_RECORD,
106    ops::ACTIVITY_QUERY,
107    ops::ACTIVITY_SUBSCRIBE,
108    ops::SERVER_DESCRIBE,
109    ops::CATALOG_LIST,
110    ops::CATALOG_WATCH,
111    ops::ROUTE_OPEN,
112    ops::ROUTE_POLL,
113    ops::ROUTE_CLOSING,
114    ops::ROUTE_CLOSED,
115    ops::SUPERVISOR_LIST,
116    ops::SUPERVISOR_RESTART,
117    ops::SUPERVISOR_SWAP,
118    ops::SUPERVISOR_RELOAD,
119    ops::SUPERVISOR_RESCAN,
120    ops::SUPERVISOR_RELEASE_RESERVED,
121    ops::SUPERVISOR_SET_ENABLED,
122    ops::SUPERVISOR_HEALTH_PROBE,
123    ops::SUPERVISOR_HEALTH,
124    ops::SUPERVISOR_STDERR_TAIL,
125    ops::SUPERVISOR_TERMINALS,
126    ops::SUPERVISOR_ROUTES,
127    ops::SUPERVISOR_PROVENANCE,
128    ops::SUPERVISOR_SPAWN_SNAPSHOT,
129    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
130];
131
132/// Client ops the daemon serves but leaves out of `subc_ops`. Each is
133/// announced by a capability in `server.describe` instead, and clients check
134/// that capability before sending it. `ck daemon --json` prints `subc_ops`
135/// verbatim and scripts read that output, so a new name there would change it.
136/// A malformed body for one of these is still refused `invalid_control_body`
137/// rather than as an unknown op.
138const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[
139    ops::SUPERVISOR_RESCAN_CHECKED,
140    "config.status",
141    "config.plan",
142    "config.apply",
143    "config.stamp",
144    "config.unstamp",
145    "config.history",
146    "config.rollback",
147];
148
149mod caller_attestation;
150mod channel_operations;
151#[cfg(all(test, unix, feature = "test-support"))]
152#[path = "control/channel_operations_tests.rs"]
153mod channel_operations_tests;
154pub(crate) mod claims;
155#[cfg(test)]
156#[path = "control/config_advertisement_tests.rs"]
157mod config_advertisement_tests;
158mod config_gate;
159use caller_attestation::{caller_for_target, scope_acts_for_another};
160#[cfg(all(test, unix, feature = "test-support"))]
161#[path = "control/caller_attestation_tests.rs"]
162mod caller_attestation_tests;
163mod scope_expiry;
164
165mod agent_commands;
166pub(crate) mod catalog;
167mod rescan_checked;
168use rescan_checked::{RescanMode, RescanPlan};
169
170const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
171    "claim.mint",
172    "claim.present",
173    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
174    MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS,
175    "supervisor.live_roots",
176    "supervisor.agent_commands",
177    SCOPE_SYNC_OP,
178    SCOPE_APPLY_OP,
179    SCOPE_DESCRIBE_OP,
180    subc_protocol::scope::ROUTE_AUTHORIZE_OP,
181    "operator.confirm",
182];
183
184/// Module-originated ops the daemon answers but does not advertise in
185/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
186/// belongs to is incomplete, so no module is told it works before it does.
187const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
188
189const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
190
191/// How long subc waits for a module to ack a relayed route.bind before returning
192/// `module_timeout`. The ack waits on the module's own configure, which for AFT
193/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
194/// and DB-open work — on a cold page cache or a large repo that legitimately
195/// exceeds a couple of seconds. The default is generous because rejecting a VALID
196/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
197/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
198pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
199
200/// How many CONSECUTIVE full-budget relay timeouts against one target module
201/// open that module's bind-relay breaker.
202///
203/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
204/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
205/// so three consecutive full-budget timeouts take ~36s to observe: every client
206/// whose open contributed to opening the breaker had already given up on its
207/// own. That is what makes opening the breaker unable to turn a call that would
208/// have succeeded into a refusal — it can only make an already-failing module
209/// fail faster.
210///
211/// Two would be reachable inside one default deadline. One would convict a
212/// module on a single cold-cache bind, which is exactly the valid-but-slow case
213/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
214pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
215
216/// How long a module's bind-relay breaker stays open before exactly one
217/// `route.open` is let through as a probe.
218///
219/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
220/// re-pays a full-budget stall almost continuously, and the breaker stops being
221/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
222/// deadline: a client that starts retrying after the module recovers has to get
223/// a probe opportunity inside its own deadline, or the breaker converts a
224/// recovered module into a failed call — the failure it exists to prevent,
225/// pointed the other way.
226///
227/// 20s sits between those with room on both sides, and it caps what a wedged
228/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
229/// rather than one per `route.open` per connection. The stall that motivated
230/// this, with its measurements, is written up in
231/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
232/// waited the whole budget out.
233pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
234
235const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
236const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
237
238fn reload_verdict(
239    configured: &Path,
240    spawned_from: Option<&Path>,
241    image: subc_control::RunningImageAgreement,
242) -> PendingReloadVerdict {
243    let path = match spawned_from {
244        Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
245            ReloadPathAgreement::Match
246        }
247        Some(spawned_from) => ReloadPathAgreement::Mismatch {
248            configured: configured.to_path_buf(),
249            spawned_from: spawned_from.to_path_buf(),
250        },
251        None => ReloadPathAgreement::Unavailable {
252            reason: if matches!(
253                image,
254                subc_control::RunningImageAgreement::Unavailable {
255                    reason: subc_control::RunningImageUnavailableReason::NotRunning
256                }
257            ) {
258                ReloadPathUnavailableReason::NotRunning
259            } else {
260                ReloadPathUnavailableReason::SpawnedPathUnavailable
261            },
262        },
263    };
264    PendingReloadVerdict::new(path, image)
265}
266
267fn launch_spec_agreement(
268    configured: &crate::supervise::ModuleSpec,
269    running: bool,
270    spawned: Option<&crate::supervise::LaunchSpecDigest>,
271) -> subc_control::LaunchSpecAgreement {
272    use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
273    if !running {
274        return Agreement::Unavailable {
275            reason: Reason::NotRunning,
276        };
277    }
278    match spawned {
279        None => Agreement::Unavailable {
280            reason: Reason::SpawnedSpecUnavailable,
281        },
282        Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
283            Agreement::Match
284        }
285        Some(_) => Agreement::Mismatch,
286    }
287}
288
289fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
290    if configured == spawned_from {
291        return true;
292    }
293    #[cfg(windows)]
294    {
295        // Windows spawn records the resolved verbatim path (\\?\...), while
296        // configuration keeps its original spelling. Resolve both with the OS
297        // instead of stripping prefixes or folding case, which could conflate
298        // distinct names in case-sensitive directories. Different resolved
299        // paths still differ even if their files have identical content.
300        match (
301            std::fs::canonicalize(configured),
302            std::fs::canonicalize(spawned_from),
303        ) {
304            (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
305            _ => false,
306        }
307    }
308    #[cfg(not(windows))]
309    {
310        false
311    }
312}
313
314#[derive(Clone)]
315struct DaemonProvenanceFacts {
316    build: DaemonBuildProvenance,
317    pid: Option<u32>,
318    started_at_ms: Option<u64>,
319    start_clock: Option<crate::clock::StartClock>,
320    executable_path: Option<PathBuf>,
321    executable_identity: Option<SpawnedFileIdentity>,
322    process_start_time: Option<u64>,
323    probe: ExecutableIdentityProbe,
324}
325
326impl Default for DaemonProvenanceFacts {
327    fn default() -> Self {
328        Self {
329            build: DaemonBuildProvenance {
330                build_git_sha: None,
331                build_lock_digest: None,
332            },
333            pid: None,
334            started_at_ms: None,
335            start_clock: None,
336            executable_path: None,
337            executable_identity: None,
338            process_start_time: None,
339            probe: ExecutableIdentityProbe::default(),
340        }
341    }
342}
343
344#[derive(Debug, Clone)]
345struct SupervisorRescanContext {
346    supervisor: Supervisor,
347    config_path: PathBuf,
348    configured_port: Option<u16>,
349    storage_config: Option<crate::daemon_config::StorageConfig>,
350    admission_facts_carrier_module_id: Option<String>,
351    admission_facts_targets: Option<Vec<String>>,
352    scope_authority_owners: Vec<String>,
353    federated_scope_owner: String,
354    privileged_operator_prompt: PrivilegedOperatorPrompt,
355    /// Counts executed rescans that set out to change the supervised modules'
356    /// staged or applied configuration, counted before the first change so a
357    /// rescan that fails partway still counts. With the daemon incarnation it
358    /// names the state a `supervisor.rescan_checked` preview was computed
359    /// against.
360    generation: Arc<std::sync::atomic::AtomicU64>,
361    module_entries: Arc<Mutex<rescan_checked::ModuleEntries>>,
362}
363
364/// Refusal labels passed to `observe_route_open_refusal` that mean the target
365/// module is not serving right now, and so open or extend an outage in the
366/// route outage tracker. Every one of them is only reachable after the target
367/// was found in the registry, which is what keeps an arbitrary client-chosen
368/// id from ever creating tracker state.
369///
370/// Deliberately absent: `not_registered` and `removed` (the id may be
371/// anything a client sent, and a removed module is gone on purpose),
372/// `protocol_none` (such a module never serves routes, so nothing is out),
373/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
374/// capability and admission-facts refusals (they refuse the caller, not a
375/// module outage), and `relay_reservation_failed` (its code ranges over
376/// capacity limits as well as a vanished connection). Capacity, breaker,
377/// relay-timeout and module-rejection refusals do not pass through that
378/// function at all; the breaker logs its own transitions.
379///
380/// The two not-serving refusals that bypass that function record themselves
381/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
382/// `required_capability_unprovided` is not tracked: the module itself is up,
383/// and the outage belongs to the missing provider.
384const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
385    "reloading",
386    "supervisor_not_live",
387    "registration_not_active",
388    "no_forwarding_connection",
389    "relay_send_failed",
390];
391
392/// Observe test request cancellation relative to the connection's cleanup.
393#[cfg(test)]
394struct ActivityRequestPause {
395    connection: ConnectionId,
396    started: tokio::sync::Notify,
397    cancelled: tokio::sync::Notify,
398    cleanup_started: std::sync::atomic::AtomicBool,
399    cancelled_after_cleanup: std::sync::atomic::AtomicBool,
400}
401
402#[cfg(test)]
403struct ActivityRequestCancellation(Arc<ActivityRequestPause>);
404
405#[cfg(test)]
406impl Drop for ActivityRequestCancellation {
407    fn drop(&mut self) {
408        self.0.cancelled_after_cleanup.store(
409            self.0
410                .cleanup_started
411                .load(std::sync::atomic::Ordering::Acquire),
412            std::sync::atomic::Ordering::Release,
413        );
414        self.0.cancelled.notify_one();
415    }
416}
417
418/// A test hold: signal arrival, then wait for release on the holding thread.
419#[cfg(test)]
420type TestPause = (
421    tokio::sync::oneshot::Sender<()>,
422    std::sync::mpsc::Receiver<()>,
423);
424
425/// A test hook run once at a fixed point in a handler.
426#[cfg(test)]
427type TestHook = Box<dyn FnOnce() + Send>;
428
429/// A test hold for async code: signal arrival, then await release.
430#[cfg(test)]
431type AsyncTestPause = (
432    tokio::sync::oneshot::Sender<()>,
433    tokio::sync::oneshot::Receiver<()>,
434);
435
436/// Real channel-0 control handler for subc itself.
437#[derive(Clone)]
438pub struct ControlHandler {
439    #[cfg(test)]
440    activity_request_pause: Arc<Mutex<Option<Arc<ActivityRequestPause>>>>,
441    /// Holds connection cleanup at a numbered point: 0 is just after the agent
442    /// declaration queue closes, 1 just after the registration channel is
443    /// forgotten.
444    #[cfg(test)]
445    cleanup_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
446    /// Runs once in `route.open`'s launch-nonce attestation, right after the
447    /// nonce is resolved, to interleave a nonce withdrawal there.
448    #[cfg(test)]
449    after_nonce_attestation: Arc<Mutex<Option<TestHook>>>,
450    /// Holds a registration channel's HELLO after its reader resolved the
451    /// channel and before the blocking HELLO handler starts.
452    #[cfg(test)]
453    hello_ingress_pause: Arc<Mutex<Option<AsyncTestPause>>>,
454    /// Holds an agent command declaration at a numbered point in its handler.
455    #[cfg(test)]
456    agent_commands_pause: Arc<Mutex<Option<(u8, TestPause)>>>,
457    claims: Arc<claims::Claims>,
458    registry: Arc<Registry>,
459    config_runtime: Option<Arc<crate::config_gate::runtime::RuntimeOwner>>,
460    config_review_admission: Arc<dyn crate::config_gate::ReviewAdmission>,
461    config_review_admission_is_default: bool,
462    activity: Option<crate::activity::ActivityLedger>,
463    activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
464    catalog_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
465    forwarding: Arc<ForwardingTable>,
466    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
467    supervisor: SupervisorHandle,
468    pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
469    agent_declarations: agent_commands::Queues,
470    list_admission: Arc<tokio::sync::Semaphore>,
471    subc_capabilities: Arc<[String]>,
472    /// Daemon-wide route.bind relay budget. Used as the fallback when the
473    /// target module has no per-module override in
474    /// `route_bind_relay_timeouts`.
475    route_bind_relay_timeout: Duration,
476    /// Per-module route.bind relay budget overrides, keyed by module id. When
477    /// `handle_route_open` resolves the deadline for a target module, a
478    /// per-module entry wins over the daemon-wide value above.
479    route_bind_relay_timeouts: BTreeMap<String, Duration>,
480    /// Per-target-module bind-relay breaker state. Shared with the forwarding
481    /// table, which is where a new module connection resets it.
482    route_bind_breakers: RouteBindBreakers,
483    /// Live relay admissions keyed by target module. Shared through the
484    /// forwarding table so cloned or separately built handlers enforce one cap.
485    route_bind_concurrency: RouteBindConcurrency,
486    /// Start and end of each module's not-serving period as seen by
487    /// `route.open`, so an outage gets one line at each edge instead of only
488    /// the per-refusal INFO lines. Taken from the forwarding table, so every
489    /// handler built over one table shares it.
490    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
491    /// Consecutive relay timeouts that open a module's breaker.
492    route_bind_breaker_threshold: u32,
493    /// How long a breaker stays open before one probe is admitted.
494    route_bind_breaker_cooldown: Duration,
495    health_probe_timeout: Duration,
496    /// Central storage policy. When set, each registering module receives its
497    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
498    storage_config: Option<crate::daemon_config::StorageConfig>,
499    /// The machine id established at boot, served on every HELLO_ACK and on
500    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
501    /// changes the file, never this value. `None` serves no id.
502    machine_id: Option<crate::machine_id::MachineId>,
503    admission_facts_carrier_module_id: Option<String>,
504    admission_facts_targets: Option<Vec<String>>,
505    /// Scope records with their sync authorities and tombstones; see
506    /// `crate::scopes`. Shared by clones of this handler, so every connection
507    /// reads and writes one table.
508    scopes: Arc<RwLock<ScopeTable>>,
509    /// Wakes the expiry loop when an accepted scope change may have added a
510    /// deadline, so the loop can wait without ticking while no live scope has
511    /// one. A permit is stored when nothing waits, so no wake is lost.
512    scope_deadline_added: Arc<tokio::sync::Notify>,
513    /// The configured `scope_authority_owners`, kept so a rescan can report a
514    /// changed value as needing a daemon restart; rescan never applies it.
515    scope_authority_owners: Vec<String>,
516    federated_scope_owner: String,
517    /// Fixed at startup so rescan cannot change the policy that guards rescan itself.
518    privileged_operator_prompt: PrivilegedOperatorPrompt,
519    /// The local and peer address of each accepted TCP client connection, read
520    /// from the daemon's own socket, so a privileged request can name its caller.
521    caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
522    build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
523    observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
524    captured_peers: Arc<Mutex<HashMap<ConnectionId, subc_os::local_peer::CapturedPeer>>>,
525    config_desktop_stamps:
526        Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
527    local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
528    pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
529    pub(crate) operator_apps:
530        Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
531    /// One launch entry per app, behind a leaf lock: no other lock is taken
532    /// while it is held. Entries are memory-only, so a daemon or
533    /// installed-service restart drops them and a relaunched daemon never
534    /// mistakes an older process for its own launch.
535    pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
536    /// Serializes `app.open` launches: the liveness check in the decision and
537    /// the later publish must not interleave across concurrent calls, or two
538    /// calls could both launch and the second publish would orphan the first
539    /// process. Refusals and `already_running` answers share the same lock;
540    /// `app.open` is rare, so the cost is negligible.
541    pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
542    // Portable wire samples exercise dispatch without starting an OS process.
543    #[cfg(test)]
544    pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
545    #[cfg(test)]
546    pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
547    #[cfg(test)]
548    recovery_exit_test_after_step: Option<fn(usize) -> Result<(), crate::config_gate::GateError>>,
549    #[cfg(test)]
550    recovery_exit_test_completed: Option<fn(u64)>,
551    #[cfg(test)]
552    apply_reconcile_pause: Arc<Mutex<Option<AsyncTestPause>>>,
553    #[cfg(test)]
554    recovery_exit_test_quiet_retry: bool,
555    #[cfg(test)]
556    recovery_exit_test_retry_pause: Option<fn(u64) -> Option<PathBuf>>,
557    #[cfg(test)]
558    config_delivery_test_wakes:
559        Option<tokio::sync::mpsc::UnboundedSender<(&'static str, tokio::time::Instant)>>,
560    #[cfg(all(test, unix))]
561    config_delivery_test_idle:
562        Option<tokio::sync::mpsc::UnboundedSender<tokio::sync::oneshot::Sender<()>>>,
563    /// Test-only Windows signature and flash hooks. Production builds carry
564    /// neither: with the publisher unset, the production table never reaches
565    /// verification, and the default attention hook only logs.
566    #[allow(dead_code)]
567    #[cfg(windows)]
568    pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
569    #[allow(dead_code)]
570    #[cfg(windows)]
571    pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
572    /// Test-only tracking fault hook: makes opening the tracking handle on
573    /// the launched pid fail, so a test can prove an untrackable launch is
574    /// stopped rather than left running.
575    #[allow(dead_code)]
576    #[cfg(windows)]
577    pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
578    /// Test-only count of Windows spawn attempts, so a test can prove a
579    /// refused launch started no process. Unit tests only.
580    #[cfg(all(test, windows))]
581    pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
582    #[cfg(test)]
583    caller_lookup: Option<TestCallerLookup>,
584    /// The launch nonce each module connection presented at HELLO, which is how
585    /// a `scope.sync` is matched to the owner's current launch.
586    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
587    rescan: Option<SupervisorRescanContext>,
588    connected_clients: ConnectedClients,
589    counters: DaemonCounters,
590    /// Why this daemon serves no local endpoint, when binding one failed. `None`
591    /// when the endpoint is served, and also on a handler that was never told.
592    local_endpoint_unavailable: Option<String>,
593    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
594    /// The supervisor's daemon-shutdown flag. Once it is set, requirement
595    /// changes are logged at INFO with `cause=daemon_shutdown` instead of at
596    /// their usual severity (see [`log_requirement_events`]). A handler built
597    /// without one has a flag that is never set.
598    daemon_shutdown: DaemonShutdownFlag,
599    daemon_provenance: DaemonProvenanceFacts,
600    #[cfg(test)]
601    control_dispatch_delay: Option<Duration>,
602    #[cfg(test)]
603    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
604    /// Holds `route.open` after the route.bind is queued to the module and
605    /// before it is marked enqueued, so a test can make the module's reply (or
606    /// a client teardown) land exactly inside that window.
607    #[cfg(test)]
608    route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
609    /// Holds a HELLO after its registry entry is reserved and before its
610    /// forwarding endpoint is installed, so a test can read the catalog and
611    /// open routes while the module is registered but not yet routable. A
612    /// blocking barrier because the HELLO handler is synchronous; the test
613    /// runs the HELLO on its own thread.
614    #[cfg(test)]
615    hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
616    #[cfg(test)]
617    hello_ack_to_publication_hold: Option<Arc<std::sync::Barrier>>,
618    #[cfg(test)]
619    hello_before_publication_hold: Option<Arc<std::sync::Barrier>>,
620    #[cfg(test)]
621    hello_after_launch_check_hold: Option<Arc<std::sync::Barrier>>,
622    #[cfg(test)]
623    hello_inside_publication_hold: Option<Arc<std::sync::Barrier>>,
624}
625
626impl fmt::Debug for ControlHandler {
627    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
628        f.debug_struct("ControlHandler")
629            .field("registry", &self.registry)
630            .field("forwarding", &self.forwarding)
631            .field("process_liveness", &self.process_liveness.is_some())
632            .field("supervisor", &self.supervisor)
633            .field("subc_capabilities", &self.subc_capabilities)
634            .finish()
635    }
636}
637
638struct RouteOpenRequest {
639    target: RouteTarget,
640    identity: BindIdentity,
641    consumer_identity: Option<ConsumerIdentity>,
642    consumer_capabilities: Option<Vec<String>>,
643    role_versions: Option<BTreeMap<String, String>>,
644    busy_refusals: Option<bool>,
645    admission_facts: Option<serde_json::Value>,
646    scope: Option<ScopeSelector>,
647    accepts_close_reasons: Option<Vec<String>>,
648}
649
650#[derive(Clone, Copy)]
651enum RouteOpenPhase {
652    Admission,
653    ProjectRoot,
654    EgressReservation,
655    BindSend,
656    BindAckWait,
657}
658
659/// Store elapsed time for each route-open phase, including attempts that end
660/// in an early refusal or backend error. Admission also includes rechecks and
661/// commit bookkeeping performed between the other timed phases.
662struct RouteOpenTimings {
663    module_id: String,
664    principal: String,
665    admission: Duration,
666    project_root: Duration,
667    egress_reservation: Duration,
668    bind_send: Duration,
669    bind_ack_wait: Duration,
670    phase: Option<RouteOpenPhase>,
671    phase_started_at: Instant,
672}
673
674impl RouteOpenTimings {
675    fn new() -> Self {
676        Self {
677            module_id: String::new(),
678            // Refusals before attestation must not claim a verified principal.
679            principal: "unresolved".to_string(),
680            admission: Duration::ZERO,
681            project_root: Duration::ZERO,
682            egress_reservation: Duration::ZERO,
683            bind_send: Duration::ZERO,
684            bind_ack_wait: Duration::ZERO,
685            phase: Some(RouteOpenPhase::Admission),
686            phase_started_at: Instant::now(),
687        }
688    }
689
690    fn finish(&mut self) {
691        let elapsed = self.phase_started_at.elapsed();
692        match self.phase.take() {
693            Some(RouteOpenPhase::Admission) => self.admission += elapsed,
694            Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
695            Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
696            Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
697            Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
698            None => {}
699        }
700    }
701
702    fn enter(&mut self, phase: RouteOpenPhase) {
703        self.finish();
704        self.phase = Some(phase);
705        self.phase_started_at = Instant::now();
706    }
707}
708
709struct RouteBindReservationGuard {
710    forwarding: Arc<ForwardingTable>,
711    endpoint: ModuleEndpointId,
712    relay_corr: u64,
713    armed: bool,
714}
715
716struct ModuleControlRpcGuard {
717    forwarding: Arc<ForwardingTable>,
718    endpoint: ModuleEndpointId,
719    corr: u64,
720    armed: bool,
721}
722
723impl ModuleControlRpcGuard {
724    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
725        Self {
726            forwarding,
727            endpoint,
728            corr,
729            armed: true,
730        }
731    }
732
733    fn disarm(&mut self) {
734        self.armed = false;
735    }
736}
737
738impl Drop for ModuleControlRpcGuard {
739    fn drop(&mut self) {
740        if self.armed {
741            let _ = self
742                .forwarding
743                .cancel_module_control_rpc(self.endpoint, self.corr);
744        }
745    }
746}
747
748impl RouteBindReservationGuard {
749    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
750        Self {
751            forwarding,
752            endpoint,
753            relay_corr,
754            armed: true,
755        }
756    }
757
758    fn release_and_disarm(&mut self) {
759        if !self.armed {
760            return;
761        }
762        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
763            self.endpoint,
764            self.relay_corr,
765            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
766        ) {
767            send_goodbye_target_best_effort(
768                &self.forwarding.counters(),
769                &target,
770                "canceled route.bind",
771            );
772        }
773        self.armed = false;
774    }
775
776    fn disarm(&mut self) {
777        self.armed = false;
778    }
779}
780
781impl Drop for RouteBindReservationGuard {
782    fn drop(&mut self) {
783        self.release_and_disarm();
784    }
785}
786
787/// Per-target-module circuit breaker around the `route.bind` relay.
788///
789/// The connection reader is serial per connection, so a module whose `on_bind`
790/// sits on the ack blocks every LATER frame on the connections that call it,
791/// including calls to unrelated modules. This does not make any module's bind
792/// fast; it stops the daemon paying the full budget again and again for a
793/// condition it has already observed.
794///
795/// State is keyed by TARGET MODULE and shared by every connection: a wedged
796/// module wedges everyone, so what one connection learned should protect the
797/// rest.
798///
799/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
800/// relay to that module has actually timed out, and is removed again when a
801/// relay is accepted or the module reconnects, so it cannot grow with traffic
802/// or with modules that behave.
803///
804/// # Why a `std` mutex here is not the head-of-line defect again
805///
806/// Acquisition never awaits. The critical section is a hash lookup plus a few
807/// integer updates, with no I/O and no `.await` inside it, so a reader task
808/// cannot be descheduled behind it the way it can behind
809/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
810/// primitive, held for the same kind of work, as the refusal counter this very
811/// path already increments.
812///
813/// It is also NOT on the data-plane splice path: only `route.open` and module
814/// registration touch it, so bound-route frames gain no state check and no
815/// contention.
816#[derive(Debug, Clone, Default)]
817pub(crate) struct RouteBindBreakers {
818    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
819}
820
821#[derive(Debug, Clone, Default)]
822pub(crate) struct RouteBindConcurrency {
823    modules: Arc<Mutex<HashMap<String, usize>>>,
824}
825
826struct RouteBindConcurrencyGuard {
827    concurrency: RouteBindConcurrency,
828    module_id: String,
829}
830
831impl RouteBindConcurrency {
832    /// Admit without waiting. Waiting here would move the bind stall from the
833    /// module reply to a semaphore and restore reader head-of-line blocking.
834    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
835        let mut modules = self
836            .modules
837            .lock()
838            .expect("route.bind concurrency mutex poisoned");
839        let in_flight = modules.entry(module_id.to_string()).or_default();
840        if *in_flight >= limit {
841            return Err(*in_flight);
842        }
843        *in_flight += 1;
844        Ok(RouteBindConcurrencyGuard {
845            concurrency: self.clone(),
846            module_id: module_id.to_string(),
847        })
848    }
849}
850
851impl Drop for RouteBindConcurrencyGuard {
852    fn drop(&mut self) {
853        let mut modules = self
854            .concurrency
855            .modules
856            .lock()
857            .expect("route.bind concurrency mutex poisoned");
858        let remove = {
859            let in_flight = modules
860                .get_mut(&self.module_id)
861                .expect("admitted route.bind has a concurrency entry");
862            *in_flight -= 1;
863            *in_flight == 0
864        };
865        if remove {
866            modules.remove(&self.module_id);
867        }
868    }
869}
870
871#[derive(Debug, Default)]
872struct ModuleBreakerState {
873    /// Relay timeouts observed with no accepted relay in between.
874    consecutive_timeouts: u32,
875    /// `Some` while the breaker is open: the instant the cooldown expires and
876    /// the next arrival may probe. `None` means closed.
877    cooldown_until: Option<Instant>,
878    /// A half-open probe has been admitted and has not settled yet. This is
879    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
880    /// that read the cooldown, so concurrent opens arriving at the moment the
881    /// cooldown expires cannot all decide that they are the probe.
882    probe_in_flight: Option<Arc<()>>,
883}
884
885/// What the breaker decided for one `route.open`, before any relay work.
886enum RouteBindAdmission<'a> {
887    Admitted {
888        guard: RouteBindBreakerGuard<'a>,
889        /// This open is the single half-open probe, so the transition is worth
890        /// one log line.
891        probe: bool,
892    },
893    Refused {
894        consecutive_timeouts: u32,
895        /// What is left of the cooldown. Zero when the refusal is because the
896        /// one probe is already in flight rather than because the cooldown has
897        /// not elapsed.
898        retry_in: Duration,
899        probe_in_flight: bool,
900    },
901}
902
903/// An outstanding admission, which must be told how its relay settled.
904///
905/// `Drop` settles it as inconclusive, so an early return between admission and
906/// the relay -- or the whole handler being cancelled when the client
907/// disconnects -- releases a half-open probe slot instead of leaving the
908/// breaker wedged half-open with no further probes.
909struct RouteBindBreakerGuard<'a> {
910    breakers: RouteBindBreakers,
911    module_id: &'a str,
912    probe_token: Option<Arc<()>>,
913    settled: bool,
914}
915
916impl RouteBindBreakerGuard<'_> {
917    /// The module answered within the budget and took the bind. THE ONLY
918    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
919    /// breaker, which is a transition worth logging.
920    fn record_accepted(&mut self) -> bool {
921        self.settled = true;
922        self.breakers.record_accepted(self.module_id)
923    }
924
925    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
926    /// COUNTS TOWARD OPENING.
927    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
928        self.settled = true;
929        self.breakers.record_timeout(
930            self.module_id,
931            self.probe_token.as_ref(),
932            threshold,
933            cooldown,
934        )
935    }
936
937    /// Everything else: the module REJECTED the bind, its connection went away
938    /// mid-relay, or the waiter was cancelled.
939    ///
940    /// None of these is evidence that a module is slow, and each already has
941    /// its own refusal with its own code. A module that rejects a bind in
942    /// microseconds is healthy and must never be convicted for it; a module
943    /// that died has said nothing about the module that replaces it. So these
944    /// neither increment nor reset the count -- they only release a probe slot.
945    fn record_inconclusive(&mut self) {
946        self.settled = true;
947        self.breakers
948            .record_inconclusive(self.module_id, self.probe_token.as_ref());
949    }
950}
951
952impl Drop for RouteBindBreakerGuard<'_> {
953    fn drop(&mut self) {
954        if !self.settled {
955            self.breakers
956                .record_inconclusive(self.module_id, self.probe_token.as_ref());
957        }
958    }
959}
960
961/// The breaker moved to open, reported so the caller can log it outside the
962/// lock. Opening is rare and load-bearing; the refusals that follow are
963/// frequent and are counted rather than logged.
964struct BreakerOpened {
965    consecutive_timeouts: u32,
966    /// True when a failed probe re-opened an already-open breaker, which reads
967    /// very differently in a log from a first opening.
968    reopened_after_probe: bool,
969}
970
971impl RouteBindBreakers {
972    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
973        self.modules
974            .lock()
975            .expect("route.bind breaker mutex poisoned")
976    }
977
978    /// Decide whether this `route.open` may attempt its relay. Takes the map
979    /// lock and nothing else, and never awaits.
980    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
981        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
982            probe: probe_token.is_some(),
983            guard: RouteBindBreakerGuard {
984                breakers: self.clone(),
985                module_id,
986                probe_token,
987                settled: false,
988            },
989        };
990
991        let mut modules = self.lock();
992        let Some(state) = modules.get_mut(module_id) else {
993            return admitted(None);
994        };
995        let Some(cooldown_until) = state.cooldown_until else {
996            return admitted(None);
997        };
998        if state.probe_in_flight.is_some() {
999            return RouteBindAdmission::Refused {
1000                consecutive_timeouts: state.consecutive_timeouts,
1001                retry_in: Duration::ZERO,
1002                probe_in_flight: true,
1003            };
1004        }
1005        let now = Instant::now();
1006        if now < cooldown_until {
1007            return RouteBindAdmission::Refused {
1008                consecutive_timeouts: state.consecutive_timeouts,
1009                retry_in: cooldown_until - now,
1010                probe_in_flight: false,
1011            };
1012        }
1013        let token = Arc::new(());
1014        state.probe_in_flight = Some(Arc::clone(&token));
1015        admitted(Some(token))
1016    }
1017
1018    fn record_accepted(&self, module_id: &str) -> bool {
1019        self.lock()
1020            .remove(module_id)
1021            .is_some_and(|state| state.cooldown_until.is_some())
1022    }
1023
1024    fn record_timeout(
1025        &self,
1026        module_id: &str,
1027        probe_token: Option<&Arc<()>>,
1028        threshold: u32,
1029        cooldown: Duration,
1030    ) -> Option<BreakerOpened> {
1031        let mut modules = self.lock();
1032        let state = modules.entry(module_id.to_string()).or_default();
1033        let was_open = state.cooldown_until.is_some();
1034        let was_probe = Self::owns_probe(state, probe_token);
1035        if was_probe {
1036            state.probe_in_flight = None;
1037        }
1038        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
1039        if state.consecutive_timeouts < threshold {
1040            return None;
1041        }
1042        state.cooldown_until = Some(Instant::now() + cooldown);
1043        Some(BreakerOpened {
1044            consecutive_timeouts: state.consecutive_timeouts,
1045            reopened_after_probe: was_open && was_probe,
1046        })
1047    }
1048
1049    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
1050        // A relay can finish after the breaker was reset or after it opened
1051        // again and started a new probe. Only the guard whose token matches the
1052        // active probe may release it, so a late relay never frees a newer probe.
1053        state
1054            .probe_in_flight
1055            .as_ref()
1056            .zip(token)
1057            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
1058    }
1059
1060    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
1061        if let Some(state) = self.lock().get_mut(module_id) {
1062            if Self::owns_probe(state, probe_token) {
1063                state.probe_in_flight = None;
1064            }
1065        }
1066    }
1067
1068    /// Discard what was learned about a module, because the process it was
1069    /// learned about is gone. Returns the discarded count when it was non-zero.
1070    ///
1071    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
1072    /// `module_id` is a configuration identity that outlives any particular
1073    /// child; what the breaker observed was the process behind the module
1074    /// connection of the moment. When a new connection registers under that id
1075    /// the verdict's subject no longer exists, so the verdict is stale by
1076    /// construction rather than merely likely to be wrong. Keeping it would
1077    /// apply a dead process's record to a live one, which is the same defect
1078    /// class this breaker exists to stop the daemon committing.
1079    ///
1080    /// A half-open probe in flight is discarded with the rest: it was a
1081    /// question about the old process.
1082    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
1083        self.lock()
1084            .remove(module_id)
1085            .map(|state| state.consecutive_timeouts)
1086            .filter(|discarded| *discarded > 0)
1087    }
1088
1089    /// Open breakers, for the `server.describe` counters object. `None` when
1090    /// none is open, so the key stays absent rather than present-and-empty.
1091    ///
1092    /// This is the operator's answer to "is this module refusing instantly or
1093    /// is it fine?", which look identical from a client that retries and then
1094    /// succeeds.
1095    fn open_snapshot(&self) -> Option<serde_json::Value> {
1096        let now = Instant::now();
1097        let modules = self.lock();
1098        let open = modules
1099            .iter()
1100            .filter_map(|(module_id, state)| {
1101                let cooldown_until = state.cooldown_until?;
1102                Some((
1103                    module_id.clone(),
1104                    serde_json::json!({
1105                        "consecutive_timeouts": state.consecutive_timeouts,
1106                        "cooldown_remaining_ms":
1107                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
1108                        "probe_in_flight": state.probe_in_flight.is_some(),
1109                    }),
1110                ))
1111            })
1112            .collect::<serde_json::Map<String, serde_json::Value>>();
1113        (!open.is_empty()).then_some(serde_json::Value::Object(open))
1114    }
1115}
1116
1117impl ControlHandler {
1118    pub(crate) fn install_registration_service(
1119        &self,
1120        router: std::sync::Weak<crate::Router>,
1121        auth: crate::server::ServerAuth,
1122    ) {
1123        self.supervisor.install_registration_service(router, auth);
1124    }
1125    pub fn new(registry: Arc<Registry>) -> Self {
1126        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
1127    }
1128
1129    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
1130        let counters = forwarding.counters();
1131        // Taken from the forwarding table rather than created here, so that the
1132        // breaker a `route.open` consults is the same one a module's
1133        // registration resets, however many handlers are built over one table.
1134        let route_bind_breakers = forwarding.route_bind_breakers();
1135        let route_bind_concurrency = forwarding.route_bind_concurrency();
1136        let route_outages = forwarding.route_outages();
1137        Self {
1138            registry,
1139            #[cfg(test)]
1140            activity_request_pause: Arc::new(Mutex::new(None)),
1141            #[cfg(test)]
1142            cleanup_pause: Arc::new(Mutex::new(None)),
1143            #[cfg(test)]
1144            after_nonce_attestation: Arc::new(Mutex::new(None)),
1145            #[cfg(test)]
1146            hello_ingress_pause: Arc::new(Mutex::new(None)),
1147            #[cfg(test)]
1148            agent_commands_pause: Arc::new(Mutex::new(None)),
1149            claims: Arc::new(claims::Claims::default()),
1150            config_runtime: None,
1151            config_review_admission: Arc::new(crate::config_gate::MissingReviewAdmission),
1152            config_review_admission_is_default: true,
1153            activity: None,
1154            activity_streams: Arc::new(Mutex::new(HashMap::new())),
1155            catalog_streams: Arc::new(Mutex::new(HashMap::new())),
1156            forwarding,
1157            process_liveness: None,
1158            supervisor: SupervisorHandle::new(),
1159            resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1160                env!("CARGO_PKG_VERSION").to_owned(),
1161            )),
1162            agent_declarations: agent_commands::Queues::default(),
1163            list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1164            subc_capabilities: Arc::from([
1165                CAP_MANIFEST_REGISTRATION.to_string(),
1166                CAP_CHANNEL_LIFECYCLE.to_string(),
1167                CAP_PING_PONG.to_string(),
1168                CAP_SESSION_ATTACH.to_string(),
1169                CAP_ADMISSION_FACTS_RELAY.to_string(),
1170                CAP_SCOPES_V1.to_string(),
1171                subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1172                subc_protocol::peer::CALLER_ATTESTATION_CAPABILITY.to_string(),
1173                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1174                subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1175                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1176                subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1177                subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1178                subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1179                subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1180                subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1181                subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1182                subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1183                subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1184                subc_protocol::ROUTE_CLOSE_REASONS_CAPABILITY.to_string(),
1185            ]),
1186            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1187            route_bind_relay_timeouts: BTreeMap::new(),
1188            route_bind_breakers,
1189            route_bind_concurrency,
1190            route_outages,
1191            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1192            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1193            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1194            storage_config: None,
1195            machine_id: None,
1196            admission_facts_carrier_module_id: None,
1197            admission_facts_targets: None,
1198            scopes: Arc::new(RwLock::new(ScopeTable::new(
1199                crate::daemon_config::default_scope_authority_owners(),
1200            ))),
1201            scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1202            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1203            federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1204            privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1205            caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1206            build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1207            observed_peers: Arc::new(Mutex::new(HashMap::new())),
1208            captured_peers: Arc::new(Mutex::new(HashMap::new())),
1209            config_desktop_stamps: Arc::new(Mutex::new(HashMap::new())),
1210            local_connections: Arc::new(Mutex::new(HashSet::new())),
1211            operator_app_policy: Default::default(),
1212            operator_apps: Arc::new(Mutex::new(HashMap::new())),
1213            launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1214            app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1215            #[cfg(test)]
1216            app_open_test_platform: None,
1217            #[cfg(test)]
1218            app_open_test_launch: None,
1219            #[cfg(test)]
1220            recovery_exit_test_after_step: None,
1221            #[cfg(test)]
1222            recovery_exit_test_completed: None,
1223            #[cfg(test)]
1224            apply_reconcile_pause: Arc::new(Mutex::new(None)),
1225            #[cfg(test)]
1226            recovery_exit_test_quiet_retry: false,
1227            #[cfg(test)]
1228            recovery_exit_test_retry_pause: None,
1229            #[cfg(test)]
1230            config_delivery_test_wakes: None,
1231            #[cfg(all(test, unix))]
1232            config_delivery_test_idle: None,
1233            #[cfg(windows)]
1234            windows_verify: None,
1235            #[cfg(windows)]
1236            windows_flash: None,
1237            #[cfg(windows)]
1238            windows_track: None,
1239            #[cfg(all(test, windows))]
1240            windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1241            #[cfg(test)]
1242            caller_lookup: None,
1243            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1244            rescan: None,
1245            connected_clients: ConnectedClients::new(),
1246            local_endpoint_unavailable: None,
1247            counters,
1248            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1249            daemon_shutdown: DaemonShutdownFlag::default(),
1250            daemon_provenance: DaemonProvenanceFacts::default(),
1251            #[cfg(test)]
1252            control_dispatch_delay: None,
1253            #[cfg(test)]
1254            provenance_probe_override: None,
1255            #[cfg(test)]
1256            route_bind_relay_sent_hold: None,
1257            #[cfg(test)]
1258            hello_registry_to_forwarding_hold: None,
1259            #[cfg(test)]
1260            hello_ack_to_publication_hold: None,
1261            #[cfg(test)]
1262            hello_before_publication_hold: None,
1263            #[cfg(test)]
1264            hello_after_launch_check_hold: None,
1265            #[cfg(test)]
1266            hello_inside_publication_hold: None,
1267        }
1268    }
1269
1270    /// Set the central storage policy: registering modules then receive their
1271    /// resolved storage descriptor in HELLO_ACK.
1272    pub fn with_storage_config(
1273        mut self,
1274        storage_config: Option<crate::daemon_config::StorageConfig>,
1275    ) -> Self {
1276        self.storage_config = storage_config;
1277        self
1278    }
1279
1280    /// Open the ledger on its own writer thread; an unavailable sink does not
1281    /// prevent the daemon from serving routing or supervisor requests.
1282    pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1283        self.activity = Some(crate::activity::ActivityLedger::open(
1284            run_dir.join("activity.jsonl"),
1285            incarnation,
1286        ));
1287        self.supervisor.set_activity(self.activity.clone());
1288        self
1289    }
1290
1291    /// Set the machine id served to every registering module (HELLO_ACK) and on
1292    /// `server.describe`.
1293    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1294        self.machine_id = machine_id;
1295        self
1296    }
1297
1298    /// Configure the exact reserved module and target ids permitted to relay
1299    /// opaque admission facts. Config-file loading validates this authority;
1300    /// this builder keeps the same policy available to embedded test daemons.
1301    pub fn with_admission_facts_config(
1302        mut self,
1303        carrier_module_id: Option<String>,
1304        targets: Option<Vec<String>>,
1305    ) -> Self {
1306        self.admission_facts_carrier_module_id = carrier_module_id;
1307        self.admission_facts_targets = targets;
1308        self
1309    }
1310
1311    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
1312    /// Replaces the scope table with an empty one under the new list, so call it
1313    /// while building the handler, before any module can sync.
1314    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1315        self.scopes = Arc::new(RwLock::new(
1316            ScopeTable::new(owners.iter().cloned())
1317                .with_federated_owner(self.federated_scope_owner.clone()),
1318        ));
1319        self.scope_authority_owners = owners;
1320        self
1321    }
1322
1323    /// Set the exclusive federated owner while building the handler, before syncs.
1324    pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1325        self.scopes = Arc::new(RwLock::new(
1326            ScopeTable::new(self.scope_authority_owners.iter().cloned())
1327                .with_federated_owner(owner.clone()),
1328        ));
1329        self.federated_scope_owner = owner;
1330        self
1331    }
1332
1333    /// Set the startup policy before installing the rescan context or serving requests.
1334    pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1335        self.privileged_operator_prompt = mode;
1336        self
1337    }
1338
1339    /// Override the route.bind relay timeout. Used by tests that assert the
1340    /// timeout path so they don't block on the production-safe default.
1341    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1342        self.route_bind_relay_timeout = timeout;
1343        self
1344    }
1345
1346    /// Install per-module route.bind relay budget overrides. A module id
1347    /// listed here wins over the daemon-wide default set via
1348    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
1349    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
1350    /// the same `Duration` the bind path will use.
1351    pub fn with_route_bind_relay_timeouts(
1352        mut self,
1353        timeouts: impl IntoIterator<Item = (String, Duration)>,
1354    ) -> Self {
1355        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1356        self
1357    }
1358
1359    /// Resolve the route.bind relay budget for a specific target module id.
1360    /// Per-module overrides win; the daemon-wide value (set via
1361    /// `with_route_bind_relay_timeout` or the built-in default) is the
1362    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
1363    /// the same resolution `handle_route_open` will use.
1364    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1365        self.route_bind_relay_timeouts
1366            .get(module_id)
1367            .copied()
1368            .unwrap_or(self.route_bind_relay_timeout)
1369    }
1370
1371    /// Override the per-module bind-relay breaker policy.
1372    ///
1373    /// Used by tests, which cannot spend three production budgets opening a
1374    /// breaker or twenty seconds waiting for its cooldown. The production
1375    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
1376    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
1377    /// reasoning for the numbers.
1378    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1379        self.route_bind_breaker_threshold = threshold.max(1);
1380        self.route_bind_breaker_cooldown = cooldown;
1381        self
1382    }
1383
1384    #[cfg(test)]
1385    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1386        self.health_probe_timeout = timeout;
1387        self
1388    }
1389
1390    #[cfg(test)]
1391    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1392        self.control_dispatch_delay = Some(delay);
1393        self
1394    }
1395
1396    /// `route.open` waits on `hold` twice after queueing a route.bind: once to
1397    /// announce it is parked, once to be released. Build the barrier for two
1398    /// parties, the test being the other one.
1399    #[cfg(test)]
1400    pub(crate) fn with_route_bind_relay_sent_hold(
1401        mut self,
1402        hold: Arc<tokio::sync::Barrier>,
1403    ) -> Self {
1404        self.route_bind_relay_sent_hold = Some(hold);
1405        self
1406    }
1407
1408    /// A HELLO waits on `hold` twice between reserving its registry entry and
1409    /// installing its forwarding endpoint: once to announce it is parked, once
1410    /// to be released. Build the barrier for two parties, the test being the
1411    /// other one, and run the HELLO on a thread of its own.
1412    #[cfg(test)]
1413    pub(crate) fn with_hello_registry_to_forwarding_hold(
1414        mut self,
1415        hold: Arc<std::sync::Barrier>,
1416    ) -> Self {
1417        self.hello_registry_to_forwarding_hold = Some(hold);
1418        self
1419    }
1420
1421    #[cfg(test)]
1422    fn hold_between_registry_and_forwarding(&self) {
1423        if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1424            hold.wait();
1425            hold.wait();
1426        }
1427    }
1428
1429    /// Park after the ACK is queued and forwarding is installed, until a test
1430    /// has read the ACK and issued a registry lookup from another connection.
1431    #[cfg(test)]
1432    fn hold_between_ack_and_publication(&self) {
1433        if let Some(hold) = &self.hello_ack_to_publication_hold {
1434            hold.wait();
1435            hold.wait();
1436        }
1437    }
1438
1439    #[cfg(all(test, unix))]
1440    pub(crate) fn with_hello_before_publication_hold(
1441        mut self,
1442        hold: Arc<std::sync::Barrier>,
1443    ) -> Self {
1444        self.hello_before_publication_hold = Some(hold);
1445        self
1446    }
1447
1448    #[cfg(all(test, unix))]
1449    pub(crate) fn with_hello_after_launch_check_hold(
1450        mut self,
1451        hold: Arc<std::sync::Barrier>,
1452    ) -> Self {
1453        self.hello_after_launch_check_hold = Some(hold);
1454        self
1455    }
1456
1457    #[cfg(all(test, unix))]
1458    pub(crate) fn with_hello_inside_publication_hold(
1459        mut self,
1460        hold: Arc<std::sync::Barrier>,
1461    ) -> Self {
1462        self.hello_inside_publication_hold = Some(hold);
1463        self
1464    }
1465
1466    pub fn with_process_liveness(
1467        mut self,
1468        process_liveness: Arc<dyn ModuleProcessLiveness>,
1469    ) -> Self {
1470        self.process_liveness = Some(process_liveness);
1471        self
1472    }
1473
1474    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1475        self.supervisor = supervisor;
1476        if let Some(owner) = &self.config_runtime {
1477            self.supervisor.install_binding_retirement(owner);
1478        }
1479        self.supervisor.set_activity(self.activity.clone());
1480        self
1481    }
1482
1483    pub fn with_daemon_provenance(
1484        mut self,
1485        pid: u32,
1486        started_at_ms: u64,
1487        executable_path: Option<PathBuf>,
1488        build_git_sha: Option<String>,
1489        build_lock_digest: Option<String>,
1490    ) -> Self {
1491        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1492        let process_start_time = process_start_time(pid);
1493        self.daemon_provenance = DaemonProvenanceFacts {
1494            build: DaemonBuildProvenance {
1495                build_git_sha,
1496                build_lock_digest,
1497            },
1498            pid: Some(pid),
1499            started_at_ms: Some(started_at_ms),
1500            start_clock: None,
1501            executable_path,
1502            executable_identity,
1503            process_start_time,
1504            probe: ExecutableIdentityProbe::default(),
1505        };
1506        self
1507    }
1508
1509    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1510        self.daemon_provenance.start_clock = Some(clock);
1511        self
1512    }
1513
1514    /// Inject the Unix wall-clock millisecond source used by every scope expiry check.
1515    pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1516        self.scopes
1517            .write()
1518            .unwrap_or_else(|poisoned| poisoned.into_inner())
1519            .set_wall_clock(Arc::new(clock));
1520        self
1521    }
1522
1523    #[cfg(test)]
1524    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1525        self.provenance_probe_override = Some(result);
1526        self
1527    }
1528
1529    /// Install the configured module set and its reserved capability bindings.
1530    /// Bindings are configuration-scoped and may point at a provider that has not
1531    /// been installed yet, so this does not require the bound module to exist.
1532    pub fn with_capability_config(
1533        self,
1534        modules: impl IntoIterator<Item = (String, bool)>,
1535        reserved_capabilities: BTreeMap<String, String>,
1536    ) -> Self {
1537        self.capability_evaluator
1538            .configure(modules, reserved_capabilities);
1539        self
1540    }
1541
1542    /// Share the supervisor's daemon-shutdown flag with this handler, so
1543    /// requirement changes after the daemon has begun its announced shutdown
1544    /// are not reported as faults.
1545    pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1546        self.daemon_shutdown = flag;
1547        self
1548    }
1549
1550    pub fn with_supervisor_rescan(
1551        mut self,
1552        supervisor: Supervisor,
1553        config_path: impl Into<PathBuf>,
1554        configured_port: Option<u16>,
1555    ) -> Self {
1556        self.rescan = Some(SupervisorRescanContext {
1557            supervisor,
1558            config_path: config_path.into(),
1559            configured_port,
1560            storage_config: self.storage_config.clone(),
1561            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1562            admission_facts_targets: self.admission_facts_targets.clone(),
1563            scope_authority_owners: self.scope_authority_owners.clone(),
1564            federated_scope_owner: self.federated_scope_owner.clone(),
1565            privileged_operator_prompt: self.privileged_operator_prompt,
1566            generation: Arc::default(),
1567            module_entries: Arc::default(),
1568        });
1569        self
1570    }
1571
1572    pub(crate) fn with_supervisor_rescan_entries(
1573        self,
1574        modules: &[crate::daemon_config::ConfiguredModule],
1575    ) -> Self {
1576        if let Some(context) = &self.rescan {
1577            let mut entries = context
1578                .module_entries
1579                .lock()
1580                .unwrap_or_else(|e| e.into_inner());
1581            entries.seed(modules);
1582        }
1583        self
1584    }
1585
1586    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1587        self.connected_clients = connected_clients;
1588        self
1589    }
1590
1591    /// Report why this daemon serves no local endpoint, in `server.describe`, so
1592    /// an operator can see a bind failure without reading the daemon log.
1593    pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1594        self.local_endpoint_unavailable = reason;
1595        self
1596    }
1597
1598    pub fn forwarding(&self) -> Arc<ForwardingTable> {
1599        Arc::clone(&self.forwarding)
1600    }
1601
1602    pub(crate) fn counters(&self) -> DaemonCounters {
1603        self.counters.clone()
1604    }
1605
1606    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1607    /// requirement event without depending on an operator polling a status command.
1608    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1609        let handler = Arc::downgrade(&self);
1610        let evaluator = self.capability_evaluator.clone();
1611        tokio::spawn(async move {
1612            loop {
1613                evaluator.wait_for_change_or_deadline().await;
1614                let Some(handler) = handler.upgrade() else {
1615                    break;
1616                };
1617                handler.refresh_capability_requirements();
1618            }
1619        });
1620    }
1621
1622    /// The timer only prompts sweeps. Each sweep reads wall time again, so a
1623    /// deadline passed during sleep is enforced on the first tick after wake.
1624    /// While no live scope carries a deadline the loop waits for an accepted
1625    /// scope change instead of ticking, so a daemon without deadlines never
1626    /// wakes for expiry.
1627    pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1628        let context = scope_expiry::ScopeExpiryContext::new(&self);
1629        let deadline_added = self.scope_deadline_added.clone();
1630        tokio::spawn(async move {
1631            loop {
1632                if !context.has_deadlines() {
1633                    deadline_added.notified().await;
1634                    continue;
1635                }
1636                tokio::time::sleep(Duration::from_secs(1)).await;
1637                if let Err(error) = context.sweep() {
1638                    warn!(%error, "scope expiry sweep failed");
1639                }
1640            }
1641        });
1642    }
1643
1644    #[cfg(test)]
1645    pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1646        scope_expiry::ScopeExpiryContext::new(self).sweep()
1647    }
1648
1649    fn publish_scope_changes(
1650        &self,
1651        changes: &[crate::scopes::ScopeTagChange],
1652        expired: &[crate::scopes::ScopeExpired],
1653    ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1654        scope_expiry::ScopeExpiryContext::new(self).publish_scope_changes(changes, expired)
1655    }
1656
1657    fn runtime_capability_snapshot(
1658        &self,
1659    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1660        let runtime = self
1661            .supervisor
1662            .list()
1663            .into_iter()
1664            .map(|module| {
1665                let status = module.status().map_err(|err| {
1666                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1667                })?;
1668                Ok(RuntimeModule {
1669                    module_id: status.module_id,
1670                    state: status.state,
1671                    enabled: status.enabled,
1672                })
1673            })
1674            .collect::<Result<Vec<_>, RouterError>>()?;
1675        // Include every registration, published or not. The capability
1676        // evaluator counts a module as providing its capabilities as soon as it
1677        // has registered, even before `catalog.list` shows it, so a HELLO in
1678        // progress does not change other modules' required-capability
1679        // verdicts. See `Registry::list_registered_modules`.
1680        let registrations = self.registry.list_registered_modules().map_err(|err| {
1681            RouterError::backend(
1682                0,
1683                0,
1684                format!("failed to list capability registrations: {err}"),
1685            )
1686        })?;
1687        let registrations = registrations
1688            .into_iter()
1689            .map(|registration| RegisteredModule {
1690                module_id: registration.manifest.module_id,
1691                module_version: registration.manifest.module_version,
1692                capabilities: registration.manifest.capabilities,
1693            })
1694            .collect();
1695        Ok((runtime, registrations))
1696    }
1697
1698    /// The capability side effects of a module becoming the active registration
1699    /// for its id: cache its manifest (logging how it differs from the cached
1700    /// one), run the deny census when its declarations call for one, and recompute the
1701    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1702    /// candidate's does not, and the supervisor does it at promotion instead,
1703    /// through [`crate::supervise::SwapPromotionObserver`].
1704    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1705        let cached_registration = RegisteredModule {
1706            module_id: registration.manifest.module_id.clone(),
1707            module_version: registration.manifest.module_version.clone(),
1708            capabilities: registration.manifest.capabilities.clone(),
1709        };
1710        let change = self.capability_evaluator.record_hello(&cached_registration);
1711        log_manifest_change(&cached_registration.module_id, &change);
1712        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1713            self.enforce_capability_denies();
1714        }
1715        self.refresh_capability_requirements();
1716    }
1717
1718    /// Point the shared supervisor handle at this handler for swap promotions.
1719    /// Called wherever a handler is put behind the `Arc` the router serves, so
1720    /// it can be held weakly.
1721    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1722        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1723            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1724        self.supervisor.set_swap_promotion_observer(observer);
1725    }
1726
1727    pub fn refresh_capability_requirements(&self) {
1728        match self.runtime_capability_snapshot() {
1729            Ok((runtime, registrations)) => {
1730                log_requirement_events(
1731                    self.capability_evaluator
1732                        .evaluate_now(&runtime, &registrations),
1733                    &self.daemon_shutdown,
1734                );
1735            }
1736            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1737        }
1738    }
1739
1740    /// Reconcile only live, attested route bindings after a capability deny edge
1741    /// or target claim was added. This is deliberately a control-plane census:
1742    /// the opaque forwarding hot path must not grow a per-frame capability check.
1743    fn enforce_capability_denies(&self) {
1744        let registrations = match self.registry.list_registered_modules() {
1745            Ok(snapshot) => snapshot,
1746            Err(err) => {
1747                warn!(error = %err, "failed to read registrations for capability deny census");
1748                return;
1749            }
1750        };
1751        let manifests = registrations
1752            .into_iter()
1753            .map(|registration| {
1754                (
1755                    registration.manifest.module_id.clone(),
1756                    registration.manifest,
1757                )
1758            })
1759            .collect::<BTreeMap<_, _>>();
1760        let census = match self.forwarding.route_census(None) {
1761            Ok(census) => census,
1762            Err(err) => {
1763                warn!(error = %err, "failed to read route census for capability deny enforcement");
1764                return;
1765            }
1766        };
1767
1768        for (target_module_id, routes) in census {
1769            let Some(target_manifest) = manifests.get(&target_module_id) else {
1770                continue;
1771            };
1772            let mut closed_routes = Vec::new();
1773            let mut goodbyes = Vec::new();
1774            for route in routes {
1775                let Principal::Reserved {
1776                    module_id: opening_module_id,
1777                } = &route.principal
1778                else {
1779                    continue;
1780                };
1781                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1782                    continue;
1783                };
1784                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1785                    continue;
1786                };
1787
1788                match self.forwarding.release_client_route(
1789                    route.goodbye_target.connection_id,
1790                    route.goodbye_target.channel,
1791                    route.goodbye_target.epoch,
1792                ) {
1793                    Ok(RouteRelease::Removed(module_goodbye)) => {
1794                        warn!(
1795                            opening_module_id,
1796                            target_module_id,
1797                            capability,
1798                            "force-closing route because an attested capability deny edge now matches"
1799                        );
1800                        goodbyes.push(module_goodbye);
1801                        goodbyes.push(route.goodbye_target.clone());
1802                        closed_routes.push(route);
1803                    }
1804                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1805                    Err(err) => warn!(
1806                        opening_module_id,
1807                        target_module_id,
1808                        capability,
1809                        error = %err,
1810                        "failed to force-close capability-denied route"
1811                    ),
1812                }
1813            }
1814
1815            if closed_routes.is_empty() {
1816                continue;
1817            }
1818            send_route_control_pushes(
1819                &self.forwarding,
1820                closed_routes,
1821                ClientControlPush::RouteClosed {
1822                    module_id: target_module_id,
1823                    channels: Vec::new(),
1824                    reason: RouteCloseReason::CapabilityDenied,
1825                    drained: false,
1826                    abandoned: 0,
1827                    excluded_subscriptions: 0,
1828                    terminal: Some(false),
1829                },
1830            );
1831            // route.closed records the reason; the route GOODBYE ends the
1832            // client's handle and settles any calls still pending on it.
1833            self.emit_route_goodbyes(goodbyes);
1834        }
1835    }
1836
1837    /// Why a registered module is not accepting new route binds, or `None` when
1838    /// it is. This is the module's effective readiness: its declared readiness
1839    /// first, then every `need: required` capability it declares evaluating to
1840    /// `provided`. `route.open` and `catalog.list` both read it here so the
1841    /// catalog never reports a module routable that `route.open` would refuse.
1842    fn not_ready_reason(
1843        &self,
1844        registration: &crate::registry::ModuleRegistration,
1845    ) -> Option<NotReadyReason> {
1846        if !registration.ready {
1847            return Some(NotReadyReason {
1848                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1849                capability: None,
1850            });
1851        }
1852        self.first_unprovided_required_capability(registration)
1853            .map(|capability| NotReadyReason {
1854                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1855                capability: Some(capability),
1856            })
1857    }
1858
1859    /// The lexicographically first capability this registration declares
1860    /// `need: required` whose evaluator verdict is not `provided`.
1861    ///
1862    /// The verdicts are the capability evaluator's own; nothing here decides
1863    /// what "provided" means. The evaluator counts a capability provided as
1864    /// soon as a module claiming it has REGISTERED, not once that module is
1865    /// ready. That distinction is what keeps two modules that require each
1866    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1867    /// is ready", each would wait for the other to become ready first and
1868    /// neither ever would. Do not tighten it to readiness.
1869    ///
1870    /// A required capability with no verdict at all means this registration's
1871    /// HELLO or catalog.update landed after the last recompute; recompute once
1872    /// rather than let a missing verdict read as either answer. If it is still
1873    /// missing (the recompute itself failed) the capability counts as
1874    /// unprovided: the refusal is retryable, and routing a module whose
1875    /// required provider is unknown is the outcome this check exists to stop.
1876    fn first_unprovided_required_capability(
1877        &self,
1878        registration: &crate::registry::ModuleRegistration,
1879    ) -> Option<String> {
1880        let required = registration
1881            .manifest
1882            .capabilities
1883            .iter()
1884            .flat_map(|declarations| declarations.requires.iter())
1885            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1886            .map(|requirement| requirement.capability.as_str())
1887            .collect::<BTreeSet<_>>();
1888        if required.is_empty() {
1889            return None;
1890        }
1891        let module_id = registration.manifest.module_id.as_str();
1892        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1893        if required
1894            .iter()
1895            .any(|capability| verdict(capability).is_none())
1896        {
1897            self.refresh_capability_requirements();
1898        }
1899        required
1900            .into_iter()
1901            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1902            .map(str::to_string)
1903    }
1904
1905    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1906        self.capability_evaluator
1907            .statuses()
1908            .into_iter()
1909            .map(capability_requirement_status)
1910            .collect()
1911    }
1912
1913    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1914    /// registration-release watch. The signal is what the supervisor waits on
1915    /// before spawning a replacement, so it must only fire once forwarding
1916    /// teardown is also done (see [`Self::cleanup_connection`] /
1917    /// [`Self::handle_goodbye`]). Unpublished HELLO rollback uses the separate
1918    /// registry-only path below because publication already owns the spawn lock.
1919    fn deregister_connection(
1920        &self,
1921        connection_id: ConnectionId,
1922        reason: RegistrationEndReason,
1923    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1924        if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
1925            spawn.end_primary_registration();
1926        }
1927        self.registry
1928            .deregister_connection_with_reason(connection_id, reason)
1929    }
1930
1931    // HELLO publication owns the spawn transition guard. claim.mint is allowed
1932    // only after HELLO acceptance, so a failed first HELLO has no bearer tokens
1933    // or claim-bound consumers to revoke. Remove only its unpublished registry
1934    // entry without re-entering primary-registration teardown.
1935    fn rollback_unpublished_hello(
1936        &self,
1937        connection: ConnectionId,
1938    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1939        self.registry.deregister_connection_with_reason(
1940            connection,
1941            RegistrationEndReason::RegistrationFailed,
1942        )
1943    }
1944
1945    fn request_refusal_close(
1946        &self,
1947        connection: ConnectionId,
1948        sink: Option<&crate::FrameSink>,
1949        reason: CloseReason,
1950    ) {
1951        if let Some(sink) = sink {
1952            sink.terminate_ingress();
1953        }
1954        if self.claims.close_notifications_enabled() {
1955            self.forwarding.request_connection_close(connection, reason);
1956        }
1957    }
1958
1959    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1960        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1961            return None;
1962        }
1963        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1964            parse_client_control_request(&frame.body)
1965        else {
1966            return None;
1967        };
1968        Some(target_module_id(&target).to_string())
1969    }
1970
1971    pub(crate) fn route_open_capacity_refusal(
1972        &self,
1973        ctx: &RouteCtx,
1974        frame: &Frame,
1975        target_module_id: &str,
1976        in_flight: usize,
1977        limit: usize,
1978    ) -> Result<Frame, RouterError> {
1979        // Sent before the request reaches `handle_route_open`, so the caller
1980        // fields come from the raw frame here.
1981        let caller = self.route_open_caller_from_frame(ctx, frame);
1982        self.route_open_admission_refusal_frame(
1983            ctx,
1984            &caller,
1985            frame,
1986            target_module_id,
1987            "open_admission_full",
1988            (in_flight, limit),
1989            format!(
1990                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1991            ),
1992        )
1993    }
1994
1995    fn route_open_target_capacity_refusal(
1996        &self,
1997        ctx: &RouteCtx,
1998        caller: &RouteOpenCaller,
1999        frame: &Frame,
2000        target_module_id: &str,
2001        in_flight: usize,
2002    ) -> Result<Frame, RouterError> {
2003        self.route_open_admission_refusal_frame(
2004            ctx,
2005            caller,
2006            frame,
2007            target_module_id,
2008            "target_binds_full",
2009            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
2010            format!(
2011                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2012            ),
2013        )
2014    }
2015
2016    /// Admission pressure clears as existing binds settle, so its refusal must
2017    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
2018    /// `module_reloading`, `module_warming`, `target_unavailable`, or
2019    /// `module_timeout`. `target_unavailable` is honest for an attempt that
2020    /// cannot currently reach its target; `module_timeout` would falsely claim
2021    /// that a wait expired. A new, cleaner code would be terminal to deployed
2022    /// clients, so it requires a client-tolerance rollout before daemon emission.
2023    #[allow(clippy::too_many_arguments)]
2024    fn route_open_admission_refusal_frame(
2025        &self,
2026        ctx: &RouteCtx,
2027        caller: &RouteOpenCaller,
2028        frame: &Frame,
2029        target_module_id: &str,
2030        reason: &'static str,
2031        (in_flight, limit): (usize, usize),
2032        message: impl Into<String>,
2033    ) -> Result<Frame, RouterError> {
2034        let code = error_codes::TARGET_UNAVAILABLE;
2035        self.counters.increment_route_open_refused(code);
2036        info!(
2037            target: "control",
2038            code,
2039            reason,
2040            module_id = ?target_module_id,
2041            connection_id = ctx.connection_id.get(),
2042            principal = %caller.principal,
2043            scope = %caller.scope,
2044            harness = %caller.harness,
2045            in_flight,
2046            limit,
2047            "route.open refused"
2048        );
2049        control_error_frame(frame, code, message.into())
2050    }
2051
2052    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
2053    ///
2054    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
2055    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
2056    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
2057    #[cfg(test)]
2058    pub fn handle_control(
2059        &self,
2060        connection_id: ConnectionId,
2061        frame: Frame,
2062    ) -> Result<Vec<Frame>, RouterError> {
2063        match frame.header.ty {
2064            FrameType::Ping => Ok(vec![pong(&frame)?]),
2065            FrameType::Hello => self.handle_hello(connection_id, None, frame),
2066            FrameType::Goodbye => self.handle_goodbye(connection_id),
2067            ty => Ok(vec![control_error_frame(
2068                &frame,
2069                "unsupported_control_frame",
2070                format!("unsupported channel-0 frame {ty:?}"),
2071            )?]),
2072        }
2073    }
2074
2075    pub async fn handle_control_frame(
2076        &self,
2077        ctx: &RouteCtx,
2078        frame: Frame,
2079    ) -> Result<Vec<Frame>, RouterError> {
2080        self.handle_control_frame_timed(ctx, frame, None).await
2081    }
2082
2083    #[cfg(any(test, feature = "test-support"))]
2084    pub fn with_test_operator_app_policy(
2085        mut self,
2086        policy: crate::operator_apps::TestOperatorAppPolicy,
2087    ) -> Self {
2088        self.operator_app_policy = policy.into();
2089        self
2090    }
2091
2092    /// Inject the Windows signature check for tests. The injected check
2093    /// answers the verified version string or the rejection reason; the
2094    /// daemon enforces the floor itself on the returned version.
2095    #[cfg(all(windows, any(test, feature = "test-support")))]
2096    pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
2097        self.windows_verify = Some(verify);
2098        self
2099    }
2100
2101    /// Record which pids the daemon would flash for `already_running`.
2102    #[cfg(all(windows, any(test, feature = "test-support")))]
2103    pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
2104        self.windows_flash = Some(flash);
2105        self
2106    }
2107
2108    /// Inject a tracking failure for tests: makes `track_launched_process`
2109    /// fail so a test can prove a launch that cannot be tracked is stopped,
2110    /// never left running.
2111    #[cfg(all(windows, any(test, feature = "test-support")))]
2112    pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
2113        self.windows_track = Some(track);
2114        self
2115    }
2116
2117    pub(crate) fn record_captured_peer(
2118        &self,
2119        connection: ConnectionId,
2120        peer: subc_os::local_peer::CapturedPeer,
2121    ) {
2122        self.record_observed_peer(connection, peer.observation().clone());
2123        let identity = self.operator_app_policy.config_identity(peer.clone());
2124        #[cfg(windows)]
2125        let identity = identity.or_else(|| self.config_windows_launch_identity(&peer));
2126        if let Some(identity) = identity {
2127            // Keep the connect-time launch incarnation. Looking it up afresh
2128            // from a numeric pipe-client PID could match a later reused PID.
2129            self.config_desktop_stamps
2130                .lock()
2131                .unwrap_or_else(|p| p.into_inner())
2132                .insert(connection, identity);
2133        }
2134        self.captured_peers
2135            .lock()
2136            .unwrap_or_else(|p| p.into_inner())
2137            .insert(connection, peer.clone());
2138        #[cfg(windows)]
2139        if let Some(stamp) = self.stamp_windows_launch(&peer) {
2140            self.operator_apps
2141                .lock()
2142                .unwrap_or_else(|p| p.into_inner())
2143                .insert(connection, stamp);
2144        }
2145        if let Some(stamp) = self.operator_app_policy.stamp(peer) {
2146            self.operator_apps
2147                .lock()
2148                .unwrap_or_else(|p| p.into_inner())
2149                .insert(connection, stamp);
2150        }
2151    }
2152
2153    pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
2154        self.local_connections
2155            .lock()
2156            .unwrap_or_else(|p| p.into_inner())
2157            .insert(connection);
2158    }
2159
2160    // Keep the connection-level delivery decision shared with route.bind. The
2161    // target's capability opt-in is separate from recognition of the caller.
2162    fn connection_operator_app(
2163        &self,
2164        stamp: Option<&crate::operator_apps::OperatorAppStamp>,
2165    ) -> Option<subc_protocol::peer::OperatorApp> {
2166        stamp
2167            .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
2168            .map(|stamp| stamp.app.clone())
2169    }
2170
2171    fn handle_connection_describe(
2172        &self,
2173        ctx: &RouteCtx,
2174        frame: Frame,
2175    ) -> Result<Vec<Frame>, RouterError> {
2176        if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
2177            return Ok(vec![control_error_frame(
2178                &frame,
2179                "invalid_control_body",
2180                "malformed connection.describe body",
2181            )?]);
2182        }
2183        let local = self
2184            .local_connections
2185            .lock()
2186            .unwrap_or_else(|p| p.into_inner())
2187            .contains(&ctx.connection_id);
2188        let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
2189        let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
2190        Ok(vec![control_response_body_frame(
2191            &frame,
2192            &subc_control::ConnectionDescribeReply::new(local, operator_app),
2193            ops::CONNECTION_DESCRIBE,
2194        )?])
2195    }
2196
2197    pub(crate) fn record_observed_peer(
2198        &self,
2199        connection: ConnectionId,
2200        peer: subc_os::local_peer::ObservedPeer,
2201    ) {
2202        use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
2203        let peer = match peer {
2204            OsPeer::Unobserved => ObservedPeer::unobserved(),
2205            OsPeer::Observed(peer) => {
2206                let observed = match peer.user {
2207                    PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
2208                    PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
2209                };
2210                observed
2211                    .with_exe(
2212                        peer.exe
2213                            .and_then(|path| path.into_os_string().into_string().ok()),
2214                    )
2215                    .with_signing_id(peer.signing_id)
2216            }
2217        };
2218        self.observed_peers
2219            .lock()
2220            .unwrap_or_else(|p| p.into_inner())
2221            .insert(connection, peer);
2222    }
2223
2224    pub(crate) fn record_caller_endpoints(
2225        &self,
2226        connection: ConnectionId,
2227        endpoints: Option<(SocketAddr, SocketAddr)>,
2228    ) {
2229        let mut callers = self
2230            .caller_endpoints
2231            .lock()
2232            .unwrap_or_else(|p| p.into_inner());
2233        callers.remove(&connection);
2234        if let Some(endpoints) = endpoints {
2235            callers.insert(connection, endpoints);
2236        }
2237    }
2238
2239    #[cfg(test)]
2240    pub(crate) fn with_caller_lookup_for_test(
2241        mut self,
2242        lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2243    ) -> Self {
2244        self.caller_lookup = Some(Arc::new(lookup));
2245        self
2246    }
2247
2248    async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2249        let endpoints = self
2250            .caller_endpoints
2251            .lock()
2252            .unwrap_or_else(|p| p.into_inner())
2253            .get(&connection)
2254            .copied();
2255        #[cfg(test)]
2256        let lookup = self.caller_lookup.clone();
2257        tokio::task::spawn_blocking(move || {
2258            #[cfg(test)]
2259            if let Some(lookup) = lookup {
2260                return lookup(endpoints);
2261            }
2262            endpoints
2263                .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2264                .unwrap_or_default()
2265        })
2266        .await
2267        .unwrap_or_default()
2268    }
2269
2270    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2271        frame.header.channel == 0
2272            && frame.header.ty == FrameType::Request
2273            && self
2274                .forwarding
2275                .module_endpoint_for_connection(ctx.connection_id)
2276                .is_ok_and(|endpoint| endpoint.is_none())
2277            && parse_client_control_request(&frame.body)
2278                .ok()
2279                .is_some_and(|request| privileged_operation(&request).is_some())
2280    }
2281
2282    pub(crate) async fn handle_control_frame_timed(
2283        &self,
2284        ctx: &RouteCtx,
2285        frame: Frame,
2286        dispatch_started_at: Option<StdInstant>,
2287    ) -> Result<Vec<Frame>, RouterError> {
2288        let first = ctx.egress.first_ingress_frame();
2289        self.handle_control_frame_ingress(ctx, frame, dispatch_started_at, first)
2290            .await
2291    }
2292
2293    pub(crate) async fn handle_control_frame_ingress(
2294        &self,
2295        ctx: &RouteCtx,
2296        frame: Frame,
2297        dispatch_started_at: Option<StdInstant>,
2298        first: bool,
2299    ) -> Result<Vec<Frame>, RouterError> {
2300        if ctx.egress.ingress_terminal() {
2301            return Ok(Vec::new());
2302        }
2303        #[cfg(test)]
2304        self.claims.record_dispatch(frame.header.corr);
2305        let claim_binding = self.claims.binding(ctx.connection_id);
2306        let mut prefix_attempt = (frame.header.ty == FrameType::Hello
2307            && claim_binding
2308                .as_ref()
2309                .is_some_and(|binding| binding.prefix_module_id().is_some()))
2310        .then(|| self.prefix_hello_attempt(ctx));
2311        if claim_binding
2312            .as_ref()
2313            .is_some_and(|binding| binding.prefix_module_id().is_some())
2314        {
2315            let registered = self
2316                .forwarding
2317                .module_endpoint_for_connection(ctx.connection_id)
2318                .map_err(RouterError::Forwarding)?
2319                .is_some();
2320            if !registered && frame.header.ty != FrameType::Hello {
2321                return self
2322                    .refuse_claim_connection(ctx, &frame, "claim_hello_required")
2323                    .await;
2324            }
2325            if registered && frame.header.ty == FrameType::Goodbye {
2326                ctx.egress.terminate_ingress();
2327                let result = self.handle_goodbye(ctx.connection_id);
2328                self.claims.connection_closed(ctx.connection_id);
2329                self.request_refusal_close(
2330                    ctx.connection_id,
2331                    Some(&ctx.egress),
2332                    CloseReason::new("claim_registration_ended", "prefix registration ended"),
2333                );
2334                return result;
2335            }
2336            if registered && frame.header.ty == FrameType::Request {
2337                let allowed =
2338                    serde_json::from_slice::<ControlOpProbe>(&frame.body).is_ok_and(|probe| {
2339                        matches!(
2340                            probe.op.as_str(),
2341                            "catalog.update"
2342                                | "catalog.update_role_declarations"
2343                                | "scope.describe"
2344                                | "supervisor.live_roots"
2345                                | "activity.record"
2346                        )
2347                    });
2348                if !allowed {
2349                    return Ok(vec![control_error_frame(
2350                        &frame,
2351                        channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2352                        "prefix registrations have only brokered module operations",
2353                    )?]);
2354                }
2355            }
2356            if registered
2357                && frame.header.ty == FrameType::Push
2358                && !serde_json::from_slice::<ControlOpProbe>(&frame.body)
2359                    .is_ok_and(|probe| probe.op == "route.status")
2360            {
2361                return Ok(vec![control_error_frame(
2362                    &frame,
2363                    channel_operations::CLAIM_OPERATION_NOT_ALLOWED,
2364                    "prefix registrations may push only route.status for routes they serve",
2365                )?]);
2366            }
2367        }
2368        match frame.header.ty {
2369            FrameType::Ping => Ok(vec![pong(&frame)?]),
2370            FrameType::Hello => {
2371                if claim_binding
2372                    .as_ref()
2373                    .is_some_and(|binding| binding.prefix_module_id().is_none())
2374                {
2375                    return self
2376                        .refuse_claim_connection(ctx, &frame, "claim_hello_not_allowed")
2377                        .await;
2378                }
2379                // Resolved once, here: the blocking HELLO below may outlive this
2380                // reader and run after cleanup has forgotten the channel, and it
2381                // must still be judged as this channel's HELLO.
2382                let channel_spawn = self.supervisor.registration_spawn(ctx.connection_id);
2383                if let Some(spawn) = &channel_spawn {
2384                    spawn.wait_launch().await;
2385                    // Refused launches close the channel. Leave their pending
2386                    // HELLO unanswered instead of racing a reply against shutdown.
2387                    if !spawn.launch_ready() {
2388                        ctx.egress.terminate_ingress();
2389                        return Ok(Vec::new());
2390                    }
2391                    #[cfg(test)]
2392                    self.pause_hello_ingress_for_test().await;
2393                } else if claim_binding.is_none() {
2394                    if let Ok(hello) = serde_json::from_slice::<serde_json::Value>(&frame.body) {
2395                        if let (Some(module), Some(nonce)) = (
2396                            hello
2397                                .get("manifest")
2398                                .and_then(|value| value.get("module_id"))
2399                                .and_then(|value| value.as_str()),
2400                            hello.get("launch_nonce").and_then(|value| value.as_str()),
2401                        ) {
2402                            if !self
2403                                .config_runtime
2404                                .as_ref()
2405                                .is_some_and(|owner| owner.module_is_gated(module))
2406                                && !self.supervisor.wait_program_binding(module, nonce).await
2407                            {
2408                                self.request_refusal_close(
2409                                    ctx.connection_id,
2410                                    Some(&ctx.egress),
2411                                    CloseReason::new(
2412                                        "program_refused",
2413                                        "the spawned program binding failed",
2414                                    ),
2415                                );
2416                                return Ok(vec![control_error_frame(
2417                                    &frame,
2418                                    if self
2419                                        .config_runtime
2420                                        .as_ref()
2421                                        .is_some_and(|owner| owner.is_recovering())
2422                                    {
2423                                        "config_gated"
2424                                    } else {
2425                                        "program_refused"
2426                                    },
2427                                    "the spawned program binding failed",
2428                                )?]);
2429                            }
2430                        }
2431                    }
2432                }
2433                let handler = self.clone();
2434                let connection = ctx.connection_id;
2435                let egress = ctx.egress.clone();
2436                let correlation = frame.header.corr;
2437                let dispatcher = tracing::dispatcher::get_default(|dispatcher| dispatcher.clone());
2438                let span = tracing::Span::current();
2439                let hello_binding = claim_binding.clone();
2440                let replies = tokio::task::spawn_blocking(move || {
2441                    tracing::dispatcher::with_default(&dispatcher, || {
2442                        span.in_scope(|| {
2443                            handler.handle_hello_checked(
2444                                connection,
2445                                Some(egress),
2446                                frame,
2447                                true,
2448                                hello_binding,
2449                                channel_spawn,
2450                            )
2451                        })
2452                    })
2453                })
2454                .await
2455                .map_err(|error| RouterError::backend(0, correlation, error.to_string()))??;
2456                #[cfg(test)]
2457                if prefix_attempt.is_some() {
2458                    self.claims.pause_prefix_commit_for_test(false).await;
2459                }
2460                if let Some(attempt) = &mut prefix_attempt {
2461                    let accepted = !replies
2462                        .iter()
2463                        .any(|reply| reply.header.ty == FrameType::Error)
2464                        && self
2465                            .registry
2466                            .get_module_by_connection(ctx.connection_id)
2467                            .map_err(|err| RouterError::backend(0, correlation, err.to_string()))?
2468                            .is_some();
2469                    if accepted {
2470                        attempt.accepted = true;
2471                    } else {
2472                        ctx.egress.terminate_ingress();
2473                        for reply in replies {
2474                            ctx.egress.send_flushed(reply).await?;
2475                        }
2476                        return Ok(Vec::new());
2477                    }
2478                }
2479                Ok(replies)
2480            }
2481            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2482            FrameType::Cancel => {
2483                // Channel-0 cancellation withdraws approvals and subscriptions
2484                // and fences pending agent declarations;
2485                // successful cancellation has no ordinary reply.
2486                // A config Apply may also own an OS prompt. Cancel that prompt
2487                // before config cancellation can report the request handled.
2488                if self
2489                    .agent_declarations
2490                    .cancel(ctx.connection_id, frame.header.corr)
2491                    || self
2492                        .forwarding
2493                        .operator_confirms()
2494                        .cancel(ctx.connection_id, frame.header.corr)
2495                    || self.cancel_config_approval(ctx.connection_id, frame.header.corr)
2496                    || self
2497                        .supervisor
2498                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2499                    || self.cancel_activity_subscription(ctx, &frame)?
2500                    || self.cancel_catalog_subscription(ctx.connection_id, frame.header.corr)
2501                {
2502                    Ok(Vec::new())
2503                } else {
2504                    Ok(vec![control_error_frame(
2505                        &frame,
2506                        "unknown_subscription",
2507                        "no subscription has this correlation id",
2508                    )?])
2509                }
2510            }
2511            FrameType::Request => {
2512                // This additive operation is not part of the existing exhaustive
2513                // module-control enum. Probe the op before decoding that enum.
2514                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2515                let claim_present = op.as_ref().is_some_and(|probe| probe.op == "claim.present");
2516                if let Some(refusal) = self
2517                    .recovery_request_refusal(
2518                        ctx,
2519                        &frame,
2520                        op.as_ref().map(|probe| probe.op.as_str()),
2521                    )
2522                    .await?
2523                {
2524                    if claim_present {
2525                        self.reclaim_presented_expiry(&frame);
2526                        return self
2527                            .refuse_claim_connection(ctx, &frame, "config_gated")
2528                            .await;
2529                    }
2530                    return Ok(vec![refusal]);
2531                }
2532                if claim_present {
2533                    return self.handle_claim_present(ctx, &frame, first).await;
2534                }
2535                if op.as_ref().is_some_and(|probe| probe.op == "claim.mint") {
2536                    return self.handle_claim_mint(ctx, &frame).await;
2537                }
2538                if let Some(refusal) = self.registration_operation_refusal(
2539                    ctx,
2540                    &frame,
2541                    op.as_ref().map(|probe| probe.op.as_str()),
2542                )? {
2543                    return Ok(vec![refusal]);
2544                }
2545                if op
2546                    .as_ref()
2547                    .is_some_and(|probe| probe.op == ops::CATALOG_WATCH)
2548                {
2549                    return self.handle_catalog_watch(ctx, frame);
2550                }
2551                if op
2552                    .as_ref()
2553                    .is_some_and(|probe| probe.op == MODULE_TO_SUBC_OP_CATALOG_UPDATE)
2554                    && subc_protocol::provider_declarations::catalog_update_has_provider_declarations(&frame.body)
2555                        .unwrap_or(false)
2556                {
2557                    return Ok(vec![control_error_frame(
2558                        &frame,
2559                        error_codes::CATALOG_UPDATE_FROZEN_DECLARATIONS,
2560                        "provider_declarations are fixed at HELLO; restart the module to change them",
2561                    )?]);
2562                }
2563                if let Some(probe) = op
2564                    .as_ref()
2565                    .filter(|probe| config_gate::is_module_config_op(&probe.op))
2566                {
2567                    return self
2568                        .handle_module_config_request(ctx, frame, &probe.op)
2569                        .await;
2570                }
2571                if op
2572                    .as_ref()
2573                    .is_some_and(|probe| config_gate::is_config_op(&probe.op))
2574                {
2575                    return self.handle_config_gate_request(ctx, frame).await;
2576                }
2577                if op
2578                    .as_ref()
2579                    .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2580                {
2581                    return self.handle_activity_record(ctx, frame).await;
2582                }
2583                if op.as_ref().is_some_and(|probe| {
2584                    probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2585                }) {
2586                    return self.handle_activity_read(ctx, frame).await;
2587                }
2588                if op
2589                    .as_ref()
2590                    .is_some_and(|probe| probe.op == "operator.confirm")
2591                {
2592                    return self.handle_operator_confirm(ctx, frame);
2593                }
2594                if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2595                    return self
2596                        .observe_control_wait(
2597                            ctx,
2598                            None,
2599                            ops::APP_OPEN,
2600                            self.handle_app_open(ctx, frame),
2601                        )
2602                        .await;
2603                }
2604                if op
2605                    .as_ref()
2606                    .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2607                {
2608                    return self.handle_connection_describe(ctx, frame);
2609                }
2610                if op
2611                    .as_ref()
2612                    .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2613                {
2614                    return self.handle_scope_served_by(ctx.connection_id, frame);
2615                }
2616                if self
2617                    .forwarding
2618                    .module_endpoint_for_connection(ctx.connection_id)
2619                    .map_err(RouterError::Forwarding)?
2620                    .is_some()
2621                {
2622                    if !is_known_module_request_op(&frame.body) {
2623                        return Ok(vec![control_error_frame(
2624                            &frame,
2625                            "unsupported_control_frame",
2626                            "module-originated channel-0 REQUEST is not supported",
2627                        )?]);
2628                    }
2629                    let request = match parse_module_control_request_from_module(&frame.body) {
2630                        Ok(request) => request,
2631                        Err((err, ControlRequestBodyError::UnknownOp)) => {
2632                            return Ok(vec![control_error_frame(
2633                                &frame,
2634                                "unsupported_control_frame",
2635                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
2636                            )?])
2637                        }
2638                        Err((err, ControlRequestBodyError::InvalidBody)) => {
2639                            return Ok(vec![control_error_frame(
2640                                &frame,
2641                                "invalid_control_body",
2642                                format!("malformed module control body: {err}"),
2643                            )?])
2644                        }
2645                    };
2646                    let op = module_control_request_op(&request);
2647                    let corr = frame.header.corr;
2648                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
2649                    if let ModuleControlRequestFromModule::AgentCommands {
2650                        started,
2651                        finished,
2652                        known,
2653                    } = request
2654                    {
2655                        return self
2656                            .agent_declarations
2657                            .enqueue(
2658                                self,
2659                                ctx,
2660                                frame,
2661                                crate::resource_totals::Declaration {
2662                                    started,
2663                                    finished,
2664                                    known,
2665                                },
2666                            )
2667                            .await;
2668                    }
2669                    let result =
2670                        self.handle_module_control_request(ctx.connection_id, frame, request);
2671                    log_slow_control_dispatch(
2672                        dispatch_started_at,
2673                        op,
2674                        ctx.connection_id,
2675                        corr,
2676                        None,
2677                    );
2678                    return result;
2679                }
2680
2681                if is_known_module_request_op(&frame.body) {
2682                    let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2683                        .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2684                    let code = if is_agents {
2685                        "not_supervised"
2686                    } else if op
2687                        .as_ref()
2688                        .is_some_and(|probe| probe.op == subc_protocol::scope::ROUTE_AUTHORIZE_OP)
2689                    {
2690                        error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED
2691                    } else {
2692                        "not_registered"
2693                    };
2694                    return Ok(vec![control_error_frame(
2695                        &frame,
2696                        code,
2697                        "module-originated control requests require the module's own registered connection",
2698                    )?]);
2699                }
2700
2701                let request = match parse_client_control_request(&frame.body) {
2702                    Ok(request) => request,
2703                    Err((err, ControlRequestBodyError::UnknownOp)) => {
2704                        return Ok(vec![control_error_frame(
2705                            &frame,
2706                            "unknown_control_op",
2707                            format!("unknown client control op: {err}"),
2708                        )?])
2709                    }
2710                    Err((err, ControlRequestBodyError::InvalidBody)) => {
2711                        return Ok(vec![control_error_frame(
2712                            &frame,
2713                            "invalid_control_body",
2714                            format!("malformed client control body: {err}"),
2715                        )?])
2716                    }
2717                };
2718                let op = client_control_request_op(&request);
2719                let corr = frame.header.corr;
2720                log_control_dispatch_arrival(op, ctx.connection_id, corr);
2721                let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2722                #[cfg(test)]
2723                if let Some(delay) = self.control_dispatch_delay {
2724                    tokio::time::sleep(delay).await;
2725                }
2726                let result = self
2727                    .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2728                    .await;
2729                if let Some(timings) = route_open_timings.as_mut() {
2730                    timings.finish();
2731                }
2732                log_slow_control_dispatch(
2733                    dispatch_started_at,
2734                    op,
2735                    ctx.connection_id,
2736                    corr,
2737                    route_open_timings.as_ref(),
2738                );
2739                result
2740            }
2741            FrameType::Push => {
2742                let Some(endpoint) = self
2743                    .forwarding
2744                    .module_endpoint_for_connection(ctx.connection_id)
2745                    .map_err(RouterError::Forwarding)?
2746                else {
2747                    return Ok(vec![control_error_frame(
2748                        &frame,
2749                        "unsupported_control_frame",
2750                        "client-originated channel-0 PUSH is not supported",
2751                    )?]);
2752                };
2753                self.handle_status_update(endpoint, frame)
2754            }
2755            FrameType::Response | FrameType::Error
2756                if self
2757                    .forwarding
2758                    .module_endpoint_for_connection(ctx.connection_id)
2759                    .map_err(RouterError::Forwarding)?
2760                    .is_some() =>
2761            {
2762                self.handle_module_relay_response(ctx.connection_id, frame)
2763            }
2764            ty => Ok(vec![control_error_frame(
2765                &frame,
2766                "unsupported_control_frame",
2767                format!("unsupported channel-0 frame {ty:?}"),
2768            )?]),
2769        }
2770    }
2771
2772    #[cfg(test)]
2773    fn pause_cleanup_for_test(&self, stage: u8) {
2774        let mut slot = self.cleanup_pause.lock().unwrap_or_else(|p| p.into_inner());
2775        if slot.as_ref().is_some_and(|(at, _)| *at == stage) {
2776            let (_, (reached, release)) = slot.take().unwrap();
2777            drop(slot);
2778            let _ = reached.send(());
2779            let _ = release.recv_timeout(Duration::from_secs(5));
2780        }
2781    }
2782
2783    /// Run `hook` once, in the next `route.open` that attests a launch nonce,
2784    /// right after the nonce is resolved.
2785    #[cfg(all(test, unix, feature = "test-support"))]
2786    pub(crate) fn after_nonce_attestation_for_test(&self, hook: impl FnOnce() + Send + 'static) {
2787        *self
2788            .after_nonce_attestation
2789            .lock()
2790            .unwrap_or_else(|p| p.into_inner()) = Some(Box::new(hook));
2791    }
2792
2793    #[cfg(test)]
2794    pub(super) fn pause_after_nonce_attestation_for_test(&self) {
2795        let hook = self
2796            .after_nonce_attestation
2797            .lock()
2798            .unwrap_or_else(|p| p.into_inner())
2799            .take();
2800        if let Some(hook) = hook {
2801            hook();
2802        }
2803    }
2804
2805    #[cfg(test)]
2806    async fn pause_hello_ingress_for_test(&self) {
2807        let pause = self
2808            .hello_ingress_pause
2809            .lock()
2810            .unwrap_or_else(|p| p.into_inner())
2811            .take();
2812        if let Some((reached, release)) = pause {
2813            let _ = reached.send(());
2814            let _ = release.await;
2815        }
2816    }
2817
2818    pub fn cleanup_connection(
2819        &self,
2820        connection_id: ConnectionId,
2821    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2822        // The server loop joins its offloaded request tasks (route opens,
2823        // privileged requests and activity reads) before this cleanup runs, and
2824        // inline dispatch has ended. Two kinds of work are not joined. Agent
2825        // command declarations run on their queue's detached worker, so the
2826        // queue is closed first: every declaration still queued or validating
2827        // has its commit fence cancelled before anything below can change how
2828        // the connection resolves. A HELLO whose reader was dropped keeps
2829        // running on its blocking thread. It carries the channel spawn its
2830        // reader resolved at ingress instead of re-reading the index forgotten
2831        // below, so it is still judged as that channel's HELLO: the retired
2832        // spawn refuses it before it records anything, and a channel HELLO
2833        // never records a launch nonce, so it cannot leave this connection any
2834        // nonce authority.
2835        self.agent_declarations.close(connection_id);
2836        self.close_catalog_subscriptions(connection_id);
2837        #[cfg(test)]
2838        self.pause_cleanup_for_test(0);
2839        if let Some(spawn) = self.supervisor.registration_spawn(connection_id) {
2840            spawn.retire_registration_channel();
2841        }
2842        self.supervisor
2843            .forget_registration_connection(connection_id);
2844        #[cfg(test)]
2845        self.pause_cleanup_for_test(1);
2846        self.local_connections
2847            .lock()
2848            .unwrap_or_else(|p| p.into_inner())
2849            .remove(&connection_id);
2850        self.caller_endpoints
2851            .lock()
2852            .unwrap_or_else(|p| p.into_inner())
2853            .remove(&connection_id);
2854        self.build_class_warned
2855            .lock()
2856            .unwrap_or_else(|p| p.into_inner())
2857            .remove(&connection_id);
2858        self.observed_peers
2859            .lock()
2860            .unwrap_or_else(|p| p.into_inner())
2861            .remove(&connection_id);
2862        self.captured_peers
2863            .lock()
2864            .unwrap_or_else(|p| p.into_inner())
2865            .remove(&connection_id);
2866        self.config_desktop_stamps
2867            .lock()
2868            .unwrap_or_else(|p| p.into_inner())
2869            .remove(&connection_id);
2870        self.operator_apps
2871            .lock()
2872            .unwrap_or_else(|p| p.into_inner())
2873            .remove(&connection_id);
2874        self.forwarding
2875            .operator_confirms()
2876            .connection_closed(connection_id);
2877        let cleanup = self.cleanup_connection_with_end_reason(
2878            connection_id,
2879            RegistrationEndReason::ConnectionClosed,
2880        );
2881        // A draining writer can still be reached by a route until forwarding
2882        // cleanup finishes. Keep its claim sink visible to launch retirement.
2883        self.claims.connection_closed(connection_id);
2884        cleanup
2885    }
2886
2887    fn cleanup_connection_with_end_reason(
2888        &self,
2889        connection_id: ConnectionId,
2890        requested_reason: RegistrationEndReason,
2891    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2892        #[cfg(test)]
2893        if let Some(pause) = self
2894            .activity_request_pause
2895            .lock()
2896            .unwrap()
2897            .as_ref()
2898            .filter(|pause| pause.connection == connection_id)
2899        {
2900            pause
2901                .cleanup_started
2902                .store(true, std::sync::atomic::Ordering::Release);
2903        }
2904        if let Some(runtime) = &self.config_runtime {
2905            runtime.disconnected(connection_id);
2906        }
2907        self.agent_declarations.close(connection_id);
2908        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2909            self.registry
2910                .get_module_by_connection(connection_id)?
2911                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
2912                .and_then(|module| module.registration_end_reason().ok().flatten())
2913                .unwrap_or(requested_reason)
2914        } else {
2915            requested_reason
2916        };
2917        let crash_closed = self
2918            .registry
2919            .get_module_by_connection(connection_id)?
2920            .and_then(|registration| {
2921                self.forwarding
2922                    .module_endpoint_for_connection(connection_id)
2923                    .ok()
2924                    .flatten()
2925                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2926                    .map(|routes| (registration.manifest.module_id, routes))
2927            });
2928        let crash_closed = crash_closed.map(|(module_id, routes)| {
2929            let terminal = match self.supervisor.get(&module_id) {
2930                None => false,
2931                Some(module) => match module.will_recover_after_connection_loss() {
2932                    Ok(will_recover) => !will_recover,
2933                    Err(err) => {
2934                        warn!(
2935                            %module_id,
2936                            error = %err,
2937                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
2938                        );
2939                        false
2940                    }
2941                },
2942            };
2943            // The forwarding table gates all providers at the start of daemon
2944            // shutdown, before their connections are closed. An ordinary
2945            // module disconnect still reports crash if that gate is not set.
2946            let (reason, drained) = match self
2947                .forwarding
2948                .daemon_drain_result_for_connection(connection_id)
2949            {
2950                Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2951                Ok(None) => (RouteCloseReason::Crash, false),
2952                Err(err) => {
2953                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2954                    (RouteCloseReason::Crash, false)
2955                }
2956            };
2957            (module_id, routes, reason, drained, terminal)
2958        });
2959        let registrations = self.deregister_connection(connection_id, end_reason);
2960        // Read before teardown: the spawn whose registration this connection
2961        // was. Its registration ending is an authority change for it.
2962        let ended_authority = self
2963            .forwarding
2964            .target_authority_for_connection(connection_id)
2965            .ok()
2966            .flatten();
2967        let cleanup = if crash_closed.is_some() {
2968            self.forwarding.cleanup_connection_counted(connection_id)
2969        } else {
2970            // A module connection's teardown needs the count of abandoned
2971            // route.bind relays for its route.closed notice below. Any other
2972            // connection, such as a client's, sends no such notice and needs
2973            // only its routes released, so it uses the route-only wrapper and
2974            // reports zero.
2975            self.forwarding
2976                .cleanup_connection(connection_id)
2977                .map(|released| crate::forwarding::ConnectionCleanup {
2978                    released,
2979                    abandoned_relays: 0,
2980                })
2981        };
2982        // The route.closed push waits for forwarding teardown because only
2983        // teardown knows how many pending route.bind relays it aborted. It still
2984        // goes out before the GOODBYEs for the released routes, and its targets
2985        // were captured above, before teardown removed those routes.
2986        if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2987            let abandoned = cleanup
2988                .as_ref()
2989                .map_or(0, |cleanup| cleanup.abandoned_relays);
2990            send_route_control_pushes(
2991                &self.forwarding,
2992                routes,
2993                ClientControlPush::RouteClosed {
2994                    module_id,
2995                    channels: Vec::new(),
2996                    reason,
2997                    drained,
2998                    abandoned,
2999                    excluded_subscriptions: 0,
3000                    terminal: Some(terminal),
3001                },
3002            );
3003        }
3004        if let Ok(cleanup) = cleanup {
3005            self.emit_route_goodbyes(cleanup.released);
3006        }
3007        if let Some(record) = ended_authority {
3008            self.change_spawn_authority(&record);
3009        }
3010        // Signal the registration-release watch only now that BOTH registry and
3011        // forwarding teardown are done, so a supervisor waiting to spawn a
3012        // replacement never observes release while old routes still exist.
3013        if matches!(&registrations, Ok(r) if !r.is_empty()) {
3014            crate::supervise::notify_registration_release();
3015            self.capability_evaluator.wake_deadline_loop();
3016            self.refresh_capability_requirements();
3017        }
3018        self.supervisor.remove_spawn_subscribers(connection_id);
3019        self.activity_streams
3020            .lock()
3021            .unwrap_or_else(|p| p.into_inner())
3022            .retain(|(id, _), task| {
3023                if *id == connection_id {
3024                    task.abort();
3025                    false
3026                } else {
3027                    true
3028                }
3029            });
3030        // Sync authority dies with its connection, so the owner's next
3031        // connection can take it; the owner's scopes stay as they are.
3032        self.hello_launch_nonces
3033            .lock()
3034            .unwrap_or_else(|poisoned| poisoned.into_inner())
3035            .forget(connection_id);
3036        self.scopes
3037            .write()
3038            .unwrap_or_else(|poisoned| poisoned.into_inner())
3039            .release_connection(connection_id);
3040        registrations
3041    }
3042
3043    pub(crate) fn handle_route_goodbye(
3044        &self,
3045        connection_id: ConnectionId,
3046        route_channel: u16,
3047        route_epoch: u32,
3048    ) -> Result<bool, RouterError> {
3049        debug!(
3050            connection_id = connection_id.get(),
3051            route_channel, route_epoch, "handling route GOODBYE"
3052        );
3053        let RouteRelease::Removed(released_route) = self
3054            .forwarding
3055            .release_client_route(connection_id, route_channel, route_epoch)
3056            .map_err(RouterError::Forwarding)?
3057        else {
3058            return Ok(false);
3059        };
3060        self.emit_route_goodbyes(vec![released_route]);
3061        Ok(true)
3062    }
3063
3064    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
3065        scope_expiry::ScopeExpiryContext::new(self).emit_route_goodbyes(released_routes);
3066    }
3067
3068    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
3069    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
3070    /// own commit failed after the module had already accepted). Without this, a
3071    /// module that accepts late keeps a binding subc has torn down, so a later
3072    /// frame on that module channel could misdeliver if the channel is reused.
3073    ///
3074    /// Never closes the shared module connection on failure: a dropped notification
3075    /// only wastes a bounded amount of warm module-side state, which the module's
3076    /// own idle reaper reclaims. Only call this once the route.bind relay was
3077    /// actually enqueued to the module — if the relay send itself failed, the
3078    /// module never created a binding and there is nothing to tear down.
3079    fn send_abandoned_route_bind_goodbye(
3080        &self,
3081        module_sink: &crate::FrameSink,
3082        negotiated_ver: u8,
3083        module_channel: u16,
3084        module_epoch: u32,
3085    ) {
3086        let frame = match Frame::build_with_version(
3087            negotiated_ver,
3088            FrameType::Goodbye,
3089            control_flags(),
3090            module_channel,
3091            module_epoch,
3092            0,
3093            Vec::new(),
3094        ) {
3095            Ok(frame) => frame,
3096            Err(err) => {
3097                warn!(
3098                    route_channel = module_channel,
3099                    error = %err,
3100                    "failed to build GOODBYE for abandoned route.bind"
3101                );
3102                return;
3103            }
3104        };
3105        crate::forwarding::send_module_route_goodbye(
3106            &self.counters,
3107            module_sink,
3108            frame,
3109            None,
3110            "abandoned route.bind",
3111        );
3112    }
3113
3114    #[cfg(test)]
3115    fn handle_hello(
3116        &self,
3117        connection_id: ConnectionId,
3118        egress: Option<crate::router::FrameSink>,
3119        frame: Frame,
3120    ) -> Result<Vec<Frame>, RouterError> {
3121        let binding = self.claims.binding(connection_id);
3122        let channel_spawn = self.supervisor.registration_spawn(connection_id);
3123        self.handle_hello_checked(connection_id, egress, frame, false, binding, channel_spawn)
3124    }
3125
3126    fn handle_hello_checked(
3127        &self,
3128        connection_id: ConnectionId,
3129        sink: Option<crate::FrameSink>,
3130        frame: Frame,
3131        wait_for_gate: bool,
3132        prefix_binding: Option<claims::ClaimBinding>,
3133        channel_spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3134    ) -> Result<Vec<Frame>, RouterError> {
3135        let primary_sink = channel_spawn.as_ref().and_then(|_| sink.clone());
3136        #[cfg(test)]
3137        if prefix_binding.is_some() {
3138            self.claims.pause_prefix_hello_for_test(5);
3139            match self.claims.take_prefix_hello_failure_for_test() {
3140                Some("error") => {
3141                    return Err(RouterError::backend(
3142                        0,
3143                        frame.header.corr,
3144                        "injected HELLO failure",
3145                    ))
3146                }
3147                Some("panic") => panic!("injected HELLO task panic"),
3148                _ => {}
3149            }
3150        }
3151        if let Some(binding) = &prefix_binding {
3152            let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3153            let module = value
3154                .as_ref()
3155                .and_then(|value| value["manifest"]["module_id"].as_str());
3156            if !binding.admit_prefix_hello(module)
3157                || !self.prefix_binding_live(binding, connection_id)
3158                || !module.is_some_and(|module| {
3159                    self.supervisor
3160                        .prefix_claim_authorized(&binding.registration.module_id, module)
3161                })
3162                || binding.registration.check().is_err()
3163            {
3164                return Ok(vec![control_error_frame(
3165                    &frame,
3166                    "claim_prefix_refused",
3167                    "HELLO must name the prefix claim's exact module id once",
3168                )?]);
3169            }
3170        }
3171        struct RefuseUnregisteredChannel {
3172            spawn: Option<Arc<crate::supervise::RegistrationSpawn>>,
3173            sink: Option<crate::FrameSink>,
3174        }
3175        impl Drop for RefuseUnregisteredChannel {
3176            fn drop(&mut self) {
3177                if let Some(spawn) = &self.spawn {
3178                    if !spawn.registered() {
3179                        if let Some(sink) = &self.sink {
3180                            sink.terminate_ingress();
3181                        }
3182                        spawn.retire_registration_channel();
3183                    }
3184                }
3185            }
3186        }
3187        let _channel_attempt = RefuseUnregisteredChannel {
3188            spawn: channel_spawn.clone(),
3189            sink: sink.clone(),
3190        };
3191        if let Some(spawn) = &channel_spawn {
3192            let value = serde_json::from_slice::<serde_json::Value>(&frame.body).ok();
3193            if let Err(code) = spawn.admit_hello(
3194                value
3195                    .as_ref()
3196                    .and_then(|value| value["manifest"]["module_id"].as_str()),
3197            ) {
3198                return Ok(vec![control_error_frame(
3199                    &frame,
3200                    code,
3201                    "registration channel HELLO refused",
3202                )?]);
3203            }
3204        }
3205        debug!(
3206            connection_id = connection_id.get(),
3207            corr = frame.header.corr,
3208            "handling HELLO"
3209        );
3210        // A module connection has one identity for its entire lifetime. A second
3211        // registration would leave the old registry owner behind while replacing
3212        // its forwarding endpoint and launch nonce.
3213        if self
3214            .registry
3215            .get_module_by_connection(connection_id)
3216            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3217            .is_some()
3218        {
3219            return Ok(vec![control_error_frame(
3220                &frame,
3221                "invalid_hello",
3222                "connection is already registered as a module",
3223            )?]);
3224        }
3225        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
3226            Ok(value) => value,
3227            Err(err) => {
3228                return Ok(vec![control_error_frame(
3229                    &frame,
3230                    "invalid_hello",
3231                    format!("malformed HELLO body: {err}"),
3232                )?])
3233            }
3234        };
3235        if let Err(detail) = catalog::validate_hello_declarations(&hello_value) {
3236            return Ok(vec![control_error_frame(
3237                &frame,
3238                error_codes::INVALID_PROVIDER_DECLARATIONS,
3239                detail,
3240            )?]);
3241        }
3242        if let Some(module) = hello_value
3243            .get("manifest")
3244            .and_then(|value| value.get("module_id"))
3245            .and_then(|value| value.as_str())
3246        {
3247            let recovering = self
3248                .config_runtime
3249                .as_ref()
3250                .is_some_and(|owner| owner.check_recovery().unwrap_or(true));
3251            // Defence in depth: RuntimeOwner::module_is_gated in
3252            // config_gate/admission.rs already gates every id during recovery.
3253            // A registration channel still never uses the nonce fallback.
3254            if recovering
3255                && channel_spawn.as_ref().is_none_or(|spawn| {
3256                    !self.supervisor.has_live_verified_channel_binding(spawn)
3257                        || spawn.check().is_err()
3258                })
3259                && !(channel_spawn.is_none()
3260                    && hello_value
3261                        .get("launch_nonce")
3262                        .and_then(|value| value.as_str())
3263                        .is_some_and(|nonce| {
3264                            self.supervisor.spawned_consumer_authorized(module, nonce)
3265                                && self
3266                                    .supervisor
3267                                    .has_live_verified_program_binding(module, nonce)
3268                        }))
3269            {
3270                return Ok(vec![control_error_frame(
3271                    &frame,
3272                    "config_gated",
3273                    "recovery admits only previously bound supervised launches",
3274                )?]);
3275            }
3276            if self
3277                .config_runtime
3278                .as_ref()
3279                .is_some_and(|owner| owner.module_is_gated(module))
3280                && (channel_spawn
3281                    .as_ref()
3282                    .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3283                    || self.config_runtime.as_ref().is_some_and(|owner| {
3284                        !recovering && owner.check_module_hold(module).is_err()
3285                    }))
3286            {
3287                return Ok(vec![control_error_frame(
3288                    &frame,
3289                    "reserved_module",
3290                    "a gated identity belongs only to its supervised child",
3291                )?]);
3292            }
3293        }
3294        if channel_spawn.is_none() {
3295            if let (Some(module), Some(nonce)) = (
3296                hello_value
3297                    .get("manifest")
3298                    .and_then(|value| value.get("module_id"))
3299                    .and_then(|value| value.as_str()),
3300                hello_value
3301                    .get("launch_nonce")
3302                    .and_then(|value| value.as_str()),
3303            ) {
3304                let binding = self.supervisor.has_program_binding_record(module, nonce);
3305                let required = self
3306                    .config_runtime
3307                    .as_ref()
3308                    .is_some_and(|owner| owner.module_is_gated(module));
3309                if let Some(error) = (binding || required)
3310                    .then(|| {
3311                        self.supervisor
3312                            .check_current_program_binding(module, nonce)
3313                            .err()
3314                    })
3315                    .flatten()
3316                {
3317                    let code = if error.code == "program_refused" {
3318                        "program_refused"
3319                    } else {
3320                        "config_delivery_deferred"
3321                    };
3322                    self.request_refusal_close(
3323                        connection_id,
3324                        sink.as_ref(),
3325                        CloseReason::new(code, error.detail.clone()),
3326                    );
3327                    return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3328                }
3329            }
3330        }
3331        if let Err(err) = validate_hello_display_name(&hello_value) {
3332            return Ok(vec![control_error_frame(
3333                &frame,
3334                "invalid_manifest",
3335                err.to_string(),
3336            )?]);
3337        }
3338        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
3339            return Ok(vec![control_error_frame(
3340                &frame,
3341                "invalid_capability_grammar",
3342                err.to_string(),
3343            )?]);
3344        }
3345        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
3346            return Ok(vec![control_error_frame(
3347                &frame,
3348                "invalid_manifest",
3349                err.to_string(),
3350            )?]);
3351        }
3352        if let Err(err) = validate_hello_event_declarations(&hello_value) {
3353            return Ok(vec![control_error_frame(
3354                &frame,
3355                "invalid_event_declaration",
3356                err.to_string(),
3357            )?]);
3358        }
3359        if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
3360            return Ok(vec![control_error_frame(
3361                &frame,
3362                "invalid_manifest",
3363                err.to_string(),
3364            )?]);
3365        }
3366        if let Some(provenance) = hello_value
3367            .get("manifest")
3368            .and_then(|manifest| manifest.get("provenance"))
3369        {
3370            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
3371                return Ok(vec![control_error_frame(
3372                    &frame,
3373                    "invalid_manifest",
3374                    format!("malformed manifest provenance: {err}"),
3375                )?]);
3376            }
3377        }
3378        if let Err(reason) = validate_hello_role_declarations(&hello_value) {
3379            let module_id = hello_value["manifest"]["module_id"]
3380                .as_str()
3381                .unwrap_or("<unknown>");
3382            warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
3383        }
3384        if let Err(reason) = validate_hello_operation_deferral(&hello_value) {
3385            let module_id = hello_value
3386                .get("manifest")
3387                .and_then(|m| m.get("module_id"))
3388                .and_then(serde_json::Value::as_str)
3389                .unwrap_or("unknown");
3390            warn!(module_id, %reason, "dropping operation_deferral from HELLO");
3391        }
3392        if let Err(reason) = validate_hello_flow_operations(&hello_value) {
3393            let module_id = hello_value["manifest"]["module_id"]
3394                .as_str()
3395                .unwrap_or("<unknown>");
3396            warn!(module_id, %reason, "dropping flow_operations from HELLO");
3397        }
3398        if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
3399            let module_id = hello_value["manifest"]["module_id"]
3400                .as_str()
3401                .unwrap_or("<unknown>");
3402            warn!(module_id, %reason, "dropping confirm_operations from HELLO");
3403        }
3404        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
3405            Ok(hello) => hello,
3406            Err(err) => {
3407                return Ok(vec![control_error_frame(
3408                    &frame,
3409                    "invalid_hello",
3410                    format!("malformed HELLO body: {err}"),
3411                )?])
3412            }
3413        };
3414
3415        if hello.protocol_ver != hello.manifest.protocol_ver {
3416            return Ok(vec![control_error_frame(
3417                &frame,
3418                "invalid_manifest",
3419                format!(
3420                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
3421                    hello.protocol_ver, hello.manifest.protocol_ver
3422                ),
3423            )?]);
3424        }
3425
3426        if hello.manifest.module_id.trim().is_empty() {
3427            return Ok(vec![control_error_frame(
3428                &frame,
3429                "invalid_manifest",
3430                "manifest module_id must not be empty",
3431            )?]);
3432        }
3433
3434        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
3435            Ok(negotiated_ver) => negotiated_ver,
3436            Err(message) => {
3437                return Ok(vec![control_error_frame(
3438                    &frame,
3439                    "version_unsupported",
3440                    message,
3441                )?])
3442            }
3443        };
3444
3445        // Registration and forwarding publication use the same module gate as
3446        // stamp, unstamp and spawning. Recheck after taking it; parsing may
3447        // precede those transitions. A program-refusal hold blocks future
3448        // spawns, but does not take this gate or revoke a HELLO whose supervised
3449        // child's image has already passed the check.
3450        let _publication_gate = if let Some(owner) = &self.config_runtime {
3451            let gate = owner
3452                .module_gate(&hello.manifest.module_id)
3453                .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3454            Some(if wait_for_gate {
3455                gate.blocking_lock_owned()
3456            } else {
3457                match gate.try_lock_owned() {
3458                    Ok(guard) => guard,
3459                    Err(_) => {
3460                        return Ok(vec![control_error_frame(
3461                            &frame,
3462                            "config_delivery_deferred",
3463                            "module admission is busy",
3464                        )?])
3465                    }
3466                }
3467            })
3468        } else {
3469            None
3470        };
3471        #[cfg(test)]
3472        if let Some(hold) = &self.hello_before_publication_hold {
3473            hold.wait();
3474            hold.wait();
3475        }
3476        let module = &hello.manifest.module_id;
3477        let required = self
3478            .config_runtime
3479            .as_ref()
3480            .is_some_and(|owner| owner.module_is_gated(module));
3481        let recovering = self
3482            .config_runtime
3483            .as_ref()
3484            .is_some_and(|owner| owner.is_recovering());
3485        // Both entry paths (handle_control -> handle_hello and
3486        // handle_control_frame_timed -> spawn_blocking) enter this same
3487        // handle_hello_checked function. Its fresh, fail-closed store check
3488        // precedes parsing and this publication-gate check; this cached read
3489        // also observes any recovery transition published during that work.
3490        // The nonce-connection-only fallback is defence in depth alongside
3491        // RuntimeOwner::module_is_gated's all-id recovery gate.
3492        if recovering
3493            && channel_spawn.as_ref().is_none_or(|spawn| {
3494                !self.supervisor.has_live_verified_channel_binding(spawn) || spawn.check().is_err()
3495            })
3496            && !(channel_spawn.is_none()
3497                && hello.launch_nonce.as_deref().is_some_and(|nonce| {
3498                    self.supervisor.spawned_consumer_authorized(module, nonce)
3499                        && self
3500                            .supervisor
3501                            .has_live_verified_program_binding(module, nonce)
3502                }))
3503        {
3504            return Ok(vec![control_error_frame(
3505                &frame,
3506                "config_gated",
3507                "recovery admits only previously bound supervised launches",
3508            )?]);
3509        }
3510        if required
3511            && (channel_spawn
3512                .as_ref()
3513                .is_none_or(|spawn| !self.supervisor.has_live_verified_channel_binding(spawn))
3514                || self
3515                    .config_runtime
3516                    .as_ref()
3517                    .is_some_and(|owner| !recovering && owner.check_module_hold(module).is_err()))
3518        {
3519            return Ok(vec![control_error_frame(
3520                &frame,
3521                "reserved_module",
3522                "gated admission requires the current supervised child's registration channel and verified image binding",
3523            )?]);
3524        }
3525        if let Some(spawn) = &channel_spawn {
3526            if let Err(code) = spawn.check() {
3527                return Ok(vec![control_error_frame(
3528                    &frame,
3529                    code,
3530                    "registration launch is no longer live",
3531                )?]);
3532            }
3533        } else if let Some(nonce) = hello.launch_nonce.as_deref() {
3534            if required || self.supervisor.has_program_binding_record(module, nonce) {
3535                if let Err(error) = self.supervisor.check_current_program_binding(module, nonce) {
3536                    let code = if error.code == "program_refused" {
3537                        "program_refused"
3538                    } else {
3539                        "config_delivery_deferred"
3540                    };
3541                    self.request_refusal_close(
3542                        connection_id,
3543                        sink.as_ref(),
3544                        CloseReason::new(code, error.detail.clone()),
3545                    );
3546                    return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
3547                }
3548            }
3549        }
3550
3551        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
3552        // swap is open for this id, the only HELLO admitted as a second process
3553        // is the one carrying the candidate's launch nonce (the swap token), and
3554        // it registers into the candidate slot rather than being refused as a
3555        // duplicate. Run after the reserved gate, a reserved module's candidate
3556        // would be refused `reserved_module` for presenting a nonce that gate
3557        // does not know. See `SupervisorHandle::swap_hello_admission`.
3558        let swap_admission = match &channel_spawn {
3559            Some(spawn) if spawn.is_candidate() => SwapHelloAdmission::Candidate,
3560            Some(_) => SwapHelloAdmission::NotSwapping,
3561            None if prefix_binding.is_some() => SwapHelloAdmission::NotSwapping,
3562            None => self
3563                .supervisor
3564                .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref()),
3565        };
3566        if swap_admission == SwapHelloAdmission::ProtocolNone {
3567            warn!(
3568                module_id = %hello.manifest.module_id,
3569                connection_id = connection_id.get(),
3570                "HELLO refused: the module_id belongs to a protocol:none module, which never registers"
3571            );
3572            // The code stays `swap_token_invalid` for existing callers; only the
3573            // explanation names the actual reason.
3574            return Ok(vec![control_error_frame(
3575                &frame,
3576                "swap_token_invalid",
3577                format!(
3578                    "module_id '{}' is a protocol:none module; a protocol:none module may not register",
3579                    hello.manifest.module_id
3580                ),
3581            )?]);
3582        }
3583        if swap_admission == SwapHelloAdmission::ChannelRequired {
3584            warn!(
3585                module_id = %hello.manifest.module_id,
3586                connection_id = connection_id.get(),
3587                "HELLO refused: the module_id is gated and the HELLO did not arrive over its registration channel"
3588            );
3589            // The code stays `swap_token_invalid` for existing callers; only the
3590            // explanation names the actual reason.
3591            return Ok(vec![control_error_frame(
3592                &frame,
3593                "swap_token_invalid",
3594                format!(
3595                    "module_id '{}' is gated; it must register over its supervised registration channel",
3596                    hello.manifest.module_id
3597                ),
3598            )?]);
3599        }
3600        if swap_admission == SwapHelloAdmission::Refused {
3601            warn!(
3602                module_id = %hello.manifest.module_id,
3603                connection_id = connection_id.get(),
3604                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
3605            );
3606            return Ok(vec![control_error_frame(
3607                &frame,
3608                "swap_token_invalid",
3609                format!(
3610                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
3611                    hello.manifest.module_id
3612                ),
3613            )?]);
3614        }
3615        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
3616
3617        // A swap candidate proves its identity with the candidate nonce above.
3618        // Other registrations receive this catalog fact only when reserved-module
3619        // policy required and accepted a nonce; unreserved registrations require
3620        // no nonce check.
3621        let verification = if swap_candidate && channel_spawn.is_none() {
3622            ReservedHelloVerification {
3623                nonce_verified: true,
3624                prefix_owner: None,
3625            }
3626        } else {
3627            let verification = if let Some(binding) = &prefix_binding {
3628                if !self.supervisor.prefix_claim_authorized(
3629                    &binding.registration.module_id,
3630                    &hello.manifest.module_id,
3631                ) {
3632                    return Ok(vec![control_error_frame(
3633                        &frame,
3634                        "claim_prefix_refused",
3635                        "prefix ownership changed",
3636                    )?]);
3637                }
3638                Ok(ReservedHelloVerification {
3639                    nonce_verified: true,
3640                    prefix_owner: Some(binding.registration.module_id.clone()),
3641                })
3642            } else if channel_spawn.is_some() {
3643                self.supervisor
3644                    .channel_hello_verification(&hello.manifest.module_id)
3645            } else {
3646                self.supervisor.reserved_hello_verification(
3647                    &hello.manifest.module_id,
3648                    hello.launch_nonce.as_deref(),
3649                )
3650            };
3651            match verification {
3652                Ok(verified) => verified,
3653                Err(rejection) => {
3654                    let message = match rejection {
3655                        ReservedHelloRejection::Exact { module_id } => format!(
3656                            "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
3657                        ),
3658                        ReservedHelloRejection::Prefix {
3659                            prefix,
3660                            owner_module_id,
3661                        } => format!(
3662                            "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
3663                            hello.manifest.module_id
3664                        ),
3665                    };
3666                    return Ok(vec![control_error_frame(
3667                        &frame,
3668                        "reserved_module",
3669                        message,
3670                    )?]);
3671                }
3672            }
3673        };
3674
3675        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
3676            &hello.manifest.module_id,
3677            hello.manifest.capabilities.as_ref(),
3678        );
3679        if let Some(refusal) = reserved_capability_refusals.first() {
3680            let capability = refusal.capability.clone();
3681            let bound_module = refusal.claimants[0].clone();
3682            log_duplicate_claim_events(reserved_capability_refusals);
3683            return Ok(vec![control_error_frame(
3684                &frame,
3685                "reserved_capability",
3686                format!(
3687                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3688                    capability, bound_module, hello.manifest.module_id
3689                ),
3690            )?]);
3691        }
3692
3693        // A connection that already opened client routes must not also register as
3694        // a module: cleanup would then release only one side and leak the other.
3695        if self
3696            .forwarding
3697            .connection_has_client_routes(connection_id)
3698            .map_err(RouterError::Forwarding)?
3699        {
3700            return Ok(vec![control_error_frame(
3701                &frame,
3702                "invalid_hello",
3703                "connection has open client routes and cannot also register as a module",
3704            )?]);
3705        }
3706
3707        // Kept for scope sync authority, which goes only to the connection that
3708        // presented the module's current launch nonce. Recorded before the
3709        // registration is attempted: a connection whose registration then fails
3710        // has no registration, so it cannot sync anyway, and cleanup forgets it.
3711        // A registration channel's authority is the channel itself, never a
3712        // nonce it also presented, and a prefix registration's is its claim:
3713        // neither records one, so neither can fall back to nonce authority.
3714        self.hello_launch_nonces
3715            .lock()
3716            .unwrap_or_else(|poisoned| poisoned.into_inner())
3717            .record(
3718                connection_id,
3719                if prefix_binding.is_some() || channel_spawn.is_some() {
3720                    None
3721                } else {
3722                    hello.launch_nonce.as_deref()
3723                },
3724            );
3725        let control_ops = effective_module_control_ops(hello.control_ops);
3726        let prefix_submodule = verification.prefix_owner.is_some();
3727        // Built before anything is registered so an encoding failure leaves no
3728        // registry or forwarding state behind.
3729        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
3730        #[cfg(test)]
3731        struct PublicationFinished(Arc<std::sync::Barrier>);
3732        #[cfg(test)]
3733        impl Drop for PublicationFinished {
3734            fn drop(&mut self) {
3735                self.0.wait();
3736            }
3737        }
3738        #[cfg(test)]
3739        let _publication_finished = self.hello_after_launch_check_hold.as_ref().map(|hold| {
3740            hold.wait();
3741            hold.wait();
3742            PublicationFinished(hold.clone())
3743        });
3744        let publication_spawn = channel_spawn
3745            .as_ref()
3746            .or_else(|| prefix_binding.as_ref().map(|binding| &binding.registration));
3747        #[cfg(test)]
3748        if prefix_binding.is_some() {
3749            self.claims.pause_prefix_hello_for_test(7);
3750        }
3751        let _registration_publication = match publication_spawn {
3752            Some(spawn) => Some(match spawn.begin_publication() {
3753                Ok(guard) => guard,
3754                Err(code) => {
3755                    return Ok(vec![control_error_frame(
3756                        &frame,
3757                        code,
3758                        "registration ended before HELLO publication",
3759                    )?])
3760                }
3761            }),
3762            None => None,
3763        };
3764        let (_prefix_live, _prefix_ownership) = if let Some(binding) = &prefix_binding {
3765            let live = self.claims.lock_prefix_binding(binding, connection_id);
3766            let ownership = self.supervisor.prefix_claim_publication(
3767                &binding.registration.module_id,
3768                &hello.manifest.module_id,
3769            );
3770            if live.is_none()
3771                || ownership.is_none()
3772                || !self.claim_primary_registered(binding.spawn, &binding.registration)
3773                || binding.prefix_module_id() != Some(hello.manifest.module_id.as_str())
3774            {
3775                return Ok(vec![control_error_frame(
3776                    &frame,
3777                    "claim_prefix_refused",
3778                    "prefix authority ended before publication",
3779                )?]);
3780            }
3781            (live, ownership)
3782        } else {
3783            (None, None)
3784        };
3785        if let Some(spawn) = publication_spawn {
3786            self.forwarding
3787                .bind_spawn(connection_id, spawn)
3788                .map_err(RouterError::Forwarding)?;
3789        }
3790        // The spawn whose own registration this HELLO is, for `target` on
3791        // the route.open reply: its channel, or the launch its nonce was
3792        // minted for. A prefix child's HELLO names no spawn of its own. It is
3793        // only looked up here. The connection is bound to it and its
3794        // requested declarations are frozen once the registry has accepted
3795        // this HELLO (`admit_hello_authority`), so a HELLO refused as a
3796        // duplicate, or for any other reason, leaves the spawn untouched: its
3797        // closing connection is then no registration of the spawn and ends
3798        // none of the spawn's routes.
3799        let hello_authority = match (&channel_spawn, &prefix_binding) {
3800            (Some(spawn), _) => Some(spawn.authority.clone()),
3801            (None, None) => hello.launch_nonce.as_deref().and_then(|nonce| {
3802                self.supervisor
3803                    .spawned_consumer_record(&hello.manifest.module_id, nonce)
3804            }),
3805            (None, Some(_)) => None,
3806        }
3807        .map(|record| HelloAuthority {
3808            record,
3809            requested: hello
3810                .manifest
3811                .provider_declarations
3812                .clone()
3813                .unwrap_or_default(),
3814            froze: Default::default(),
3815        });
3816        #[cfg(test)]
3817        if prefix_binding.is_some() {
3818            self.claims.pause_prefix_hello_for_test(8);
3819        }
3820        if swap_candidate {
3821            let module_id = hello.manifest.module_id.clone();
3822            let mut differing_declarations = Default::default();
3823            let reply = self.register_swap_candidate(
3824                connection_id,
3825                sink,
3826                &frame,
3827                hello.manifest,
3828                negotiated_ver,
3829                control_ops,
3830                hello_ack,
3831                hello_authority.as_ref(),
3832                &mut differing_declarations,
3833                #[cfg(windows)]
3834                hello.launch_nonce.as_deref(),
3835            )?;
3836            if let Some(spawn) = &channel_spawn {
3837                if self
3838                    .registry
3839                    .get_module_by_connection(connection_id)
3840                    .ok()
3841                    .flatten()
3842                    .is_some()
3843                {
3844                    if let Some(sink) = primary_sink {
3845                        spawn.bind_primary_sink(sink);
3846                    }
3847                    spawn.hello_accepted();
3848                }
3849            } else if self
3850                .registry
3851                .get_module_by_connection(connection_id)
3852                .ok()
3853                .flatten()
3854                .is_some()
3855            {
3856                self.supervisor
3857                    .retire_unused_registration_channel(&module_id, true);
3858            }
3859            if self
3860                .registry
3861                .get_module_by_connection(connection_id)
3862                .ok()
3863                .flatten()
3864                .is_some()
3865            {
3866                self.file_development_widening(
3867                    hello_authority.map(|authority| authority.record),
3868                    differing_declarations,
3869                );
3870            }
3871            return Ok(reply);
3872        }
3873        // Registered unpublished: the module id is claimed, so a duplicate HELLO
3874        // is refused, but `catalog.list` does not show the module and the
3875        // catalog generation does not move until it is published below, once
3876        // its forwarding endpoint exists. Listing it now would advertise a
3877        // module that `route.open` still refuses for having no forwarding
3878        // connection.
3879        let registration = match self
3880            .registry
3881            .register_unpublished_verified_with_control_ops(
3882                hello.manifest,
3883                negotiated_ver,
3884                connection_id,
3885                control_ops,
3886                verification.nonce_verified,
3887            ) {
3888            Ok(registration) => registration,
3889            Err(RegistryError::DuplicateModuleId { module_id }) => {
3890                return Ok(vec![control_error_frame(
3891                    &frame,
3892                    "duplicate_module_id",
3893                    format!(
3894                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
3895                    ),
3896                )?])
3897            }
3898            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3899                return Ok(vec![control_error_frame(
3900                    &frame,
3901                    "invalid_module_id",
3902                    err.to_string(),
3903                )?])
3904            }
3905            Err(err) => {
3906                return Ok(vec![control_error_frame(
3907                    &frame,
3908                    "registry_error",
3909                    err.to_string(),
3910                )?])
3911            }
3912        };
3913        let differing_declarations =
3914            match self.admit_hello_authority(connection_id, hello_authority.as_ref()) {
3915                Ok(differing) => differing,
3916                Err(err) => {
3917                    if matches!(
3918                        self.rollback_unpublished_hello(connection_id),
3919                        Ok(r) if !r.is_empty()
3920                    ) {
3921                        crate::supervise::notify_registration_release();
3922                    }
3923                    return Err(RouterError::Forwarding(err));
3924                }
3925            };
3926        #[cfg(test)]
3927        self.hold_between_registry_and_forwarding();
3928
3929        let mut reply = Vec::new();
3930        // Registry readers must wait from before the ACK can be read until
3931        // publication. Nest forwarding inside the registry lock, never reverse.
3932        let publication = self
3933            .registry
3934            .publish_registration_after(connection_id, || {
3935                if let Some(sink) = sink {
3936                    // The forwarding table's module store is also the daemon-to-module
3937                    // control-RPC lane, so every HELLO gets a live endpoint even when the
3938                    // manifest has no routable provider role. Non-routable modules still
3939                    // cannot receive route.bind in production: `handle_route_open` checks
3940                    // the registry manifest with `target_has_required_role` before the
3941                    // only production call to `begin_route_bind_relay_for` below that
3942                    // route.open path. The remaining direct relay callers are unit tests
3943                    // and benchmark harnesses that construct forwarding state explicitly.
3944                    //
3945                    // The HELLO_ACK is queued by the forwarding table itself, before the
3946                    // endpoint becomes visible, and is NOT returned as a reply. A module
3947                    // reads HELLO_ACK first and exits on anything else; a reply is only
3948                    // written after this handler returns, by which time a route.open on
3949                    // another connection could already have queued a route.bind request
3950                    // for this module ahead of it.
3951                    let concurrency = manifest_concurrency(&registration.manifest);
3952                    self.forwarding.register_module_connection_acked(
3953                        connection_id,
3954                        registration.manifest.module_id.clone(),
3955                        negotiated_ver,
3956                        concurrency,
3957                        sink,
3958                        crate::forwarding::ModuleHelloAck {
3959                            frame: hello_ack,
3960                            verified_prefix_owner: verification.prefix_owner,
3961                        },
3962                    )?;
3963                } else {
3964                    // No sink means no forwarding endpoint, so nothing can be routed
3965                    // ahead of the ack; it goes out as the reply.
3966                    reply.push(hello_ack);
3967                }
3968                #[cfg(test)]
3969                self.hold_between_ack_and_publication();
3970                Ok(())
3971            })
3972            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3973        let published = match publication {
3974            Ok(published) => published,
3975            Err(err) => {
3976                // The registry lock has been released, so rollback can take it
3977                // again. No forwarding endpoint or route was published. Keep
3978                // forwarding's weak connection-to-spawn association until the
3979                // connection's read loop ends and removes its remaining routes.
3980                // Removing this unpublished entry does not advance the catalog
3981                // generation.
3982                if matches!(
3983                    self.rollback_unpublished_hello(connection_id),
3984                    Ok(r) if !r.is_empty()
3985                ) {
3986                    crate::supervise::notify_registration_release();
3987                }
3988                self.refuse_hello_authority(connection_id, hello_authority.as_ref());
3989                return Ok(vec![control_error_frame(
3990                    &frame,
3991                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
3992                        "invalid_hello"
3993                    } else {
3994                        forwarding_error_code(&err)
3995                    },
3996                    err.to_string(),
3997                )?]);
3998            }
3999        };
4000        if !published {
4001            debug!(
4002                module_id = %registration.manifest.module_id,
4003                connection_id = connection_id.get(),
4004                "registration was removed before it could be published"
4005            );
4006            self.refuse_hello_authority(connection_id, hello_authority.as_ref());
4007            return Ok(reply);
4008        }
4009
4010        drop(_prefix_ownership);
4011        drop(_prefix_live);
4012
4013        // Exposure over assumption: Concurrency's serde default is pinned to the
4014        // pre-field behavior (ModuleManaged), so a management surface that is
4015        // genuinely Serial and just never declared it inherits concurrent
4016        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
4017        // turns "no module has been bitten yet" into the checkable claim "no
4018        // module is exposed" -- one read of the boot log instead of a fleet
4019        // audit. Detected from the raw HELLO bytes because the serde default
4020        // deliberately erases the absent/declared distinction from the type.
4021        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
4022            info!(
4023                module_id = %registration.manifest.module_id,
4024                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
4025            );
4026        }
4027        if registration.manifest.registration_channel == Some(false)
4028            && self
4029                .supervisor
4030                .module_is_gated(&registration.manifest.module_id)
4031            && self
4032                .config_runtime
4033                .as_ref()
4034                .and_then(|owner| owner.registration_declaration(&registration.manifest.module_id))
4035                .as_deref()
4036                == Some("declared")
4037        {
4038            warn!(module_id = %registration.manifest.module_id,
4039                "HELLO registration_channel is false despite the approved channel declaration; registration remains admitted");
4040        }
4041
4042        self.apply_registration_capabilities(&registration);
4043
4044        self.supervisor
4045            .record_activity_registration(&registration.manifest.module_id);
4046
4047        info!(
4048            module_id = %registration.manifest.module_id,
4049            module_version = %registration.manifest.module_version,
4050            negotiated_ver,
4051            routable_provider = manifest_provides_routable_role(&registration.manifest),
4052            connection_id = connection_id.get(),
4053            "module registered"
4054        );
4055
4056        if let Some(spawn) = &channel_spawn {
4057            if let Some(sink) = primary_sink {
4058                spawn.bind_primary_sink(sink);
4059            }
4060            spawn.hello_accepted();
4061        } else if !prefix_submodule {
4062            self.supervisor
4063                .retire_unused_registration_channel(&registration.manifest.module_id, false);
4064        }
4065        self.file_development_widening(
4066            hello_authority.map(|authority| authority.record),
4067            differing_declarations,
4068        );
4069        #[cfg(windows)]
4070        if let Some(nonce) = hello.launch_nonce.as_deref() {
4071            self.supervisor
4072                .program_hello_accepted(&registration.manifest.module_id, nonce);
4073        }
4074
4075        #[cfg(test)]
4076        if let Some(hold) = &self.hello_inside_publication_hold {
4077            hold.wait();
4078            hold.wait();
4079        }
4080
4081        Ok(reply)
4082    }
4083
4084    /// Register a HELLO the swap gate admitted into the candidate slot of the
4085    /// registry and of forwarding, where it is reachable over its own
4086    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
4087    /// nothing routes to it until the supervisor cuts over.
4088    ///
4089    /// Registry first (unpublished), then forwarding, then publication, the
4090    /// same order as an ordinary HELLO; a forwarding failure removes the
4091    /// registry entry again. The swap supervisor treats a published, ready
4092    /// candidate as registered and promotes it, and promotion needs the
4093    /// candidate's forwarding endpoint, so the candidate is published only
4094    /// once that endpoint exists. Unlike an ordinary HELLO, this does not
4095    /// recompute capability verdicts or run the capability deny census
4096    /// (`enforce_capability_denies`): both are about the module in the active
4097    /// slot, which `route.open` can target. A candidate sits in the candidate
4098    /// slot, so it neither provides capabilities to other modules nor can be
4099    /// routed to until promotion moves it into the active slot.
4100    #[allow(clippy::too_many_arguments)]
4101    fn register_swap_candidate(
4102        &self,
4103        connection_id: ConnectionId,
4104        sink: Option<crate::FrameSink>,
4105        frame: &Frame,
4106        manifest: ModuleManifest,
4107        negotiated_ver: u8,
4108        control_ops: Vec<String>,
4109        hello_ack: Frame,
4110        hello_authority: Option<&HelloAuthority>,
4111        differing_declarations: &mut ProviderDeclarations,
4112        #[cfg(windows)] launch_nonce: Option<&str>,
4113    ) -> Result<Vec<Frame>, RouterError> {
4114        let module_id = manifest.module_id.clone();
4115        // Closing a swap and publishing its candidate must be ordered. The
4116        // spawn role proves which slot is intended, not that the slot is open.
4117        self.supervisor
4118            .with_swap_candidate_admission(&module_id, || {
4119                let reply = self.publish_swap_candidate(
4120                    connection_id,
4121                    sink,
4122                    frame,
4123                    manifest,
4124                    negotiated_ver,
4125                    control_ops,
4126                    hello_ack,
4127                    hello_authority,
4128                    differing_declarations,
4129                    #[cfg(windows)]
4130                    launch_nonce,
4131                );
4132                let admitted = self
4133                    .registry
4134                    .get_candidate(&module_id)
4135                    .ok()
4136                    .flatten()
4137                    .is_some_and(|candidate| candidate.connection_id == connection_id);
4138                (reply, admitted)
4139            })
4140            .unwrap_or_else(|| {
4141                Ok(vec![control_error_frame(
4142                    frame,
4143                    "swap_token_invalid",
4144                    "swap ended before candidate HELLO publication",
4145                )?])
4146            })
4147    }
4148
4149    #[allow(clippy::too_many_arguments)]
4150    fn publish_swap_candidate(
4151        &self,
4152        connection_id: ConnectionId,
4153        sink: Option<crate::FrameSink>,
4154        frame: &Frame,
4155        manifest: ModuleManifest,
4156        negotiated_ver: u8,
4157        control_ops: Vec<String>,
4158        hello_ack: Frame,
4159        hello_authority: Option<&HelloAuthority>,
4160        differing_declarations: &mut ProviderDeclarations,
4161        #[cfg(windows)] launch_nonce: Option<&str>,
4162    ) -> Result<Vec<Frame>, RouterError> {
4163        let module_id = manifest.module_id.clone();
4164        let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
4165            manifest,
4166            negotiated_ver,
4167            connection_id,
4168            control_ops,
4169        ) {
4170            Ok(registration) => registration,
4171            Err(RegistryError::DuplicateModuleId { module_id }) => {
4172                return Ok(vec![control_error_frame(
4173                    frame,
4174                    "duplicate_module_id",
4175                    format!(
4176                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
4177                    ),
4178                )?])
4179            }
4180            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
4181                return Ok(vec![control_error_frame(
4182                    frame,
4183                    "invalid_module_id",
4184                    err.to_string(),
4185                )?])
4186            }
4187            Err(err) => {
4188                return Ok(vec![control_error_frame(
4189                    frame,
4190                    "registry_error",
4191                    err.to_string(),
4192                )?])
4193            }
4194        };
4195        *differing_declarations = match self.admit_hello_authority(connection_id, hello_authority) {
4196            Ok(differing) => differing,
4197            Err(err) => {
4198                if matches!(
4199                    self.rollback_unpublished_hello(connection_id),
4200                    Ok(r) if !r.is_empty()
4201                ) {
4202                    crate::supervise::notify_registration_release();
4203                }
4204                return Err(RouterError::Forwarding(err));
4205            }
4206        };
4207        #[cfg(test)]
4208        self.hold_between_registry_and_forwarding();
4209        let mut reply = Vec::new();
4210        let publication = self
4211            .registry
4212            .publish_registration_after(connection_id, || {
4213                if let Some(sink) = sink {
4214                    // Same ordering as an ordinary HELLO: queue the ACK before
4215                    // inserting the endpoint, and keep registry readers excluded
4216                    // until the candidate is published.
4217                    let concurrency = manifest_concurrency(&registration.manifest);
4218                    self.forwarding.register_candidate_module_connection_acked(
4219                        connection_id,
4220                        module_id.clone(),
4221                        negotiated_ver,
4222                        concurrency,
4223                        sink,
4224                        hello_ack,
4225                    )?;
4226                } else {
4227                    reply.push(hello_ack);
4228                }
4229                #[cfg(test)]
4230                self.hold_between_ack_and_publication();
4231                Ok(())
4232            })
4233            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4234        let published = match publication {
4235            Ok(published) => published,
4236            Err(err) => {
4237                if matches!(
4238                    self.rollback_unpublished_hello(connection_id),
4239                    Ok(r) if !r.is_empty()
4240                ) {
4241                    crate::supervise::notify_registration_release();
4242                }
4243                self.refuse_hello_authority(connection_id, hello_authority);
4244                return Ok(vec![control_error_frame(
4245                    frame,
4246                    forwarding_error_code(&err),
4247                    err.to_string(),
4248                )?]);
4249            }
4250        };
4251        if !published {
4252            debug!(
4253                module_id = %module_id,
4254                connection_id = connection_id.get(),
4255                "swap candidate registration was removed before it could be published"
4256            );
4257            self.refuse_hello_authority(connection_id, hello_authority);
4258            return Ok(reply);
4259        }
4260        info!(
4261            module_id = %module_id,
4262            module_version = %registration.manifest.module_version,
4263            negotiated_ver,
4264            ready = registration.ready,
4265            connection_id = connection_id.get(),
4266            "swap candidate registered; not routable until cutover"
4267        );
4268        #[cfg(windows)]
4269        if let Some(nonce) = launch_nonce {
4270            self.supervisor.program_hello_accepted(&module_id, nonce);
4271        }
4272        Ok(reply)
4273    }
4274
4275    /// Bind `connection` to the spawn its HELLO registers as and freeze the
4276    /// spawn's requested provider declarations. Runs only once the registry
4277    /// has accepted the HELLO, before its endpoint is published, so a route
4278    /// reserved on the endpoint always finds the spawn. Returns the requested
4279    /// entries that differ from the saved approvals, for a widening request.
4280    fn admit_hello_authority(
4281        &self,
4282        connection: ConnectionId,
4283        authority: Option<&HelloAuthority>,
4284    ) -> Result<ProviderDeclarations, ForwardingError> {
4285        let Some(authority) = authority else {
4286            return Ok(ProviderDeclarations::new());
4287        };
4288        let froze = authority
4289            .record
4290            .freeze_declarations(authority.requested.clone());
4291        authority
4292            .froze
4293            .store(froze.is_some(), std::sync::atomic::Ordering::Relaxed);
4294        self.forwarding
4295            .bind_target_authority(connection, authority.record.clone())
4296            .inspect_err(|_| self.refuse_hello_authority(connection, Some(authority)))?;
4297        Ok(froze.unwrap_or_default())
4298    }
4299
4300    /// Undo [`Self::admit_hello_authority`] for a HELLO refused after the
4301    /// registry accepted it, so the refused connection is no registration of
4302    /// the spawn and the spawn's declarations are left for the HELLO that
4303    /// does register it to freeze.
4304    fn refuse_hello_authority(&self, connection: ConnectionId, authority: Option<&HelloAuthority>) {
4305        let Some(authority) = authority else {
4306            return;
4307        };
4308        if let Err(err) = self.forwarding.unbind_target_authority(connection) {
4309            warn!(error = %err, "failed to unbind a refused HELLO from its spawn");
4310        }
4311        if authority
4312            .froze
4313            .swap(false, std::sync::atomic::Ordering::Relaxed)
4314        {
4315            authority.record.unfreeze_refused_declarations();
4316        }
4317    }
4318
4319    fn build_hello_ack(
4320        &self,
4321        frame: &Frame,
4322        negotiated_ver: u8,
4323        module_id: &str,
4324    ) -> Result<Frame, RouterError> {
4325        let ack = ModuleHelloAckBody {
4326            negotiated_ver,
4327            subc_ops: module_subc_ops(),
4328            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
4329            storage: self
4330                .storage_config
4331                .as_ref()
4332                .map(|cfg| cfg.descriptor_for(module_id)),
4333            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4334        };
4335        let body = serde_json::to_vec(&ack).map_err(|err| {
4336            RouterError::backend(
4337                0,
4338                frame.header.corr,
4339                format!("failed to encode HELLO_ACK: {err}"),
4340            )
4341        })?;
4342
4343        Frame::build_with_version(
4344            negotiated_ver,
4345            FrameType::HelloAck,
4346            control_flags(),
4347            0,
4348            0,
4349            frame.header.corr,
4350            body,
4351        )
4352        .map_err(RouterError::FrameBuild)
4353    }
4354
4355    async fn handle_client_control_request(
4356        &self,
4357        ctx: &RouteCtx,
4358        frame: Frame,
4359        request: ClientControlRequest,
4360        route_open_timings: &mut Option<RouteOpenTimings>,
4361    ) -> Result<Vec<Frame>, RouterError> {
4362        if let Some(owner) = &self.config_runtime {
4363            let module = match &request {
4364                ClientControlRequest::SupervisorRestart { module_id, .. }
4365                | ClientControlRequest::SupervisorReload { module_id }
4366                | ClientControlRequest::SupervisorSwap { module_id, .. }
4367                | ClientControlRequest::SupervisorSetEnabled {
4368                    module_id,
4369                    enabled: true,
4370                } => Some(module_id.as_str()),
4371                _ => None,
4372            };
4373            let rescan = matches!(
4374                &request,
4375                ClientControlRequest::SupervisorRescan { preview: false }
4376                    | ClientControlRequest::SupervisorRescanChecked { preview: false, .. }
4377            );
4378            let held = if module.is_some() || rescan {
4379                let module = module.map(str::to_owned);
4380                owner
4381                    .on_blocking(move |owner| owner.checked_admission_hold(module.as_deref()))
4382                    .await
4383            } else {
4384                Ok(())
4385            };
4386            if let Err(error) = held {
4387                if error.code != "config_delivery_deferred" {
4388                    let code = if error.code == "store_tampered" {
4389                        "store_tampered"
4390                    } else {
4391                        "config_gated"
4392                    };
4393                    return Ok(vec![control_error_frame(&frame, code, error.detail)?]);
4394                }
4395            }
4396        }
4397        if let Some(operation) = privileged_operation(&request) {
4398            let caller = self.resolve_caller(ctx.connection_id).await;
4399            let op = client_control_request_op(&request);
4400            let text = prompt_text(operation, &caller);
4401            match self.privileged_operator_prompt {
4402                PrivilegedOperatorPrompt::Log => {
4403                    info!(target: "subc_daemon::privileged_prompt", operation = op,
4404                        target = ?operation, caller = %caller.description(), summary = %text,
4405                        "privileged request would prompt the operator");
4406                }
4407                PrivilegedOperatorPrompt::Enforce => {
4408                    let key =
4409                        crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
4410                    let stamp = self
4411                        .operator_apps
4412                        .lock()
4413                        .unwrap_or_else(|p| p.into_inner())
4414                        .get(&ctx.connection_id)
4415                        .cloned();
4416                    let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
4417                        &self.operator_app_policy,
4418                        &self.launch_table,
4419                        stamp.as_ref(),
4420                    )
4421                    .then(|| self.shortcut_context(stamp, None, None));
4422                    let outcome = self
4423                        .forwarding
4424                        .operator_confirms()
4425                        .approve_privileged_with_shortcut(
4426                            ctx,
4427                            &frame,
4428                            op,
4429                            text,
4430                            caller.description(),
4431                            key,
4432                            shortcut,
4433                        )
4434                        .await;
4435                    if outcome.code != "confirmed" {
4436                        return Ok(vec![outcome.frame(&frame)]);
4437                    }
4438                }
4439            }
4440        }
4441        match request {
4442            ClientControlRequest::DaemonShutdownIntent { intent } => {
4443                if intent.as_str() != Some("restart") {
4444                    return Ok(vec![control_error_frame(
4445                        &frame,
4446                        "invalid_params",
4447                        "intent must be 'restart'",
4448                    )?]);
4449                }
4450                // A stamped daemon boots a newer approved generation only from a
4451                // durable restart intent, written here before the daemon exits.
4452                // Failing to write it never blocks the restart: the next start
4453                // then boots the last active generation and the newer one waits.
4454                if let Some(owner) = &self.config_runtime {
4455                    match owner
4456                        .on_blocking(|owner| owner.assign_daemon_restart())
4457                        .await
4458                    {
4459                        Ok(Some(generation)) => {
4460                            info!(
4461                                generation,
4462                                "daemon restart will warm up approved generation"
4463                            )
4464                        }
4465                        Ok(None) => {}
4466                        Err(error) => warn!(
4467                            %error,
4468                            "daemon restart intent not recorded; the next start boots the last active generation"
4469                        ),
4470                    }
4471                }
4472                let response = ClientControlResponse::DaemonShutdownIntent {
4473                    intent: "restart".to_owned(),
4474                    expires_at_ms: self.supervisor.announce_daemon_restart(),
4475                };
4476                Ok(vec![control_response_body_frame(
4477                    &frame,
4478                    &response,
4479                    "ClientControlResponse::DaemonShutdownIntent",
4480                )?])
4481            }
4482            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
4483            ClientControlRequest::CatalogList { module_id } => {
4484                self.handle_catalog_list(frame, module_id)
4485            }
4486            ClientControlRequest::RouteOpen {
4487                target,
4488                identity,
4489                consumer_identity,
4490                consumer_capabilities,
4491                role_versions,
4492                busy_refusals,
4493                admission_facts,
4494                scope,
4495                ..
4496            } => {
4497                let extensions = match serde_json::from_slice::<subc_control::RouteOpenRequestExt>(
4498                    &frame.body,
4499                ) {
4500                    Ok(extensions) => extensions,
4501                    Err(err) => {
4502                        return Ok(vec![control_error_frame(
4503                            &frame,
4504                            "invalid_control_body",
4505                            format!("malformed route.open extensions: {err}"),
4506                        )?]);
4507                    }
4508                };
4509                self.handle_route_open(
4510                    ctx,
4511                    frame,
4512                    RouteOpenRequest {
4513                        target,
4514                        identity,
4515                        consumer_identity,
4516                        consumer_capabilities,
4517                        role_versions,
4518                        busy_refusals,
4519                        admission_facts,
4520                        scope,
4521                        accepts_close_reasons: extensions.accepts_close_reasons,
4522                    },
4523                    route_open_timings.get_or_insert_with(RouteOpenTimings::new),
4524                )
4525                .await
4526            }
4527            ClientControlRequest::RoutePoll {
4528                route_channel,
4529                route_epoch,
4530                kind,
4531            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
4532            ClientControlRequest::SupervisorList {} => {
4533                let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
4534                    self.agent_declarations.refuse(
4535                        self,
4536                        ctx,
4537                        control_error_frame(
4538                            &frame,
4539                            "busy",
4540                            "supervisor list work is at capacity; retry",
4541                        )?,
4542                    );
4543                    return Ok(Vec::new());
4544                };
4545                // Filesystem-backed diagnostics must not hold up this connection's reader.
4546                let handler = self.clone();
4547                let sink = ctx.egress.clone();
4548                tokio::spawn(
4549                    async move {
4550                        let _permit = permit;
4551                        let result = handler.handle_supervisor_list(frame.clone()).await;
4552                        let replies = match result {
4553                            Ok(replies) => replies,
4554                            Err(error) => {
4555                                control_error_frame(&frame, "backend_error", error.to_string())
4556                                    .into_iter()
4557                                    .collect()
4558                            }
4559                        };
4560                        for reply in replies {
4561                            if !matches!(
4562                                tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
4563                                    .await,
4564                                Ok(Ok(()))
4565                            ) {
4566                                break;
4567                            }
4568                        }
4569                    }
4570                    .with_current_subscriber(),
4571                );
4572                Ok(Vec::new())
4573            }
4574            ClientControlRequest::SupervisorSpawnSnapshot {} => {
4575                self.handle_supervisor_spawn_snapshot(frame)
4576            }
4577            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
4578                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
4579            }
4580            ClientControlRequest::SupervisorRestart {
4581                module_id,
4582                drain_timeout_ms,
4583            } => {
4584                self.log_supervisor_request_received(
4585                    ctx,
4586                    frame.header.corr,
4587                    ops::SUPERVISOR_RESTART,
4588                    Some(&module_id),
4589                    None,
4590                )?;
4591                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4592                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
4593                    .await
4594            }
4595            ClientControlRequest::SupervisorSwap {
4596                module_id,
4597                ready_timeout_ms,
4598            } => {
4599                self.log_supervisor_request_received(
4600                    ctx,
4601                    frame.header.corr,
4602                    ops::SUPERVISOR_SWAP,
4603                    Some(&module_id),
4604                    None,
4605                )?;
4606                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4607                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
4608                    .await
4609            }
4610            ClientControlRequest::SupervisorReload { module_id } => {
4611                self.log_supervisor_request_received(
4612                    ctx,
4613                    frame.header.corr,
4614                    ops::SUPERVISOR_RELOAD,
4615                    Some(&module_id),
4616                    None,
4617                )?;
4618                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4619                self.handle_supervisor_reload(frame, module_id, requester)
4620                    .await
4621            }
4622            ClientControlRequest::SupervisorRescan { preview } => {
4623                if !preview {
4624                    self.log_supervisor_request_received(
4625                        ctx,
4626                        frame.header.corr,
4627                        ops::SUPERVISOR_RESCAN,
4628                        None,
4629                        None,
4630                    )?;
4631                }
4632                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4633                let operation = self.handle_supervisor_rescan(
4634                    frame,
4635                    if preview {
4636                        RescanMode::Preview
4637                    } else {
4638                        RescanMode::Execute
4639                    },
4640                    requester,
4641                );
4642                self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
4643                    .await
4644            }
4645            ClientControlRequest::SupervisorRescanChecked {
4646                preview,
4647                candidate,
4648                expect,
4649            } => {
4650                self.observe_control_wait(
4651                    ctx,
4652                    None,
4653                    ops::SUPERVISOR_RESCAN,
4654                    self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
4655                )
4656                .await
4657            }
4658            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
4659                self.log_supervisor_request_received(
4660                    ctx,
4661                    frame.header.corr,
4662                    ops::SUPERVISOR_RELEASE_RESERVED,
4663                    Some(&module_id),
4664                    None,
4665                )?;
4666                self.handle_supervisor_release_reserved(frame, module_id)
4667                    .await
4668            }
4669            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
4670                self.log_supervisor_request_received(
4671                    ctx,
4672                    frame.header.corr,
4673                    ops::SUPERVISOR_SET_ENABLED,
4674                    Some(&module_id),
4675                    Some(enabled),
4676                )?;
4677                let requester = self.supervisor_requester(ctx, frame.header.corr)?;
4678                self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
4679                    .await
4680            }
4681            ClientControlRequest::SupervisorHealthProbe { module_id } => {
4682                self.handle_supervisor_health_probe(ctx, frame, module_id)
4683                    .await
4684            }
4685            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
4686            ClientControlRequest::SupervisorRoutes { module_id } => {
4687                self.handle_supervisor_routes(frame, module_id)
4688            }
4689            ClientControlRequest::SupervisorProvenance { module_id } => {
4690                self.handle_supervisor_provenance(ctx, frame, module_id)
4691                    .await
4692            }
4693            ClientControlRequest::SupervisorStderrTail {
4694                module_id,
4695                max_lines,
4696                max_bytes,
4697            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
4698            ClientControlRequest::SupervisorTerminals { module_id } => {
4699                self.handle_supervisor_terminals(ctx, frame, module_id)
4700                    .await
4701            }
4702        }
4703    }
4704
4705    fn log_supervisor_request_received(
4706        &self,
4707        ctx: &RouteCtx,
4708        corr: u64,
4709        op: &'static str,
4710        module_id: Option<&str>,
4711        enabled: Option<bool>,
4712    ) -> Result<(), RouterError> {
4713        let caller = self.supervisor_requester(ctx, corr)?;
4714        let caller = principal_label(&caller);
4715        // `Option` fields are recorded only when present, so a request with no
4716        // module id (rescan) or no enabled flag simply omits that field.
4717        info!(
4718            target: "control",
4719            op,
4720            module_id,
4721            enabled,
4722            connection_id = ctx.connection_id.get(),
4723            caller = %caller,
4724            "supervisor request received"
4725        );
4726        Ok(())
4727    }
4728
4729    fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
4730        Ok(self
4731            .registry
4732            .get_module_by_connection(ctx.connection_id)
4733            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4734            .map(|registration| Principal::Reserved {
4735                module_id: registration.manifest.module_id,
4736            })
4737            .unwrap_or(Principal::Direct))
4738    }
4739
4740    fn handle_module_control_request(
4741        &self,
4742        connection_id: ConnectionId,
4743        frame: Frame,
4744        request: ModuleControlRequestFromModule,
4745    ) -> Result<Vec<Frame>, RouterError> {
4746        match request {
4747            ModuleControlRequestFromModule::RouteAuthorize {
4748                principal,
4749                scope,
4750                target,
4751                expected,
4752            } => self.handle_route_authorize(
4753                connection_id,
4754                frame,
4755                principal,
4756                scope,
4757                target,
4758                expected,
4759            ),
4760            ModuleControlRequestFromModule::CatalogUpdate {
4761                provides,
4762                capabilities,
4763                ready,
4764            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
4765            ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { role_declarations } => {
4766                self.handle_catalog_update_role_declarations(
4767                    connection_id,
4768                    frame,
4769                    role_declarations,
4770                )
4771            }
4772            ModuleControlRequestFromModule::LiveRoots {} => {
4773                let registered = self
4774                    .registry
4775                    .get_module_by_connection(connection_id)
4776                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4777                let Some(registration) = registered else {
4778                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
4779                };
4780                let response = self
4781                    .forwarding
4782                    .live_roots(&registration.manifest.module_id)
4783                    .map_err(RouterError::Forwarding)?;
4784                Ok(vec![control_response_body_frame(
4785                    &frame,
4786                    &response,
4787                    "ModuleControlResponseToModule::LiveRoots",
4788                )?])
4789            }
4790            ModuleControlRequestFromModule::AgentCommands {
4791                started,
4792                finished,
4793                known,
4794            } => self.handle_agent_commands(connection_id, frame, started, finished, known),
4795            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
4796                self.handle_scope_sync(connection_id, frame, generation, scopes)
4797            }
4798            ModuleControlRequestFromModule::ScopeApply {
4799                generation,
4800                upsert,
4801                end,
4802            } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
4803            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
4804                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
4805            }
4806            _ => Ok(vec![control_error_frame(
4807                &frame,
4808                "unsupported_control_frame",
4809                "unsupported module control operation",
4810            )?]),
4811        }
4812    }
4813
4814    fn handle_scope_served_by(
4815        &self,
4816        connection_id: ConnectionId,
4817        frame: Frame,
4818    ) -> Result<Vec<Frame>, RouterError> {
4819        let registered = self
4820            .registry
4821            .get_module_by_connection(connection_id)
4822            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4823        let Some(registered) = registered else {
4824            return Ok(vec![control_error_frame(
4825                &frame,
4826                "not_registered",
4827                "scope.served_by requires an active module registration owned by this connection",
4828            )?]);
4829        };
4830        let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
4831        {
4832            Ok(request) => request,
4833            Err(err) => {
4834                return Ok(vec![control_error_frame(
4835                    &frame,
4836                    "invalid_control_body",
4837                    format!("malformed scope.served_by body: {err}"),
4838                )?])
4839            }
4840        };
4841        let answer = self
4842            .scopes
4843            .read()
4844            .unwrap_or_else(|poisoned| poisoned.into_inner())
4845            .served_by(
4846                &registered.manifest.module_id,
4847                &request.scope_ref,
4848                request.scope_epoch,
4849                &request.module_id,
4850            );
4851        let reply = subc_control::ScopeServedByReply::new(
4852            answer,
4853            self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
4854        );
4855        Ok(vec![control_response_body_frame(
4856            &frame,
4857            &reply,
4858            "ScopeServedByReply",
4859        )?])
4860    }
4861
4862    fn handle_operator_confirm(
4863        &self,
4864        ctx: &RouteCtx,
4865        frame: Frame,
4866    ) -> Result<Vec<Frame>, RouterError> {
4867        use crate::operator_confirm::{audit, Outcome};
4868        let registration = self
4869            .registry
4870            .get_module_by_connection(ctx.connection_id)
4871            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4872        let Some(registration) = registration else {
4873            let outcome = Outcome::refusal("not_registered");
4874            audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4875            return Ok(vec![outcome.frame(&frame)]);
4876        };
4877        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
4878            Ok(request) => request,
4879            Err(_) => {
4880                let outcome = Outcome::refusal("invalid_control_body");
4881                audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
4882                return Ok(vec![outcome.frame(&frame)]);
4883            }
4884        };
4885        let declared = matches!(
4886            request.kind,
4887            Some(subc_protocol::session::OperatorConfirmKind::Operation)
4888        ) && request.operation.as_ref().is_some_and(|operation| {
4889            registration
4890                .manifest
4891                .confirm_operations
4892                .as_ref()
4893                .is_some_and(|operations| operations.contains(operation))
4894        });
4895        let module_id = registration.manifest.module_id;
4896        // Read the launch nonce (under its own lock) before taking the forwarding
4897        // table's lock below: holding forwarding while waiting on another daemon
4898        // lock risks a lock-order deadlock with paths that take them the other way.
4899        let nonce = self
4900            .hello_launch_nonces
4901            .lock()
4902            .unwrap_or_else(|p| p.into_inner())
4903            .nonce(ctx.connection_id)
4904            .map(str::to_owned);
4905        // A registration channel proves its own spawn; only a connection that
4906        // registered over the launch nonce falls back to the nonce.
4907        let nonce_proven = self
4908            .confirm_channel_proven(ctx.connection_id, &module_id)
4909            .unwrap_or_else(|| {
4910                nonce.as_deref().is_some_and(|nonce| {
4911                    self.supervisor
4912                        .spawned_consumer_authorized(&module_id, nonce)
4913                })
4914            });
4915        let confirms = self.forwarding.operator_confirms();
4916        self.forwarding
4917            .with_operator_route(
4918                ctx.connection_id,
4919                request.route_channel,
4920                request.route_epoch,
4921                |binding| {
4922                    let shortcut = binding
4923                        .filter(|binding| {
4924                            nonce_proven
4925                                && crate::operator_confirm::valid_summary(&request.summary)
4926                                && declared
4927                                && crate::operator_confirm::ShortcutContext::route_eligible(
4928                                    &self.operator_app_policy,
4929                                    &self.launch_table,
4930                                    binding,
4931                                )
4932                        })
4933                        .map(|binding| {
4934                            let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
4935                                &self.operator_app_policy,
4936                                binding,
4937                            );
4938                            let stamp = if phone.is_some() {
4939                                None
4940                            } else {
4941                                binding.operator_app.clone()
4942                            };
4943                            self.shortcut_context(
4944                                stamp,
4945                                phone,
4946                                Some((request.route_channel, request.route_epoch)),
4947                            )
4948                        });
4949                    confirms.admit_with_shortcut(
4950                        ctx,
4951                        frame,
4952                        module_id,
4953                        nonce_proven,
4954                        request,
4955                        binding,
4956                        shortcut,
4957                    )
4958                },
4959            )
4960            .map_err(RouterError::Forwarding)
4961    }
4962
4963    fn shortcut_context(
4964        &self,
4965        stamp: Option<crate::operator_apps::OperatorAppStamp>,
4966        phone_fingerprint: Option<String>,
4967        route: Option<(u16, u32)>,
4968    ) -> crate::operator_confirm::ShortcutContext {
4969        crate::operator_confirm::ShortcutContext {
4970            ledger: self.activity.clone(),
4971            forwarding: self.forwarding.clone(),
4972            launches: self.launch_table.clone(),
4973            stamp,
4974            phone_fingerprint,
4975            route,
4976            connections: self.operator_apps.clone(),
4977            #[cfg(test)]
4978            hooks: Default::default(),
4979        }
4980    }
4981
4982    /// Explicit test-only phone policy; daemon configuration cannot select it.
4983    #[cfg(any(test, feature = "test-support"))]
4984    pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
4985        self.operator_app_policy.phone_enabled = enabled;
4986        self
4987    }
4988
4989    /// Resolve folders before taking the scope-table lock: a slow filesystem
4990    /// must not hold up admissions for unrelated scoped routes.
4991    pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
4992        let mut scopes = ScopeRecords::from(scopes);
4993        for (index, record) in scopes.records.iter_mut().enumerate() {
4994            if let Some(refusal) = Self::canonicalize_workdir(record) {
4995                scopes.refusals.insert(index, refusal);
4996            }
4997        }
4998        scopes
4999    }
5000
5001    fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
5002        let workdir = record.workdir.as_mut()?;
5003        let path = std::path::Path::new(workdir);
5004        if !path.is_absolute() {
5005            return Some((
5006                error_codes::INVALID_WORKDIR,
5007                "workdir must be absolute".to_string(),
5008            ));
5009        }
5010        match ProjectRootId::from_path_allowing_missing(path) {
5011            Ok(root) => match root.as_path().to_str() {
5012                Some(canonical) => {
5013                    *workdir = canonical.to_string();
5014                    None
5015                }
5016                None => Some((
5017                    error_codes::INVALID_WORKDIR,
5018                    "canonical workdir is not UTF-8".to_string(),
5019                )),
5020            },
5021            Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
5022        }
5023    }
5024
5025    /// `scope.sync`: the owner is the module registered on this connection.
5026    /// A connection with no registration (every client connection, `direct`
5027    /// included) is refused `not_registered` before the table is consulted.
5028    fn handle_scope_sync(
5029        &self,
5030        connection_id: ConnectionId,
5031        frame: Frame,
5032        generation: u64,
5033        scopes: Vec<ScopeRecord>,
5034    ) -> Result<Vec<Frame>, RouterError> {
5035        self.handle_scope_change(connection_id, frame, generation, scopes, None)
5036    }
5037
5038    fn handle_scope_change(
5039        &self,
5040        connection_id: ConnectionId,
5041        frame: Frame,
5042        generation: u64,
5043        scopes: Vec<ScopeRecord>,
5044        end: Option<Vec<ScopeEnd>>,
5045    ) -> Result<Vec<Frame>, RouterError> {
5046        let is_apply = end.is_some();
5047        let Some(registration) = self
5048            .registry
5049            .get_module_by_connection(connection_id)
5050            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5051        else {
5052            return Ok(vec![control_error_frame(
5053                &frame,
5054                "not_registered",
5055                "scope.sync and scope.apply require an active module registration owned by this connection",
5056            )?]);
5057        };
5058        let owner = registration.manifest.module_id;
5059        let scopes = Self::prepare_scope_records(scopes);
5060        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
5061        let is_current_launch = |connection: ConnectionId| {
5062            self.connection_is_current_launch(&owner, connection, current_nonce.as_deref())
5063        };
5064        // Lock order is the scope table, then the forwarding table: the new
5065        // tags are published, and the routes the change closes are selected,
5066        // while the scope table is still write-locked, so no admission can read
5067        // a record whose tag is not yet published.
5068        let mut table = self
5069            .scopes
5070            .write()
5071            .unwrap_or_else(|poisoned| poisoned.into_inner());
5072        let outcome = match end {
5073            Some(end) => table.apply(
5074                &owner,
5075                connection_id,
5076                is_current_launch,
5077                generation,
5078                scopes,
5079                end,
5080            ),
5081            None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
5082        };
5083        let drained = match &outcome {
5084            Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
5085            Err(_) => Vec::new(),
5086        };
5087        let has_deadlines = outcome.is_ok() && table.has_deadlines();
5088        drop(table);
5089        if has_deadlines {
5090            self.scope_deadline_added.notify_one();
5091        }
5092        match outcome {
5093            Ok(applied) => {
5094                let counts = ScopeOutcomeCounts::of(&applied.results);
5095                info!(
5096                    owner = %owner,
5097                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5098                    generation,
5099                    records = applied.results.len(),
5100                    created = counts.created,
5101                    replaced = counts.replaced,
5102                    updated = counts.updated,
5103                    unchanged = counts.unchanged,
5104                    refused = counts.refused,
5105                    ended = applied.ended.len(),
5106                    tag_changes = applied.tag_changes.len(),
5107                    routes_closed = drained.len(),
5108                    "scope change accepted"
5109                );
5110                // An accepted scope change can still refuse individual records, and the
5111                // owner is the only party that sees the reply. Name them here so
5112                // an operator can tell a refused session from a missing one
5113                // without the owner's logs. Capped so a call that refuses
5114                // thousands cannot flood the log; the count above is complete.
5115                for refused in applied
5116                    .results
5117                    .iter()
5118                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
5119                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
5120                {
5121                    warn!(
5122                        owner = %owner,
5123                        generation,
5124                        scope_ref = %refused.scope_ref,
5125                        scope_epoch = refused.scope_epoch,
5126                        code = refused.code.as_deref().unwrap_or(""),
5127                        "scope record refused"
5128                    );
5129                }
5130                self.close_scope_drained_routes(drained);
5131                let response = if is_apply {
5132                    ModuleControlResponseToModule::ScopeApply {
5133                        generation,
5134                        results: applied.results,
5135                        end_results: applied.end_results,
5136                        ended: applied.ended,
5137                    }
5138                } else {
5139                    ModuleControlResponseToModule::ScopeSync {
5140                        generation,
5141                        results: applied.results,
5142                        ended: applied.ended,
5143                    }
5144                };
5145                Ok(vec![control_response_body_frame(
5146                    &frame,
5147                    &response,
5148                    "scope change response",
5149                )?])
5150            }
5151            Err(refusal) => {
5152                info!(
5153                    owner = %owner,
5154                    op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
5155                    generation,
5156                    code = refusal.code,
5157                    "scope change refused"
5158                );
5159                Ok(vec![control_error_frame(
5160                    &frame,
5161                    refusal.code,
5162                    refusal.message,
5163                )?])
5164            }
5165        }
5166    }
5167
5168    /// Notify both endpoints of routes drained by a scope change.
5169    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
5170        scope_expiry::ScopeExpiryContext::new(self).close_scope_drained_routes(drained);
5171    }
5172
5173    /// Re-authorize a stored source on a principal the module supplies.
5174    ///
5175    /// That principal carries no spawn or authority epoch, so this reply never
5176    /// conveys a caller attestation or a first-party mark, and a provider must
5177    /// not apply automatic grants on anything it authorizes.
5178    fn handle_route_authorize(
5179        &self,
5180        connection_id: ConnectionId,
5181        frame: Frame,
5182        principal: Principal,
5183        scope: subc_protocol::scope::RouteAuthorizeScope,
5184        target: String,
5185        expected: subc_protocol::scope::RouteAuthorizeExpected,
5186    ) -> Result<Vec<Frame>, RouterError> {
5187        use crate::scopes::admission::{admit_route, AdmissionMode};
5188        use subc_protocol::scope::{
5189            RouteAuthorizeOutcome, RouteAuthorizeResponse, ROUTE_AUTHORIZE_CAPABILITY,
5190        };
5191        let registry = self
5192            .registry
5193            .route_admission_snapshot()
5194            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5195        let owner_configured = match &scope.owner {
5196            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5197            _ => false,
5198        };
5199        let selector = ScopeSelector::new(scope.owner, scope.scope_ref, Some(scope.epoch));
5200        let answer = {
5201            let scopes = self.scopes.read().unwrap_or_else(|p| p.into_inner());
5202            let permitted = registry
5203                .iter()
5204                .find(|r| r.connection_id == connection_id)
5205                .is_some_and(|r| {
5206                    scopes.is_federated_owner(&Principal::Reserved {
5207                        module_id: r.manifest.module_id.clone(),
5208                    }) && r.manifest.capabilities.as_ref().is_some_and(|caps| {
5209                        caps.provides
5210                            .iter()
5211                            .any(|cap| cap == ROUTE_AUTHORIZE_CAPABILITY)
5212                    })
5213                });
5214            if !permitted {
5215                drop(scopes);
5216                return Ok(vec![control_error_frame(&frame, error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED,
5217                    "route.authorize requires the configured federated owner's route-authorize/v1 registration")?]);
5218            }
5219            admit_route(
5220                &scopes,
5221                &registry,
5222                &principal,
5223                &target,
5224                Some(&selector),
5225                owner_configured,
5226                AdmissionMode::Authorize(&expected),
5227            )
5228        };
5229        let response = match answer {
5230            Ok(admission) => {
5231                let mut response = RouteAuthorizeResponse::new(
5232                    RouteAuthorizeOutcome::Authorized,
5233                    "route admission authorized",
5234                );
5235                response.content_version = admission.content_version;
5236                response
5237            }
5238            Err(refusal) => refusal.response(),
5239        };
5240        Ok(vec![control_response_body_frame(
5241            &frame,
5242            &response,
5243            "RouteAuthorizeResponse",
5244        )?])
5245    }
5246
5247    /// `scope.describe`: any registered module may read any scope, because a
5248    /// provider must read the scope a route it serves is stamped with.
5249    fn handle_scope_describe(
5250        &self,
5251        connection_id: ConnectionId,
5252        frame: Frame,
5253        owner: Principal,
5254        scope_ref: String,
5255    ) -> Result<Vec<Frame>, RouterError> {
5256        let registered = self
5257            .registry
5258            .get_module_by_connection(connection_id)
5259            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5260        let Some(registered) = registered else {
5261            return Ok(vec![control_error_frame(
5262                &frame,
5263                "not_registered",
5264                "scope.describe requires an active module registration owned by this connection",
5265            )?]);
5266        };
5267        let scopes = self
5268            .scopes
5269            .read()
5270            .unwrap_or_else(|poisoned| poisoned.into_inner());
5271        if scopes.is_federated_owner(&owner)
5272            && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
5273        {
5274            return Ok(vec![control_error_frame(
5275                &frame,
5276                error_codes::SCOPE_FEDERATED_UNSUPPORTED,
5277                "scope.describe of the federated owner requires scope-federated/v1",
5278            )?]);
5279        }
5280        let mut description = scopes.describe(&owner, &scope_ref);
5281        drop(scopes);
5282        if owner
5283            != (Principal::Reserved {
5284                module_id: registered.manifest.module_id.clone(),
5285            })
5286        {
5287            if let Some(facts) = description
5288                .stamp
5289                .as_mut()
5290                .and_then(|stamp| stamp.attributes.federated.as_mut())
5291            {
5292                facts.stamp_required = false;
5293            }
5294        }
5295        // Describe readers also decode strict ScopeAttributes. Attribution is
5296        // safe to omit unless the requesting module declares it understands it.
5297        if !registered
5298            .manifest
5299            .capabilities
5300            .as_ref()
5301            .is_some_and(|caps| {
5302                caps.provides
5303                    .iter()
5304                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5305            })
5306        {
5307            if let Some(stamp) = description.stamp.as_mut() {
5308                stamp.attributes.on_behalf_of_agent = None;
5309            }
5310        }
5311        if !registered
5312            .manifest
5313            .capabilities
5314            .as_ref()
5315            .is_some_and(|caps| {
5316                caps.provides
5317                    .iter()
5318                    .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5319            })
5320        {
5321            if let Some(stamp) = description.stamp.as_mut() {
5322                stamp.attributes.flow_version = None;
5323                stamp.attributes.flow_code_hash = None;
5324                stamp.attributes.flow_install_id = None;
5325            }
5326        }
5327        let owner_configured = match &owner {
5328            // Ask whether the owner is configured (`is_configured`), not
5329            // whether it is on the roster (`get(..).is_some()`): a supervised
5330            // module's process can register and describe a scope before the
5331            // supervisor has put it on the roster.
5332            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5333            _ => false,
5334        };
5335        let response = ModuleControlResponseToModule::ScopeDescribe {
5336            status: description.status,
5337            scope_epoch: description.scope_epoch,
5338            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
5339            owner_synced: description.owner_synced,
5340            owner_configured,
5341            scope: description.stamp,
5342        };
5343        Ok(vec![control_response_body_frame(
5344            &frame,
5345            &response,
5346            "ModuleControlResponseToModule::ScopeDescribe",
5347        )?])
5348    }
5349
5350    fn handle_catalog_update_role_declarations(
5351        &self,
5352        connection_id: ConnectionId,
5353        frame: Frame,
5354        role_declarations: serde_json::Value,
5355    ) -> Result<Vec<Frame>, RouterError> {
5356        if let Err(err) = validate_role_declarations_value(Some(&role_declarations)) {
5357            return Ok(vec![control_error_frame(
5358                &frame,
5359                "invalid_role_declarations",
5360                err.to_string(),
5361            )?]);
5362        }
5363        let declarations = serde_json::from_value(role_declarations)
5364            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5365        if self
5366            .registry
5367            .replace_role_declarations_for_connection(connection_id, declarations)
5368            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5369            .is_none()
5370        {
5371            return Ok(vec![control_error_frame(
5372                &frame,
5373                "not_registered",
5374                "catalog.update_role_declarations requires an active module registration owned by this connection",
5375            )?]);
5376        }
5377        control_response_body_frame(
5378            &frame,
5379            &ModuleControlResponseToModule::CatalogUpdateRoleDeclarations {},
5380            "ModuleControlResponseToModule::CatalogUpdateRoleDeclarations",
5381        )
5382        .map(|frame| vec![frame])
5383    }
5384
5385    fn handle_catalog_update(
5386        &self,
5387        connection_id: ConnectionId,
5388        frame: Frame,
5389        provides: Vec<ProviderRole>,
5390        capabilities: Option<CapabilityDeclarations>,
5391        ready: Option<bool>,
5392    ) -> Result<Vec<Frame>, RouterError> {
5393        self.refresh_capability_requirements();
5394        let Some(registration) = self
5395            .registry
5396            .get_module_by_connection(connection_id)
5397            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5398        else {
5399            return Ok(vec![control_error_frame(
5400                &frame,
5401                "not_registered",
5402                "catalog.update requires an active module registration owned by this connection",
5403            )?]);
5404        };
5405
5406        if let Some(message) =
5407            catalog_update_frozen_field_message(&registration.manifest, &provides)
5408        {
5409            return Ok(vec![control_error_frame(
5410                &frame,
5411                "catalog_update_frozen_field",
5412                message,
5413            )?]);
5414        }
5415
5416        let mut candidate = registration.manifest.clone();
5417        candidate.provides = provides.clone();
5418        candidate.capabilities = capabilities
5419            .clone()
5420            .or_else(|| registration.manifest.capabilities.clone());
5421        if let Some(rich_ui) = &candidate.rich_ui {
5422            if let Err(err) = rich_ui.validate() {
5423                return Ok(vec![control_error_frame(
5424                    &frame,
5425                    "invalid_manifest",
5426                    err.to_string(),
5427                )?]);
5428            }
5429        }
5430        if let Err(err) = candidate.validate_capability_grammar() {
5431            return Ok(vec![control_error_frame(
5432                &frame,
5433                "invalid_capability_grammar",
5434                err.to_string(),
5435            )?]);
5436        }
5437
5438        // Updates must honor the same reserved owner as initial registration;
5439        // otherwise an empty HELLO could acquire the claim after admission.
5440        let mut conflicts = self
5441            .capability_evaluator
5442            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
5443        if let Some(conflict) = conflicts.first() {
5444            let message = format!(
5445                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
5446                conflict.capability, conflict.claimants[0], candidate.module_id
5447            );
5448            for conflict in &mut conflicts {
5449                conflict.source = DuplicateClaimSource::CatalogUpdate;
5450            }
5451            log_duplicate_claim_events(conflicts);
5452            return Ok(vec![control_error_frame(
5453                &frame,
5454                "reserved_capability",
5455                message,
5456            )?]);
5457        }
5458
5459        let updated = match self.registry.replace_catalog_for_connection(
5460            connection_id,
5461            provides,
5462            capabilities,
5463            ready,
5464        ) {
5465            Err(crate::registry::RegistryError::FederatedCapabilityChanged) => {
5466                return Ok(vec![control_error_frame(
5467                    &frame,
5468                    error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5469                    "scope-federated/v1 is fixed at HELLO; restart the module to change it",
5470                )?])
5471            }
5472            Err(crate::registry::RegistryError::CallerAttestationCapabilityChanged) => {
5473                return Ok(vec![control_error_frame(
5474                    &frame,
5475                    error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
5476                    "caller-attestation/v1 is fixed at HELLO; restart the module to change it",
5477                )?])
5478            }
5479            result => {
5480                result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5481            }
5482        };
5483        if updated.is_none() {
5484            return Ok(vec![control_error_frame(
5485                &frame,
5486                "not_registered",
5487                "catalog.update requires an active module registration owned by this connection",
5488            )?]);
5489        }
5490        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
5491            log_duplicate_claim_events(
5492                self.capability_evaluator
5493                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
5494            );
5495        }
5496        if capability_census_trigger(
5497            registration.manifest.capabilities.as_ref(),
5498            updated
5499                .as_ref()
5500                .and_then(|entry| entry.manifest.capabilities.as_ref()),
5501        ) {
5502            self.enforce_capability_denies();
5503        }
5504        self.refresh_capability_requirements();
5505
5506        let response = ModuleControlResponseToModule::CatalogUpdate {};
5507        control_response_body_frame(
5508            &frame,
5509            &response,
5510            "ModuleControlResponseToModule::CatalogUpdate",
5511        )
5512        .map(|frame| vec![frame])
5513    }
5514
5515    pub(crate) fn with_activity_ledger_config(
5516        mut self,
5517        run_dir: &Path,
5518        incarnation: String,
5519        config: crate::activity::ActivityConfig,
5520    ) -> Self {
5521        self.activity = Some(crate::activity::ActivityLedger::open_config(
5522            run_dir.join("activity.jsonl"),
5523            incarnation,
5524            config,
5525        ));
5526        self.supervisor.set_activity(self.activity.clone());
5527        self
5528    }
5529
5530    pub(crate) async fn configure_activity(
5531        &self,
5532        config: crate::activity::ActivityConfig,
5533    ) -> Result<(), &'static str> {
5534        match &self.activity {
5535            Some(ledger) => ledger.configure(config).await,
5536            None => Ok(()),
5537        }
5538    }
5539
5540    fn cancel_activity_subscription(
5541        &self,
5542        ctx: &RouteCtx,
5543        frame: &Frame,
5544    ) -> Result<bool, RouterError> {
5545        let task = self
5546            .activity_streams
5547            .lock()
5548            .unwrap_or_else(|p| p.into_inner())
5549            .remove(&(ctx.connection_id, frame.header.corr));
5550        let Some(task) = task else {
5551            return Ok(false);
5552        };
5553        task.abort();
5554        let end = Frame::build_with_version(
5555            response_version(frame),
5556            FrameType::StreamEnd,
5557            control_flags(),
5558            0,
5559            0,
5560            frame.header.corr,
5561            Vec::new(),
5562        )
5563        .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
5564        let sink = ctx.egress.clone();
5565        tokio::spawn(async move {
5566            let _ = sink.send(end).await;
5567        });
5568        Ok(true)
5569    }
5570
5571    async fn handle_activity_read(
5572        &self,
5573        ctx: &RouteCtx,
5574        frame: Frame,
5575    ) -> Result<Vec<Frame>, RouterError> {
5576        #[cfg(test)]
5577        {
5578            let pause = self
5579                .activity_request_pause
5580                .lock()
5581                .unwrap()
5582                .clone()
5583                .filter(|pause| pause.connection == ctx.connection_id);
5584            if let Some(pause) = pause {
5585                let _cancelled = ActivityRequestCancellation(pause.clone());
5586                pause.started.notify_one();
5587                std::future::pending::<()>().await;
5588            }
5589        }
5590        use subc_control::{
5591            ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
5592            ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
5593        };
5594        #[derive(Deserialize)]
5595        struct Identity {
5596            #[serde(default)]
5597            consumer_identity: Option<ConsumerIdentity>,
5598        }
5599        // A registration channel or claim-bound connection carries its module in
5600        // its binding, so no request field can name another principal, and its
5601        // subscription ends when the bound spawn's authority does.
5602        let mut bound_spawn = None;
5603        let legacy = match self.activity_reader(ctx.connection_id) {
5604            channel_operations::ActivityReader::Module { module_id, spawn } => {
5605                bound_spawn = Some(spawn);
5606                Some(Ok(Principal::Reserved { module_id }))
5607            }
5608            // Refused outright: falling back to `direct` would admit it.
5609            channel_operations::ActivityReader::Refused => {
5610                return Ok(vec![control_error_frame(
5611                    &frame,
5612                    ACTIVITY_READ_FORBIDDEN,
5613                    "helper, relay and retired connections cannot read activity",
5614                )?]);
5615            }
5616            channel_operations::ActivityReader::Legacy => None,
5617        };
5618        // A registered module connection already identifies its caller. Ignore
5619        // consumer_identity there so a module cannot borrow a mirror's identity.
5620        // Only separate consumer connections use the Direct/nonce trust boundary.
5621        let principal = match legacy {
5622            Some(principal) => principal,
5623            None => match self
5624                .forwarding
5625                .module_id_for_connection(ctx.connection_id)
5626                .map_err(RouterError::Forwarding)?
5627            {
5628                Some(module_id) => Ok(Principal::Reserved { module_id }),
5629                None => {
5630                    let identity: Identity = match serde_json::from_slice(&frame.body) {
5631                        Ok(identity) => identity,
5632                        Err(_) => {
5633                            return Ok(vec![control_error_frame(
5634                                &frame,
5635                                ACTIVITY_MALFORMED,
5636                                "malformed activity read",
5637                            )?])
5638                        }
5639                    };
5640                    self.route_open_principal(
5641                        ctx.connection_id,
5642                        &frame,
5643                        identity.consumer_identity,
5644                    )?
5645                }
5646            },
5647        };
5648        let admitted = matches!(principal, Ok(Principal::Direct))
5649            || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
5650        if !admitted {
5651            return Ok(vec![control_error_frame(
5652                &frame,
5653                ACTIVITY_READ_FORBIDDEN,
5654                "activity reads require an operator or trusted mirror principal",
5655            )?]);
5656        }
5657        let Some(ledger) = &self.activity else {
5658            return Ok(vec![control_error_frame(
5659                &frame,
5660                ACTIVITY_UNAVAILABLE,
5661                "activity ledger unavailable",
5662            )?]);
5663        };
5664        if serde_json::from_slice::<ControlOpProbe>(&frame.body)
5665            .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
5666        {
5667            let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
5668                Ok(request) => ledger.query(request).await,
5669                Err(_) => Err(ACTIVITY_MALFORMED),
5670            };
5671            return match result {
5672                Ok(reply) => Ok(vec![control_response_body_frame(
5673                    &frame,
5674                    &reply,
5675                    ops::ACTIVITY_QUERY,
5676                )?]),
5677                Err(code) => Ok(vec![control_error_frame(
5678                    &frame,
5679                    code,
5680                    "activity query refused",
5681                )?]),
5682            };
5683        }
5684        let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
5685            Ok(request) => request,
5686            Err(_) => {
5687                return Ok(vec![control_error_frame(
5688                    &frame,
5689                    ACTIVITY_MALFORMED,
5690                    "malformed activity subscription",
5691                )?])
5692            }
5693        };
5694        let mut subscription = match ledger.subscribe(request.since).await {
5695            Ok(subscription) => subscription,
5696            Err(code) => {
5697                return Ok(vec![control_error_frame(
5698                    &frame,
5699                    code,
5700                    "activity subscription refused",
5701                )?])
5702            }
5703        };
5704        let key = (ctx.connection_id, frame.header.corr);
5705        let sink = ctx.egress.clone();
5706        let mut streams = self
5707            .activity_streams
5708            .lock()
5709            .unwrap_or_else(|p| p.into_inner());
5710        streams.retain(|_, task| !task.is_finished());
5711        if streams.get(&key).is_some_and(|task| !task.is_finished()) {
5712            return Ok(vec![control_error_frame(
5713                &frame,
5714                "duplicate_subscription",
5715                "activity subscription correlation id already in use",
5716            )?]);
5717        }
5718        let stream = async move {
5719            loop {
5720                let entry = if let Some(entry) = subscription.replay.pop_front() {
5721                    Some(entry)
5722                } else {
5723                    subscription.receiver.recv().await
5724                };
5725                let Some(entry) = entry else {
5726                    break;
5727                };
5728                let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
5729                    return;
5730                };
5731                let Ok(next) = Frame::build_with_version(
5732                    response_version(&frame),
5733                    FrameType::StreamData,
5734                    control_flags(),
5735                    0,
5736                    0,
5737                    frame.header.corr,
5738                    body,
5739                ) else {
5740                    return;
5741                };
5742                if sink.send(next).await.is_err() {
5743                    return;
5744                }
5745            }
5746            // Dropping a full subscriber's sender drains its accepted entries
5747            // before the terminal error, preserving their order on the wire.
5748            if let Some(code) = subscription.terminal_code() {
5749                if let Ok(end) =
5750                    control_error_frame(&frame, code, "activity subscriber fell behind")
5751                {
5752                    let _ = sink.send(end).await;
5753                }
5754            }
5755        };
5756        // The spawn's retirement revokes this connection's authority. End the
5757        // subscription then, rather than when the connection's close is
5758        // eventually processed or the next entry fails to send.
5759        let task = tokio::spawn(async move {
5760            match bound_spawn {
5761                Some(spawn) => {
5762                    tokio::select! {
5763                        () = stream => {}
5764                        () = spawn.wait_closed() => {}
5765                    }
5766                }
5767                None => stream.await,
5768            }
5769        });
5770        streams.insert(key, task.abort_handle());
5771        Ok(Vec::new())
5772    }
5773
5774    async fn handle_activity_record(
5775        &self,
5776        ctx: &RouteCtx,
5777        frame: Frame,
5778    ) -> Result<Vec<Frame>, RouterError> {
5779        use subc_control::{
5780            ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
5781            ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
5782        };
5783        let result = async {
5784            let module = self
5785                .forwarding
5786                .module_id_for_connection(ctx.connection_id)
5787                .ok()
5788                .flatten()
5789                .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
5790            let request: ActivityRecordRequest =
5791                serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
5792            crate::activity::validate(&module, &request)?;
5793            let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
5794            let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
5795            let at_ms = std::time::SystemTime::now()
5796                .duration_since(std::time::UNIX_EPOCH)
5797                .unwrap_or_default()
5798                .as_millis() as u64;
5799            ledger.record(module, request, actor, at_ms).await
5800        }
5801        .await;
5802        match result {
5803            Ok(entry) => Ok(vec![control_response_body_frame(
5804                &frame,
5805                &ActivityRecordResponse { seq: entry.seq },
5806                "activity.record",
5807            )?]),
5808            Err(code) => Ok(vec![control_error_frame(
5809                &frame,
5810                code,
5811                "activity recording refused",
5812            )?]),
5813        }
5814    }
5815
5816    fn activity_actor(
5817        &self,
5818        connection: ConnectionId,
5819        module: &str,
5820        on: Option<&subc_control::ActivityOn>,
5821    ) -> Result<subc_control::ActivityActor, &'static str> {
5822        use crate::forwarding::{DataRoute, DataRouteState};
5823        use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
5824        let principal = |id: &str| Principal::Reserved {
5825            module_id: id.to_owned(),
5826        };
5827        let mut actor = ActivityActor {
5828            principal: principal(module),
5829            scope_owner: None,
5830            agent_id: None,
5831            run_id: None,
5832            flow_id: None,
5833            project_root: None,
5834            workdir: None,
5835            project_id: None,
5836        };
5837        let scope = match on {
5838            None => return Ok(actor),
5839            Some(ActivityOn::Scope(scope)) => {
5840                let selector = subc_protocol::scope::ScopeSelector::new(
5841                    principal(module),
5842                    scope.scope_ref.clone(),
5843                    Some(scope.epoch),
5844                );
5845                self.scopes
5846                    .read()
5847                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5848                    .attribute(&principal(module), module, &selector, true)
5849                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5850            }
5851            Some(ActivityOn::Route(route)) => {
5852                let DataRoute::Module(DataRouteState::Bound(binding)) = self
5853                    .forwarding
5854                    .lookup_data_route(connection, route.channel, route.epoch)
5855                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5856                else {
5857                    return Err(ACTIVITY_ACTOR_UNKNOWN);
5858                };
5859                if binding.module_id != module {
5860                    return Err(ACTIVITY_ACTOR_UNKNOWN);
5861                }
5862                actor.principal = binding.principal.clone();
5863                actor.project_root = binding
5864                    .project_root
5865                    .as_ref()
5866                    .map(|root| root.as_path().to_path_buf());
5867                let Some(bound) = &binding.scope else {
5868                    return Ok(actor);
5869                };
5870                let owner = principal(&bound.owner);
5871                let selector = subc_protocol::scope::ScopeSelector::new(
5872                    owner.clone(),
5873                    bound.scope_ref.clone(),
5874                    Some(bound.tag.scope_epoch),
5875                );
5876                let admitted = self
5877                    .scopes
5878                    .read()
5879                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
5880                    .attribute(&owner, module, &selector, true)
5881                    .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
5882                if admitted.tag != bound.tag {
5883                    return Err(ACTIVITY_ACTOR_UNKNOWN);
5884                }
5885                admitted
5886            }
5887        };
5888        actor.scope_owner = Some(scope.stamp.owner);
5889        actor.agent_id = scope.stamp.attributes.agent_id;
5890        actor.run_id = scope.stamp.attributes.run_id;
5891        actor.flow_id = scope.stamp.attributes.flow_id;
5892        actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
5893        actor.project_id = scope.stamp.project_id;
5894        Ok(actor)
5895    }
5896
5897    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5898        self.refresh_capability_requirements();
5899        // A bare connection count is ambiguous between many clients holding a
5900        // route each and one client accumulating hundreds, so publish the
5901        // concentration alongside it. Route state is best-effort here: a
5902        // diagnostic endpoint must still answer if the forwarding lock is
5903        // contended.
5904        let mut counters = self.counters.snapshot();
5905        let losses = self.resource_tracker.loss_diagnostics();
5906        if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
5907            if let Some(obj) = counters.as_object_mut() {
5908                obj.insert("agent_command_declaration_loss_by_module".into(), losses);
5909            }
5910        }
5911        if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
5912            obj.insert(
5913                "activity_unparseable_lines".into(),
5914                ledger.read_errors().into(),
5915            );
5916            obj.insert(
5917                "activity_lifecycle_append_failures".into(),
5918                ledger.lifecycle_failures().into(),
5919            );
5920            obj.insert(
5921                "activity_started_digest_skips".into(),
5922                ledger.lifecycle_digest_skips().into(),
5923            );
5924        }
5925        if let (Ok((connections_with_routes, max)), Some(obj)) = (
5926            self.forwarding.client_route_concentration(),
5927            counters.as_object_mut(),
5928        ) {
5929            obj.insert(
5930                "client_connections_with_routes".into(),
5931                connections_with_routes.into(),
5932            );
5933            obj.insert("max_routes_on_one_connection".into(), max.into());
5934        }
5935        // A module that is being fast-refused and a module that is fine look
5936        // identical from a client that retries and succeeds, so name the open
5937        // breakers here. This rides the existing free-form counters object
5938        // rather than a new wire field, so no sibling that deserializes
5939        // `ServerDescribe` has to be rebuilt to keep reading it.
5940        if let (Some(open_breakers), Some(obj)) = (
5941            self.route_bind_breakers.open_snapshot(),
5942            counters.as_object_mut(),
5943        ) {
5944            obj.insert("route_bind_breakers_open".into(), open_breakers);
5945        }
5946        // A daemon whose local endpoint could not be bound keeps serving TCP,
5947        // and the connection file names no endpoint. Name the reason here, in
5948        // the free-form counters object, so an operator reading daemon facts
5949        // sees it without searching the log, and no wire field changes.
5950        if let (Some(reason), Some(obj)) = (
5951            self.local_endpoint_unavailable.clone(),
5952            counters.as_object_mut(),
5953        ) {
5954            obj.insert("local_endpoint_unavailable".into(), reason.into());
5955        }
5956        let response = ClientControlResponse::ServerDescribe {
5957            protocol_ver: PROTOCOL_VERSION,
5958            subc_ops: subc_ops(),
5959            capabilities: self.describe_capabilities(),
5960            connected_clients: self.connected_clients.count(),
5961            counters: Some(counters),
5962            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
5963            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
5964            capability_requirements: self.capability_requirement_statuses(),
5965            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
5966            config_path: self
5967                .rescan
5968                .as_ref()
5969                .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
5970        };
5971        Ok(vec![control_response_body_frame(
5972            &frame,
5973            &response,
5974            "ClientControlResponse::ServerDescribe",
5975        )?])
5976    }
5977
5978    fn handle_catalog_list(
5979        &self,
5980        frame: Frame,
5981        module_id: Option<String>,
5982    ) -> Result<Vec<Frame>, RouterError> {
5983        let (generation, modules) = self
5984            .registry
5985            .list_modules_with_nonce_verification()
5986            .map_err(|err| {
5987                RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
5988            })?;
5989        let entries = modules
5990            .into_iter()
5991            .filter(|(registration, _)| {
5992                module_id
5993                    .as_deref()
5994                    .map(|wanted| registration.manifest.module_id == wanted)
5995                    .unwrap_or(true)
5996            })
5997            .map(|(registration, nonce_verified)| {
5998                let entry = self
5999                    .catalog_discovery_entry(&registration, module_id.is_some())
6000                    .map_err(|err| RouterError::backend(0, frame.header.corr, err))?;
6001                let not_ready = self.not_ready_reason(&registration);
6002                let mut entry = entry;
6003                entry.roles = registration.manifest.provides;
6004                entry.control_ops = registration.control_ops;
6005                Ok(entry
6006                    .with_ready(not_ready.is_none())
6007                    .with_display_name(registration.manifest.display_name)
6008                    .with_not_ready(not_ready)
6009                    .with_module_version(Some(registration.manifest.module_version))
6010                    .with_provenance(registration.manifest.provenance)
6011                    .with_capabilities(registration.manifest.capabilities)
6012                    .with_self_signals(registration.manifest.self_signals)
6013                    .with_events(registration.manifest.events)
6014                    .with_operation_delivery(registration.manifest.operation_delivery)
6015                    .with_operation_payload(registration.manifest.operation_payload)
6016                    .with_flow_operations(registration.manifest.flow_operations)
6017                    .with_operation_deferral(registration.manifest.operation_deferral)
6018                    .with_confirm_operations(registration.manifest.confirm_operations)
6019                    .with_rich_ui(registration.manifest.rich_ui)
6020                    .with_role_declarations(registration.manifest.role_declarations)
6021                    .with_nonce_verified(nonce_verified))
6022            })
6023            .collect::<Result<Vec<_>, RouterError>>()?;
6024        let response = ClientControlResponse::CatalogList {
6025            generation,
6026            modules: entries,
6027            subc_ops: subc_ops(),
6028        };
6029        Ok(vec![control_response_body_frame(
6030            &frame,
6031            &response,
6032            "ClientControlResponse::CatalogList",
6033        )?])
6034    }
6035
6036    /// The caller fields for a refusal that only has the raw request frame,
6037    /// such as the connection-level admission refusal sent before the request
6038    /// reaches [`Self::handle_route_open`].
6039    fn route_open_caller_from_frame(&self, ctx: &RouteCtx, frame: &Frame) -> RouteOpenCaller {
6040        let Ok(ClientControlRequest::RouteOpen {
6041            identity,
6042            consumer_identity,
6043            scope,
6044            ..
6045        }) = parse_client_control_request(&frame.body)
6046        else {
6047            return RouteOpenCaller::unknown();
6048        };
6049        let principal = self
6050            .attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref())
6051            .map(|attested| attested.principal);
6052        RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
6053    }
6054
6055    fn route_open_principal(
6056        &self,
6057        connection_id: ConnectionId,
6058        frame: &Frame,
6059        consumer_identity: Option<ConsumerIdentity>,
6060    ) -> Result<Result<Principal, Frame>, RouterError> {
6061        let attested = self
6062            .attest_route_open_consumer(connection_id, consumer_identity.as_ref())
6063            .map(|attested| attested.principal);
6064        self.route_open_principal_attested(frame, consumer_identity, attested)
6065    }
6066
6067    /// [`Self::route_open_principal`] with `attested` already taken by
6068    /// [`Self::attest_route_open_consumer`], so one open resolves its connection
6069    /// claim or launch nonce once.
6070    fn route_open_principal_attested(
6071        &self,
6072        frame: &Frame,
6073        consumer_identity: Option<ConsumerIdentity>,
6074        attested: Option<Principal>,
6075    ) -> Result<Result<Principal, Frame>, RouterError> {
6076        if let Some(principal) = attested {
6077            return Ok(Ok(principal));
6078        }
6079        Ok(Err(control_error_frame(
6080            frame,
6081            "bad_consumer_identity",
6082            consumer_identity.map_or_else(
6083                || "connection claim is no longer authorized for routes".to_string(),
6084                |identity| format!(
6085                    "consumer_identity for module_id '{}' conflicts with its connection claim or did not match a supervised launch nonce",
6086                    identity.module_id
6087                ),
6088            ),
6089        )?))
6090    }
6091
6092    /// Ordinary `route.open` refusals go through here; admission and breaker
6093    /// refusals log separately with their capacity or breaker state. The daemon can
6094    /// attest which code it sent: without the event, a client's "the daemon
6095    /// refused me" and the daemon's own view could only be reconciled by
6096    /// argument. Malformed input (`invalid_project_root`) does not come here;
6097    /// rejecting a request that was never a valid open is not a refusal of one.
6098    #[allow(clippy::too_many_arguments)]
6099    fn route_open_refusal_frame(
6100        &self,
6101        ctx: &RouteCtx,
6102        caller: &RouteOpenCaller,
6103        frame: &Frame,
6104        module_id: &str,
6105        reason: &'static str,
6106        code: &'static str,
6107        message: impl Into<String>,
6108    ) -> Result<Frame, RouterError> {
6109        self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
6110        let detail = if matches!(
6111            reason,
6112            "registration_not_active"
6113                | "supervisor_not_live"
6114                | "no_forwarding_connection"
6115                | "reloading"
6116        ) {
6117            // A planned restart, reload or swap is already replacing the
6118            // process; it does not need a slot in the crash-restart budget.
6119            let expected_back = (reason == "reloading").then_some(true);
6120            self.route_open_recovery_detail(module_id, frame.header.corr, None, expected_back)?
6121        } else {
6122            None
6123        };
6124        control_error_body_frame(
6125            frame,
6126            ErrorBody {
6127                code: code.to_string(),
6128                message: message.into(),
6129                detail,
6130            },
6131        )
6132    }
6133
6134    fn route_open_recovery_detail(
6135        &self,
6136        module_id: &str,
6137        corr: u64,
6138        detail: Option<serde_json::Value>,
6139        expected_back: Option<bool>,
6140    ) -> Result<Option<serde_json::Value>, RouterError> {
6141        let Some((status, _)) = self.supervisor_status(module_id, corr)? else {
6142            return Ok(detail);
6143        };
6144        let mut detail = detail.unwrap_or_else(|| serde_json::json!({}));
6145        detail["state"] = serde_json::json!(status.state.to_string());
6146        if let Some(expected_back) = expected_back {
6147            detail["expected_back"] = serde_json::json!(expected_back);
6148        } else if let Some(module) = self.supervisor.get(module_id) {
6149            match module.will_recover_after_connection_loss() {
6150                Ok(expected_back) => detail["expected_back"] = serde_json::json!(expected_back),
6151                Err(err) => warn!(
6152                    %module_id,
6153                    error = %err,
6154                    "failed to read route.open recovery verdict; omitting expected_back"
6155                ),
6156            }
6157        }
6158        Ok(Some(detail))
6159    }
6160
6161    /// Answer a waiting `route.open` whose relayed bind was refused.
6162    ///
6163    /// `daemon_authored` is provenance, not a code match: the daemon
6164    /// synthesized the refusal itself (a drain, a late reply, a scope that
6165    /// ended mid-bind) or the module refused the bind with its own body. A
6166    /// daemon refusal carries the daemon-origin bit; a module's own
6167    /// rejection is relayed with the bit clear, so a module that refuses
6168    /// with `module_reloading` still cannot mint a daemon refusal.
6169    #[allow(clippy::too_many_arguments)]
6170    fn route_bind_rejection(
6171        &self,
6172        ctx: &RouteCtx,
6173        caller: &RouteOpenCaller,
6174        frame: &Frame,
6175        target_module_id: &str,
6176        body: ErrorBody,
6177        daemon_authored: bool,
6178    ) -> Result<Vec<Frame>, RouterError> {
6179        // The daemon's own commit re-check refused the bind because the
6180        // scope ended or changed after admission. The module accepted;
6181        // counting it as a module rejection would blame the module. This
6182        // branch only runs for daemon-synthesized bodies: a module that
6183        // refuses with a scope code is answered as a module rejection below.
6184        if daemon_authored {
6185            let scope_code = match body.code.as_str() {
6186                error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
6187                error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
6188                _ => None,
6189            };
6190            if let Some(code) = scope_code {
6191                self.observe_route_open_refusal(
6192                    ctx,
6193                    caller,
6194                    target_module_id,
6195                    "scope_changed_before_commit",
6196                    code,
6197                );
6198                return Ok(vec![control_error_body_frame(frame, body)?]);
6199            }
6200        }
6201        self.counters
6202            .increment_route_open_refused("module_rejected");
6203        info!(
6204            target: "control",
6205            code = "module_rejected",
6206            module_code = ?body.code,
6207            module_id = ?target_module_id,
6208            connection_id = ctx.connection_id.get(),
6209            principal = %caller.principal,
6210            scope = %caller.scope,
6211            harness = %caller.harness,
6212            "route.open refused"
6213        );
6214        let refusal = if daemon_authored {
6215            control_error_body_frame(frame, body)?
6216        } else {
6217            control_relayed_error_body_frame(frame, body)?
6218        };
6219        Ok(vec![refusal])
6220    }
6221
6222    /// Refuse a `route.open` because the target module's bind-relay breaker is
6223    /// open, without attempting the relay.
6224    ///
6225    /// The wire code is `module_timeout`, which is the truth (the module has
6226    /// not been answering binds) and which both SDKs already classify as
6227    /// retryable with capped backoff. Reusing it is what keeps this change out
6228    /// of both SDKs; the daemon-side distinction lives in the counter key
6229    /// instead.
6230    ///
6231    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
6232    /// While a breaker is open this fires on every open to that module, and the
6233    /// stall written up in `docs/designs/route-open-head-of-line.md` already
6234    /// produced 261 lines about a single module inside 3000 lines of daemon
6235    /// log. The rare transitions are logged at warn/info instead and the volume
6236    /// is carried by the counter, so the evidence survives without the flood.
6237    /// The debug line keeps a per-refusal record reachable for whoever turns
6238    /// the level up.
6239    #[allow(clippy::too_many_arguments)]
6240    fn route_open_breaker_refusal_frame(
6241        &self,
6242        ctx: &RouteCtx,
6243        caller: &RouteOpenCaller,
6244        frame: &Frame,
6245        module_id: &str,
6246        consecutive_timeouts: u32,
6247        retry_in: Duration,
6248        probe_in_flight: bool,
6249    ) -> Result<Frame, RouterError> {
6250        self.counters
6251            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
6252        debug!(
6253            target: "control",
6254            code = "module_timeout",
6255            module_id = ?module_id,
6256            connection_id = ctx.connection_id.get(),
6257            principal = %caller.principal,
6258            scope = %caller.scope,
6259            harness = %caller.harness,
6260            consecutive_timeouts,
6261            retry_in_ms = retry_in.as_millis() as u64,
6262            probe_in_flight,
6263            "route.open refused by open bind-relay breaker"
6264        );
6265        // Say what a caller can act on. An open bind-relay breaker means the
6266        // module timed out accepting several new routes in a row. The module
6267        // is still running and its established routes keep working; only new
6268        // route.open requests are refused until the cooldown ends and one
6269        // test route (the probe) gets through. A message that only counts
6270        // failed relays reads as "the module is down" to a worker that sees it.
6271        let detail = if probe_in_flight {
6272            "one test route is already being tried; retry once it settles".to_string()
6273        } else {
6274            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
6275        };
6276        control_error_frame(
6277            frame,
6278            "module_timeout",
6279            format!(
6280                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
6281                 timed out in a row); {detail}; its established routes are unaffected"
6282            ),
6283        )
6284    }
6285
6286    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
6287    /// requester's bytes (an unknown target is whatever the client sent) and
6288    /// is Debug-formatted so control characters land in the log escaped
6289    /// rather than as terminal sequences for whoever tails it.
6290    ///
6291    /// `reason` names the check that refused, because one wire code has
6292    /// several senders: after a module registers, `target_unavailable` can
6293    /// come from a missing role, an inactive registration, a supervisor that
6294    /// has not marked the process live, a missing forwarding connection, or a
6295    /// failed relay, and a log that records only the code cannot say which of
6296    /// them fired. It is a static, daemon-chosen label per branch, so it is
6297    /// safe to print plainly and stays a closed set.
6298    fn observe_route_open_refusal(
6299        &self,
6300        ctx: &RouteCtx,
6301        caller: &RouteOpenCaller,
6302        module_id: &str,
6303        reason: &'static str,
6304        code: &'static str,
6305    ) {
6306        self.counters.increment_route_open_refused(code);
6307        info!(
6308            target: "control",
6309            code,
6310            reason,
6311            module_id = ?module_id,
6312            connection_id = ctx.connection_id.get(),
6313            principal = %caller.principal,
6314            scope = %caller.scope,
6315            harness = %caller.harness,
6316            "route.open refused"
6317        );
6318        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
6319            self.route_outages.record_not_serving(module_id, reason);
6320        }
6321    }
6322
6323    /// Record an ACCEPTED route.open.
6324    ///
6325    /// Refusals have been logged and counted since the attestation work; accepts
6326    /// were invisible, so the daemon knew every principal it stamped and wrote
6327    /// none of them down. The party that attests the identity was the only party
6328    /// not recording it, which left a credential vault unable to name the sender
6329    /// of a call that reached it (claustrum #43) and left the launch-nonce
6330    /// concurrency question unanswerable from the outside.
6331    ///
6332    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
6333    /// `code`/`module_id`/`connection_id` returns both directions of the same
6334    /// decision rather than two shapes a reader has to join by hand.
6335    ///
6336    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
6337    /// and the difference carries information rather than being an
6338    /// inconsistency. This line is only reachable after a successful bind to a
6339    /// REGISTERED module, so the value has already passed HELLO validation
6340    /// including the path-hazard refusal and cannot contain control bytes. A
6341    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
6342    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
6343    ///
6344    /// Bare is also what every other daemon line already emits (`module
6345    /// registered`, `configured module supervised`). Shipping `?module_id` here
6346    /// made this instrument the only one in the file whose ids did not answer
6347    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
6348    /// line whose whole purpose is being grepped beside its sibling.
6349    ///
6350    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
6351    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
6352    /// forwards every field type through it and a bare `&str` and a `?`-escaped
6353    /// one are recorded identically. A test written against that harness passes
6354    /// either way -- I wrote one, measured it, and deleted it rather than ship a
6355    /// green assertion that cannot fail. The same limit applies to the escaping
6356    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
6357    /// it reads as a guard on the Debug escaping and cannot detect its removal.
6358    /// Fencing either needs the real formatter, not the capture layer.
6359    ///
6360    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
6361    /// unix-socket options and subc is loopback TCP, so there is no peer identity
6362    /// to record. The ephemeral port would decay within minutes and answer only a
6363    /// live question. The identity question is instead answered by counting
6364    /// distinct live connections presenting one module's `consumer_identity` --
6365    /// "is anyone else holding this secret" rather than "is this the right
6366    /// process".
6367    fn observe_route_open_accept(
6368        &self,
6369        ctx: &RouteCtx,
6370        module_id: &str,
6371        principal: &str,
6372        scope: &str,
6373    ) {
6374        self.route_outages.record_accepted(module_id);
6375        self.counters.increment_route_open_accepted(principal);
6376        info!(
6377            target: "control",
6378            principal,
6379            module_id,
6380            connection_id = ctx.connection_id.get(),
6381            scope = %scope,
6382            "route.open accepted"
6383        );
6384    }
6385
6386    fn supervised_absent_route_open_refusal_frame(
6387        &self,
6388        ctx: &RouteCtx,
6389        caller: &RouteOpenCaller,
6390        frame: &Frame,
6391        module_id: &str,
6392        code: &'static str,
6393        status: &crate::supervise::ModuleStatus,
6394    ) -> Result<Frame, RouterError> {
6395        self.counters.increment_route_open_refused(code);
6396        info!(
6397            target: "control",
6398            code,
6399            reason = "supervised_not_registered",
6400            module_id = ?module_id,
6401            connection_id = ctx.connection_id.get(),
6402            principal = %caller.principal,
6403            scope = %caller.scope,
6404            harness = %caller.harness,
6405            state = %status.state,
6406            enabled = status.enabled,
6407            live = status.live,
6408            "route.open refused"
6409        );
6410        // A supervised module whose process has not registered is not
6411        // serving, whatever the reason; the supervisor knows this id, so it is
6412        // safe to track.
6413        self.route_outages
6414            .record_not_serving(module_id, "supervised_not_registered");
6415        let message = if let Some(pid) = status.waiting_for_previous_pid {
6416            status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
6417                format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
6418            })
6419        } else {
6420            format!(
6421                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
6422                status.state, status.enabled, status.live
6423            )
6424        };
6425        // `module_warming` here means the process is starting, running or
6426        // being restarted and has not registered yet. Nothing has to be
6427        // restarted for it to serve, so the crash budget does not decide it.
6428        let expected_back = (code == error_codes::MODULE_WARMING).then_some(true);
6429        control_error_body_frame(
6430            frame,
6431            ErrorBody {
6432                code: code.to_string(),
6433                message,
6434                detail: self.route_open_recovery_detail(
6435                    module_id,
6436                    frame.header.corr,
6437                    None,
6438                    expected_back,
6439                )?,
6440            },
6441        )
6442    }
6443
6444    async fn handle_route_open(
6445        &self,
6446        ctx: &RouteCtx,
6447        frame: Frame,
6448        request: RouteOpenRequest,
6449        timings: &mut RouteOpenTimings,
6450    ) -> Result<Vec<Frame>, RouterError> {
6451        let RouteOpenRequest {
6452            target,
6453            mut identity,
6454            consumer_identity,
6455            consumer_capabilities,
6456            role_versions,
6457            busy_refusals,
6458            admission_facts,
6459            scope,
6460            accepts_close_reasons,
6461        } = request;
6462        let target_module_id = target_module_id(&target).to_string();
6463        timings.module_id.clone_from(&target_module_id);
6464        if self
6465            .registry
6466            .get_module_by_connection(ctx.connection_id)
6467            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6468            .is_some()
6469        {
6470            return Ok(vec![control_error_frame(
6471                &frame,
6472                "invalid_request",
6473                "module connections cannot open client routes",
6474            )?]);
6475        }
6476        debug!(
6477            connection_id = ctx.connection_id.get(),
6478            corr = frame.header.corr,
6479            module_id = %target_module_id,
6480            "handling route.open"
6481        );
6482
6483        // The caller's attested principal, requested scope and harness, for
6484        // every `route.open refused` line below. The connection's claim
6485        // binding or launch nonce is checked here, once, and the answer is
6486        // reused where the principal is enforced further down, so the order
6487        // of refusals is unchanged.
6488        let attested =
6489            self.attest_route_open_consumer(ctx.connection_id, consumer_identity.as_ref());
6490        let mut caller = RouteOpenCaller::new(
6491            attested.as_ref().map(|attested| &attested.principal),
6492            scope.as_ref(),
6493            &identity,
6494        );
6495
6496        // A malformed declaration is refused first, before anything about the
6497        // target is looked up: the same body would be refused against any
6498        // module, so the caller learns nothing by retrying or waiting. An empty
6499        // map declares nothing and travels as no field at all, so a provider
6500        // only ever sees a missing field or a non-empty one.
6501        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
6502        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
6503            self.observe_route_open_refusal(
6504                ctx,
6505                &caller,
6506                &target_module_id,
6507                "invalid_role_versions",
6508                error_codes::INVALID_REQUEST,
6509            );
6510            return Ok(vec![control_error_body_frame(
6511                &frame,
6512                ErrorBody {
6513                    code: error_codes::INVALID_REQUEST.to_string(),
6514                    message: error.to_string(),
6515                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
6516                },
6517            )?]);
6518        }
6519
6520        if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
6521            if status.waiting_for_previous_pid.is_some() {
6522                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6523                    ctx,
6524                    &caller,
6525                    &frame,
6526                    &target_module_id,
6527                    "module_warming",
6528                    &status,
6529                )?]);
6530            }
6531        }
6532
6533        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
6534        // opposite. Below, a caller learns whether a module is unregistered,
6535        // supervised-but-down (with state/enabled/live), or registered without the
6536        // requested role. Elsewhere that is an enumeration leak: a probe learning
6537        // the shape of a fleet it cannot otherwise see.
6538        //
6539        // It is not one here, and the reason is the ACCESS MODEL rather than
6540        // anything about these errors. Reaching route.open requires the
6541        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
6542        // connection file, so any caller who completes it already runs as this
6543        // user -- and can read subc.jsonc for the module list and `ck module
6544        // status` for live state. The reply discloses nothing the caller cannot
6545        // read more easily from disk, while the precision is load-bearing:
6546        // `unknown_module` is retryable and a missing role is not.
6547        //
6548        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
6549        // remote transport, a sandboxed caller, a shared-host mode -- THAT
6550        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
6551        let admission_registry = self
6552            .registry
6553            .route_admission_snapshot()
6554            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6555        let Some(registration) =
6556            crate::scopes::admission::registered_target(&admission_registry, &target_module_id)
6557        else {
6558            if let Some((status, warming)) =
6559                self.supervisor_status(&target_module_id, frame.header.corr)?
6560            {
6561                // BEFORE the two availability codes below, because for a module
6562                // that speaks no subc wire both of them are false comfort: they
6563                // say "not right now" and are retried, and this module will
6564                // never register no matter how long the caller waits. The
6565                // absence here is the declaration being honoured, not a module
6566                // that is late.
6567                if status.protocol == ModuleProtocol::None {
6568                    return Ok(vec![self.route_open_refusal_frame(
6569                        ctx,
6570                        &caller,
6571                        &frame,
6572                        &target_module_id,
6573                        "protocol_none",
6574                        error_codes::MODULE_NO_PROTOCOL,
6575                        format!(
6576                            "module_id '{target_module_id}' is declared protocol: none; \
6577                             it speaks no subc wire and serves no routes"
6578                        ),
6579                    )?]);
6580                }
6581                let code = if warming {
6582                    "module_warming"
6583                } else {
6584                    "target_unavailable"
6585                };
6586                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
6587                    ctx,
6588                    &caller,
6589                    &frame,
6590                    &target_module_id,
6591                    code,
6592                    &status,
6593                )?]);
6594            }
6595            if let Some(removed_ago_ms) =
6596                self.supervisor.removal_tombstone_age_ms(&target_module_id)
6597            {
6598                return Ok(vec![self.route_open_refusal_frame(
6599                    ctx,
6600                    &caller,
6601                    &frame,
6602                    &target_module_id,
6603                    "removed",
6604                    error_codes::MODULE_REMOVED,
6605                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
6606                )?]);
6607            }
6608            return Ok(vec![self.route_open_refusal_frame(
6609                ctx,
6610                &caller,
6611                &frame,
6612                &target_module_id,
6613                "not_registered",
6614                error_codes::UNKNOWN_MODULE,
6615                format!("module_id '{target_module_id}' is not registered"),
6616            )?]);
6617        };
6618
6619        // Best-effort only: registry readiness and forwarding reservation use
6620        // different locks, so a module can flip readiness between this read and
6621        // the relay. Modules must still tolerate an `on_bind` while not ready.
6622        if crate::scopes::admission::target_registration(&admission_registry, &target_module_id)
6623            .is_err()
6624        {
6625            self.counters
6626                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
6627            info!(
6628                target: "control",
6629                code = error_codes::MODULE_WARMING,
6630                module_id = ?target_module_id,
6631                connection_id = ctx.connection_id.get(),
6632                principal = %caller.principal,
6633                scope = %caller.scope,
6634                harness = %caller.harness,
6635                reason = "declared_not_ready",
6636                "route.open refused"
6637            );
6638            // The module is registered but says it cannot take work, which is
6639            // an outage from the caller's side even though its process is up.
6640            self.route_outages
6641                .record_not_serving(&target_module_id, "declared_not_ready");
6642            // The registered process can declare itself ready without a
6643            // restart, so an exhausted crash budget does not prevent serving.
6644            return Ok(vec![control_error_body_frame(
6645                &frame,
6646                ErrorBody {
6647                    code: error_codes::MODULE_WARMING.to_string(),
6648                    message: format!(
6649                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
6650                    ),
6651                    detail: self.route_open_recovery_detail(
6652                        &target_module_id,
6653                        frame.header.corr,
6654                        Some(serde_json::json!({ "reason": "declared_not_ready" })),
6655                        Some(true),
6656                    )?,
6657                },
6658            )?]);
6659        }
6660
6661        // Effective readiness, second half: a module that declares a capability
6662        // `need: required` is not routable while that capability has no
6663        // registered provider. It is enforced HERE, as a retryable routing
6664        // refusal, and deliberately not as spawn ordering or a boot block. The
6665        // module is still started and registered and can make its own calls;
6666        // spawn ordering is a promise that cannot be kept once a provider
6667        // crashes at runtime, and refusing to boot would stop the whole
6668        // machine, including the tools needed to fix its configuration.
6669        //
6670        // "Provided" is the evaluator's verdict, which counts a provider as
6671        // soon as it has REGISTERED, not once it is ready. Two modules that
6672        // require each other's capabilities are therefore both routable once
6673        // both register; counting readiness instead would deadlock them.
6674        //
6675        // Only new opens are refused. Routes already bound when a provider
6676        // goes away stay bound: nothing here tears them down, and the module
6677        // answers them as it can. Like the readiness read above this is
6678        // best-effort against a provider registering or leaving concurrently.
6679        if let Some(capability) = self.first_unprovided_required_capability(registration) {
6680            self.counters
6681                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
6682            info!(
6683                target: "control",
6684                code = error_codes::MODULE_WARMING,
6685                module_id = ?target_module_id,
6686                connection_id = ctx.connection_id.get(),
6687                principal = %caller.principal,
6688                scope = %caller.scope,
6689                harness = %caller.harness,
6690                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6691                capability = %capability,
6692                "route.open refused"
6693            );
6694            return Ok(vec![control_error_body_frame(
6695                &frame,
6696                ErrorBody {
6697                    code: error_codes::MODULE_WARMING.to_string(),
6698                    message: format!(
6699                        "module_id '{target_module_id}' requires capability '{capability}', \
6700                         which no registered module provides; retry"
6701                    ),
6702                    detail: Some(serde_json::json!({
6703                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
6704                        "capability": capability,
6705                    })),
6706                },
6707            )?]);
6708        }
6709
6710        if !target_has_required_role(&target, &registration.manifest.provides) {
6711            return Ok(vec![self.route_open_refusal_frame(
6712                ctx,
6713                &caller,
6714                &frame,
6715                &target_module_id,
6716                "role_not_provided",
6717                "target_unavailable",
6718                format!("module_id '{target_module_id}' does not provide the requested target"),
6719            )?]);
6720        }
6721
6722        if registration.state != ChannelState::Active {
6723            return Ok(vec![self.route_open_refusal_frame(
6724                ctx,
6725                &caller,
6726                &frame,
6727                &target_module_id,
6728                "registration_not_active",
6729                "target_unavailable",
6730                format!("module_id '{target_module_id}' is not active"),
6731            )?]);
6732        }
6733
6734        if self
6735            .forwarding
6736            .module_is_draining(&target_module_id)
6737            .map_err(RouterError::Forwarding)?
6738        {
6739            return Ok(vec![self.route_open_refusal_frame(
6740                ctx,
6741                &caller,
6742                &frame,
6743                &target_module_id,
6744                "reloading",
6745                "module_reloading",
6746                format!("module_id '{target_module_id}' is reloading"),
6747            )?]);
6748        }
6749
6750        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
6751            process_liveness.process_live(&target_module_id) == Some(false)
6752        }) {
6753            // A module the supervisor is restarting or reloading can still hold
6754            // a registration: the old process before its connection closes, or
6755            // a new one that registered while the supervisor was draining. The
6756            // forwarding table does not see that as draining, but the consumer
6757            // should still be told to retry soon, exactly as for the drain
6758            // above, rather than that the target is unavailable.
6759            if process_liveness.process_replacing(&target_module_id) {
6760                return Ok(vec![self.route_open_refusal_frame(
6761                    ctx,
6762                    &caller,
6763                    &frame,
6764                    &target_module_id,
6765                    "reloading",
6766                    "module_reloading",
6767                    format!("module_id '{target_module_id}' is reloading"),
6768                )?]);
6769            }
6770            return Ok(vec![self.route_open_refusal_frame(
6771                ctx,
6772                &caller,
6773                &frame,
6774                &target_module_id,
6775                "supervisor_not_live",
6776                "target_unavailable",
6777                format!("module_id '{target_module_id}' is not live"),
6778            )?]);
6779        }
6780
6781        if !self
6782            .forwarding
6783            .has_live_module_connection(&target_module_id)
6784            .map_err(RouterError::Forwarding)?
6785        {
6786            return Ok(vec![self.route_open_refusal_frame(
6787                ctx,
6788                &caller,
6789                &frame,
6790                &target_module_id,
6791                "no_forwarding_connection",
6792                "target_unavailable",
6793                format!("module_id '{target_module_id}' has no live forwarding connection"),
6794            )?]);
6795        }
6796
6797        if let Some(error) =
6798            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
6799        {
6800            self.observe_route_open_refusal(
6801                ctx,
6802                &caller,
6803                &target_module_id,
6804                "op_not_allowed",
6805                "op_not_allowed",
6806            );
6807            return Ok(vec![error]);
6808        }
6809
6810        let principal = match (&attested, consumer_identity.is_some()) {
6811            // A claim-bound connection whose launch ended attests nothing,
6812            // and it must never fall back to opening as `direct`.
6813            (None, false) => {
6814                self.observe_route_open_refusal(
6815                    ctx,
6816                    &caller,
6817                    &target_module_id,
6818                    "registration_channel_closed",
6819                    crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6820                );
6821                return Ok(vec![control_error_frame(
6822                    &frame,
6823                    crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6824                    "the claim binding's launch has ended",
6825                )?]);
6826            }
6827            _ => match self.route_open_principal_attested(
6828                &frame,
6829                consumer_identity,
6830                attested.as_ref().map(|attested| attested.principal.clone()),
6831            )? {
6832                Ok(principal) => principal,
6833                Err(error) => {
6834                    self.observe_route_open_refusal(
6835                        ctx,
6836                        &caller,
6837                        &target_module_id,
6838                        "bad_consumer_identity",
6839                        "bad_consumer_identity",
6840                    );
6841                    return Ok(vec![error]);
6842                }
6843            },
6844        };
6845        let attested = attested.expect("an attested principal was resolved above");
6846        timings.principal = principal_label(&principal);
6847
6848        // The opener's spawn authority, read once here and carried with the
6849        // relay; nothing below reads its record again.
6850        let Ok(mut caller_capture) = attested.capture() else {
6851            self.observe_route_open_refusal(
6852                ctx,
6853                &caller,
6854                &target_module_id,
6855                "registration_channel_closed",
6856                crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6857            );
6858            return Ok(vec![control_error_frame(
6859                &frame,
6860                crate::supervise::REGISTRATION_CHANNEL_CLOSED,
6861                "the opener's launch has ended",
6862            )?]);
6863        };
6864        // The caller-side checks below read the opener's own spawn: its
6865        // module from its record and its manifest from its own registration,
6866        // never another launch's active registration under the same id.
6867        let caller_module_id = match &attested.spawn {
6868            Some((record, _)) => Some(record.module_id().to_owned()),
6869            None => match &principal {
6870                Principal::Reserved { module_id } => Some(module_id.clone()),
6871                _ => None,
6872            },
6873        };
6874
6875        // Only Callosum can attest a production phone build. Consume the claim
6876        // here so it can be kept on the route without reaching any module.
6877        let build_class = identity.build_class.take().filter(|build_class| {
6878            if caller_module_id.as_deref() == Some("callosum") {
6879                build_class == subc_protocol::BUILD_CLASS_PRODUCTION
6880            } else {
6881                if self
6882                    .build_class_warned
6883                    .lock()
6884                    .unwrap_or_else(|p| p.into_inner())
6885                    .insert(ctx.connection_id)
6886                {
6887                    warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
6888                        "ignoring build_class from a non-Callosum opener");
6889                }
6890                false
6891            }
6892        });
6893
6894        // This is attested, control-plane policy for supervised module origins.
6895        // Keep it before route reservation and out of the opaque forwarding hot
6896        // path: data frames must never acquire a per-frame capability check.
6897        if let Some(opening_registration) = self.caller_registration(&attested, &admission_registry)
6898        {
6899            let opening_module_id = opening_registration.manifest.module_id.as_str();
6900            if let Some(capability) =
6901                denied_capability(&opening_registration.manifest, &registration.manifest)
6902            {
6903                warn!(
6904                    opening_module_id,
6905                    target_module_id,
6906                    capability,
6907                    "refusing route.open because an attested capability deny edge matches"
6908                );
6909                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6910                    &frame,
6911                    &target_module_id,
6912                    "capability_deny_edge",
6913                    "capability_forbidden",
6914                    format!(
6915                        "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
6916                    ),
6917                )?]);
6918            }
6919        }
6920
6921        if admission_facts.is_some() {
6922            let carrier_matches = caller_module_id.is_some()
6923                && self.admission_facts_carrier_module_id.as_deref() == caller_module_id.as_deref();
6924            if !carrier_matches {
6925                return Ok(vec![self.route_open_refusal_frame(
6926                    ctx,
6927                    &caller,
6928                    &frame,
6929                    &target_module_id,
6930                    "admission_facts_carrier_not_permitted",
6931                    "admission_facts_not_permitted",
6932                    "admission facts may only be carried by the configured reserved module",
6933                )?]);
6934            }
6935
6936            let target_allowed = self
6937                .admission_facts_targets
6938                .as_ref()
6939                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
6940            if !target_allowed {
6941                return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
6942                    &frame,
6943                    &target_module_id,
6944                    "admission_facts_target_not_listed",
6945                    "admission_facts_target_not_allowed",
6946                    format!(
6947                        "admission facts are not permitted for target module_id '{target_module_id}'"
6948                    ),
6949                )?]);
6950            }
6951
6952            // Keep the value opaque to subc. The downstream admission validator owns
6953            // schema and semantic checks; this daemon only enforces carrier authority
6954            // and the configured destination allowlist.
6955        }
6956
6957        // Scope admission, on the attested principal above and never on the
6958        // request body. The tag read here travels with the pending bind and is
6959        // compared with the published one at commit, so a sync between here
6960        // and the module's ack refuses the open instead of binding a stamp
6961        // that is no longer true.
6962        let (bound_scope, mut scope_stamp, stamp_required) = match &scope {
6963            None => (None, None, false),
6964            Some(selector) => {
6965                let owner_configured = match &selector.owner {
6966                    // A reserved owner counts as configured from before its
6967                    // process is spawned (see `SupervisorHandle::is_configured`).
6968                    // So an owner that has not synced its scopes yet is refused
6969                    // as retryable (`scope_not_synced`), not as one that will
6970                    // never sync.
6971                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
6972                    _ => false,
6973                };
6974                let mut scopes = self
6975                    .scopes
6976                    .write()
6977                    .unwrap_or_else(|poisoned| poisoned.into_inner());
6978                let admitted = crate::scopes::admission::admit_route(
6979                    &scopes,
6980                    &admission_registry,
6981                    &principal,
6982                    &target_module_id,
6983                    Some(selector),
6984                    owner_configured,
6985                    crate::scopes::admission::AdmissionMode::Open,
6986                )
6987                .map(|route| {
6988                    let admission = route.scope.expect("scoped admission");
6989                    let bound = scopes.reserve_bind(
6990                        &admission,
6991                        &principal,
6992                        ctx.connection_id,
6993                        selector.hold_presence,
6994                    );
6995                    (admission, bound)
6996                });
6997                match admitted {
6998                    Ok((admission, bound)) => {
6999                        (Some(bound), Some(admission.stamp), admission.stamp_required)
7000                    }
7001                    Err(refusal) => {
7002                        return Ok(vec![self.route_open_refusal_frame(
7003                            ctx,
7004                            &caller,
7005                            &frame,
7006                            &target_module_id,
7007                            refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
7008                            refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
7009                            refusal.message,
7010                        )?]);
7011                    }
7012                }
7013            }
7014        };
7015        // From here on a refusal names the scope by the stamp admission
7016        // issued, exactly as the `route.open accepted` line does.
7017        if let Some(stamp) = &scope_stamp {
7018            caller.scope = route_open_scope_label(stamp);
7019        }
7020        // Decided from the admitted stamp, before any field of it is stripped
7021        // for a target that lacks the capability to read it.
7022        if scope_stamp.as_ref().is_some_and(scope_acts_for_another) {
7023            caller_capture = caller_capture.map(crate::spawn_authority::CallerCapture::relayed);
7024        }
7025
7026        // Allow resolvable missing-tail paths so a vanished directory does not
7027        // prevent reopening an existing session for cleanup. Binding does not
7028        // authorize creation of durable provider state.
7029        //
7030        // Resolve real paths rather than lexical cleanup: canonicalize the
7031        // longest existing ancestor and append the missing tail, retaining the
7032        // path's real existing ancestry even when a leaf is missing.
7033        timings.enter(RouteOpenPhase::ProjectRoot);
7034        let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
7035            Ok(project) => project,
7036            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
7037        };
7038        timings.enter(RouteOpenPhase::Admission);
7039        let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
7040            Ok(root) => root,
7041            Err(err) => {
7042                return Ok(vec![control_error_frame(
7043                    &frame,
7044                    "invalid_project_root",
7045                    err.to_string(),
7046                )?])
7047            }
7048        };
7049        // Compute compatibility project fields before stripping optional stamp
7050        // members. Preserve the canonical opener root and its project ID,
7051        // including absence. For rootless openers, use stamp workdir or `/`, and
7052        // opener ID, else stamp ID, else no ID. Emit these identity fields only
7053        // to targets without bind-no-project/v1.
7054        let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
7055
7056        // Last gate before any relay work, and deliberately after the cheap
7057        // registry and availability checks above: those name a more precise
7058        // condition (unknown, removed, reloading) and a caller is better served
7059        // by the precise code than by this one.
7060        //
7061        // Everything below this point costs an egress permit, a reserved handle
7062        // pair and, if the module does not answer, the whole relay budget. The
7063        // reader no longer waits for that budget, so cap each target explicitly;
7064        // serial dispatch used to provide the accidental cap of one relay per
7065        // connection. Admission is a mutex-protected count and never waits.
7066        let _concurrency_guard = match self
7067            .route_bind_concurrency
7068            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
7069        {
7070            Ok(guard) => guard,
7071            Err(in_flight) => {
7072                return Ok(vec![self.route_open_target_capacity_refusal(
7073                    ctx,
7074                    &caller,
7075                    &frame,
7076                    &target_module_id,
7077                    in_flight,
7078                )?]);
7079            }
7080        };
7081
7082        // A module that has already burned the whole budget `threshold` times
7083        // in a row does not get to charge it again until a probe says it recovered.
7084        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
7085            RouteBindAdmission::Admitted { guard, probe } => {
7086                if probe {
7087                    info!(
7088                        module_id = %target_module_id,
7089                        connection_id = ctx.connection_id.get(),
7090                        "route.bind breaker half-open: admitting one probe"
7091                    );
7092                }
7093                guard
7094            }
7095            RouteBindAdmission::Refused {
7096                consecutive_timeouts,
7097                retry_in,
7098                probe_in_flight,
7099            } => {
7100                return Ok(vec![self.route_open_breaker_refusal_frame(
7101                    ctx,
7102                    &caller,
7103                    &frame,
7104                    &target_module_id,
7105                    consecutive_timeouts,
7106                    retry_in,
7107                    probe_in_flight,
7108                )?]);
7109            }
7110        };
7111
7112        // Resolve the per-module budget here so the wait matches the operator's
7113        // intent for this specific target. A per-module override in
7114        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
7115        // daemons) wins over the daemon-wide default.
7116        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
7117        let relay_deadline = Instant::now() + route_bind_relay_timeout;
7118        let operator_app = self
7119            .operator_apps
7120            .lock()
7121            .unwrap_or_else(|p| p.into_inner())
7122            .get(&ctx.connection_id)
7123            .cloned();
7124        timings.enter(RouteOpenPhase::EgressReservation);
7125        let pending = match self
7126            .forwarding
7127            .begin_route_bind_relay_for(
7128                ctx.connection_id,
7129                ctx.egress.clone(),
7130                response_version(&frame),
7131                frame.header.corr,
7132                &target_module_id,
7133                principal.clone(),
7134                bound_scope,
7135                project_root,
7136                identity.harness.clone(),
7137                build_class,
7138                operator_app.clone(),
7139                crate::forwarding::RouteAuthority {
7140                    incarnation: self.supervisor.daemon_incarnation(),
7141                    caller: caller_capture
7142                        .as_ref()
7143                        .map(|caller| caller.authority.pair()),
7144                },
7145                relay_deadline,
7146                busy_refusals == Some(true),
7147                accepts_close_reasons,
7148            )
7149            .await
7150        {
7151            Ok(pending) => pending,
7152            Err(err) => {
7153                return Ok(vec![self.route_open_refusal_frame(
7154                    ctx,
7155                    &caller,
7156                    &frame,
7157                    &target_module_id,
7158                    "relay_reservation_failed",
7159                    forwarding_error_code(&err),
7160                    err.to_string(),
7161                )?])
7162            }
7163        };
7164        timings.enter(RouteOpenPhase::Admission);
7165        let crate::forwarding::PendingRouteBindRelay {
7166            endpoint,
7167            module_sink,
7168            negotiated_ver,
7169            client_channel,
7170            client_epoch,
7171            module_channel,
7172            module_epoch,
7173            corr: relay_corr,
7174            receiver,
7175        } = pending;
7176        let mut reservation =
7177            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
7178
7179        // Reserving egress can wait while a module reconnects or a swap cuts
7180        // over. Check the relay's captured connection, not the earlier by-id
7181        // lookup: the original module's flow/run capabilities cannot authorize
7182        // its replacement. The captured sink stays bound to that connection.
7183        let relay_registration = self
7184            .registry
7185            .get_module_by_connection(endpoint.connection_id)
7186            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7187        let relay_capabilities = relay_registration
7188            .as_ref()
7189            .and_then(|registration| registration.manifest.capabilities.as_ref());
7190        if let Some(selector) = &scope {
7191            // Egress reservation may wait across a scope update. Recheck the
7192            // proof against the current record before sending the captured bind.
7193            if let Err(refusal) = self
7194                .scopes
7195                .read()
7196                .unwrap_or_else(|poisoned| poisoned.into_inner())
7197                .check_bind_proof(&principal, selector)
7198            {
7199                reservation.release_and_disarm();
7200                return Ok(vec![self.route_open_refusal_frame(
7201                    ctx,
7202                    &caller,
7203                    &frame,
7204                    &target_module_id,
7205                    refusal.code,
7206                    refusal.code,
7207                    refusal.message,
7208                )?]);
7209            }
7210        }
7211        if let Some(stamp) = scope_stamp.as_ref() {
7212            if let Err(refusal) = crate::scopes::check_target_required_stamp(
7213                stamp_required,
7214                &target_module_id,
7215                relay_capabilities,
7216            )
7217            .and_then(|()| {
7218                crate::scopes::check_target_flow_support(
7219                    stamp,
7220                    &target_module_id,
7221                    relay_capabilities,
7222                )
7223            })
7224            .and_then(|()| {
7225                crate::scopes::check_target_agent_run_support(
7226                    stamp,
7227                    &target_module_id,
7228                    relay_capabilities,
7229                )
7230            }) {
7231                reservation.release_and_disarm();
7232                return Ok(vec![self.route_open_refusal_frame(
7233                    ctx,
7234                    &caller,
7235                    &frame,
7236                    &target_module_id,
7237                    refusal.code,
7238                    refusal.code,
7239                    refusal.message,
7240                )?]);
7241            }
7242        }
7243
7244        // Older readers may reject unknown fields. Select optional fields from
7245        // the captured endpoint's declaration, including on unscoped binds.
7246        if scope_stamp
7247            .as_ref()
7248            .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
7249            && !crate::scopes::target_supports_federated(relay_capabilities)
7250        {
7251            // Keep bound_scope: even an unstamped legacy route belongs to this epoch.
7252            scope_stamp = None;
7253        }
7254        if !relay_capabilities.is_some_and(|caps| {
7255            caps.provides
7256                .iter()
7257                .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
7258        }) {
7259            if let Some(stamp) = scope_stamp.as_mut() {
7260                stamp.workdir = None;
7261                stamp.project_id = None;
7262            }
7263        }
7264        // A worker run has no agent_id; on_behalf_of_agent is its only agent
7265        // attribution. The run gate above has already required the target to
7266        // declare `worker-runs/v1`, which promises to read both
7267        // `on_behalf_of_agent` and `call_mode`.
7268        let worker_run = scope_stamp
7269            .as_ref()
7270            .is_some_and(crate::scopes::is_worker_run)
7271            && relay_capabilities.is_some_and(|caps| {
7272                caps.provides
7273                    .iter()
7274                    .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
7275            });
7276        if !worker_run
7277            && !relay_capabilities.is_some_and(|caps| {
7278                caps.provides
7279                    .iter()
7280                    .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
7281            })
7282        {
7283            if let Some(stamp) = scope_stamp.as_mut() {
7284                stamp.attributes.on_behalf_of_agent = None;
7285            }
7286        }
7287        if !relay_capabilities.is_some_and(|caps| {
7288            caps.provides
7289                .iter()
7290                .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
7291        }) {
7292            if let Some(stamp) = scope_stamp.as_mut() {
7293                stamp.attributes.flow_version = None;
7294                stamp.attributes.flow_code_hash = None;
7295                stamp.attributes.flow_install_id = None;
7296            }
7297        }
7298        let call_mode = if worker_run
7299            || relay_capabilities
7300                .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
7301        {
7302            Some(derive_call_mode(scope_stamp.as_ref()))
7303        } else {
7304            if let Some(stamp) = scope_stamp.as_mut() {
7305                stamp.expires_at_ms = None;
7306            }
7307            None
7308        };
7309
7310        debug!(
7311            connection_id = ctx.connection_id.get(),
7312            client_channel,
7313            client_epoch,
7314            module_channel,
7315            module_epoch,
7316            "reserved route handle pair"
7317        );
7318        // Rendered BEFORE the move into the relay, because the accept arm below
7319        // is where it is logged and the principal is gone by then.
7320        let route_principal_label = principal_label(&principal);
7321        let relay = ModuleControlRequest::RouteBind {
7322            route_channel: module_channel,
7323            epoch: module_epoch,
7324            target,
7325            identity,
7326            principal: Some(principal),
7327            consumer_capabilities,
7328            role_versions,
7329            admission_facts,
7330            scope: scope_stamp,
7331            call_mode,
7332        };
7333        let peer = RouteBindPeer::new()
7334            .with_observed_peer(
7335                relay_capabilities
7336                    .is_some_and(|caps| {
7337                        caps.provides
7338                            .iter()
7339                            .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
7340                    })
7341                    .then(|| {
7342                        self.observed_peers
7343                            .lock()
7344                            .unwrap_or_else(|p| p.into_inner())
7345                            .get(&ctx.connection_id)
7346                            .cloned()
7347                            .unwrap_or_else(ObservedPeer::unobserved)
7348                    }),
7349            )
7350            .with_operator_app(
7351                self.connection_operator_app(operator_app.as_ref())
7352                    .filter(|_| {
7353                        relay_capabilities.is_some_and(|caps| {
7354                            caps.provides
7355                                .iter()
7356                                .any(|cap| cap == OPERATOR_APP_CAPABILITY)
7357                        })
7358                    }),
7359            )
7360            .with_caller(caller_for_target(
7361                caller_capture.as_ref(),
7362                relay_capabilities,
7363                &target_module_id,
7364            ));
7365        let relay_body = crate::bind_compat::encode_bind(
7366            &relay,
7367            &peer,
7368            &legacy_bind_project,
7369            relay_capabilities,
7370        )
7371        .map_err(|err| {
7372            RouterError::backend(
7373                0,
7374                frame.header.corr,
7375                format!("failed to encode route.bind request: {err}"),
7376            )
7377        })?;
7378        let relay_frame = Frame::build_with_version(
7379            negotiated_ver,
7380            FrameType::Request,
7381            control_flags(),
7382            0,
7383            0,
7384            relay_corr,
7385            relay_body,
7386        )
7387        .map_err(RouterError::FrameBuild)?;
7388
7389        timings.enter(RouteOpenPhase::BindSend);
7390        let sent = module_sink.send(relay_frame).await;
7391        timings.enter(RouteOpenPhase::Admission);
7392        if let Err(err) = sent {
7393            reservation.release_and_disarm();
7394            return Ok(vec![self.route_open_refusal_frame(
7395                ctx,
7396                &caller,
7397                &frame,
7398                &target_module_id,
7399                "relay_send_failed",
7400                "target_unavailable",
7401                err.to_string(),
7402            )?]);
7403        }
7404
7405        #[cfg(test)]
7406        if let Some(hold) = &self.route_bind_relay_sent_hold {
7407            // First wait: tell the test the bind is queued. Second wait: stay
7408            // here until the test has driven whatever it wants to happen
7409            // between queueing the bind and marking it enqueued.
7410            hold.wait().await;
7411            hold.wait().await;
7412        }
7413
7414        // A missing relay is not by itself an abandoned bind: the module may
7415        // already have acked it and the route may be committed. Only a relay
7416        // dropped without any module reply needs this GOODBYE; one a reply
7417        // settled has already had its GOODBYE decided by the reply path.
7418        if self
7419            .forwarding
7420            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
7421            .map_err(RouterError::Forwarding)?
7422            == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
7423        {
7424            self.send_abandoned_route_bind_goodbye(
7425                &module_sink,
7426                negotiated_ver,
7427                module_channel,
7428                module_epoch,
7429            );
7430        }
7431
7432        timings.enter(RouteOpenPhase::BindAckWait);
7433        let outcome = timeout_at(relay_deadline, receiver).await;
7434        timings.enter(RouteOpenPhase::Admission);
7435        match outcome {
7436            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
7437                reservation.disarm();
7438                if breaker.record_accepted() {
7439                    info!(
7440                        module_id = %target_module_id,
7441                        "route.bind breaker closed: the probe was accepted"
7442                    );
7443                }
7444                self.observe_route_open_accept(
7445                    ctx,
7446                    &target_module_id,
7447                    &route_principal_label,
7448                    &caller.scope,
7449                );
7450                Ok(Vec::new())
7451            }
7452            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
7453                reservation.release_and_disarm();
7454                // A module that says no in microseconds is healthy. Rejection
7455                // is a different condition with its own refusal and must not
7456                // move the breaker.
7457                breaker.record_inconclusive();
7458                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
7459            }
7460            Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
7461                reservation.release_and_disarm();
7462                breaker.record_inconclusive();
7463                self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
7464            }
7465            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
7466                reservation.release_and_disarm();
7467                breaker.record_inconclusive();
7468                // Fires when the module's connection closes while a relayed
7469                // bind is pending -- typically a caller racing a module restart
7470                // whose bind was relayed BEFORE the drain mark went up. Logged
7471                // because the caller sees only its own error and the fleet has
7472                // already spent one diagnosis round unable to tell this arm
7473                // from a relay timeout without daemon-side evidence.
7474                tracing::warn!(
7475                    module_id = %target_module_id,
7476                    "route.bind relay abandoned: {message}"
7477                );
7478                Ok(vec![self.route_open_refusal_frame(
7479                    ctx,
7480                    &caller,
7481                    &frame,
7482                    &target_module_id,
7483                    "relay_abandoned",
7484                    "target_unavailable",
7485                    message,
7486                )?])
7487            }
7488            Ok(Err(_)) => {
7489                reservation.release_and_disarm();
7490                breaker.record_inconclusive();
7491                Ok(vec![self.route_open_refusal_frame(
7492                    ctx,
7493                    &caller,
7494                    &frame,
7495                    &target_module_id,
7496                    "relay_waiter_canceled",
7497                    "target_unavailable",
7498                    "route.bind relay waiter was canceled before the module responded",
7499                )?])
7500            }
7501            Err(_) => {
7502                reservation.release_and_disarm();
7503                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
7504                // answer at all is the one condition a fast refusal can
7505                // usefully stand in for; every other arm already answered.
7506                if let Some(opened) = breaker.record_timeout(
7507                    self.route_bind_breaker_threshold,
7508                    self.route_bind_breaker_cooldown,
7509                ) {
7510                    warn!(
7511                        module_id = %target_module_id,
7512                        consecutive_timeouts = opened.consecutive_timeouts,
7513                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
7514                        reopened_after_probe = opened.reopened_after_probe,
7515                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
7516                    );
7517                }
7518                // The generous budget just burned to no answer: the module is
7519                // registered and its connection is up, but its bind handler sat
7520                // on the ack for the full budget (warm-on-bind, cold configure,
7521                // or a wedged handler). Every earlier unavailability shape
7522                // fast-refuses BEFORE the relay, so this arm firing means the
7523                // slowness is module-side -- log it so the per-module timeline
7524                // is reconstructable without client audit rows.
7525                tracing::warn!(
7526                    module_id = %target_module_id,
7527                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
7528                    "route.bind relay timed out: module did not ack within budget"
7529                );
7530                Ok(vec![self.route_open_refusal_frame(
7531                    ctx,
7532                    &caller,
7533                    &frame,
7534                    &target_module_id,
7535                    "relay_timed_out",
7536                    "module_timeout",
7537                    format!(
7538                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
7539                        route_bind_relay_timeout
7540                    ),
7541                )?])
7542            }
7543        }
7544    }
7545
7546    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7547        let response = ClientControlResponse::SupervisorSpawnSnapshot {
7548            snapshot: self.supervisor.spawn_snapshot(),
7549        };
7550        Ok(vec![control_response_body_frame(
7551            &frame,
7552            &response,
7553            "ClientControlResponse::SupervisorSpawnSnapshot",
7554        )?])
7555    }
7556
7557    fn handle_supervisor_spawn_subscribe(
7558        &self,
7559        ctx: &RouteCtx,
7560        frame: Frame,
7561        since: Option<SpawnCursor>,
7562    ) -> Result<Vec<Frame>, RouterError> {
7563        match self.supervisor.subscribe_spawns(
7564            ctx.connection_id,
7565            frame.header.corr,
7566            response_version(&frame),
7567            since,
7568            ctx.egress.clone(),
7569        ) {
7570            Ok(()) => Ok(Vec::new()),
7571            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
7572                Ok(vec![control_error_body_frame(
7573                    &frame,
7574                    ErrorBody {
7575                        code: "spawn_cursor_incarnation_mismatch".to_string(),
7576                        message: "spawn cursor belongs to a different daemon incarnation"
7577                            .to_string(),
7578                        detail: Some(serde_json::json!({
7579                            "current_daemon_incarnation": current
7580                        })),
7581                    },
7582                )?])
7583            }
7584            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
7585                &frame,
7586                ErrorBody {
7587                    code: "spawn_cursor_too_old".to_string(),
7588                    message: "spawn cursor predates the retained event ring".to_string(),
7589                    detail: Some(serde_json::json!({
7590                        "oldest_retained_cursor": oldest
7591                    })),
7592                },
7593            )?]),
7594            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
7595                0,
7596                frame.header.corr,
7597                format!("failed to open supervisor spawn subscription: {error}"),
7598            )),
7599        }
7600    }
7601
7602    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7603        // Status/configuration snapshots may wait on synchronous supervisor
7604        // locks. Keep that work off the connection's async runtime thread.
7605        let handler = self.clone();
7606        let runtime = tokio::runtime::Handle::current();
7607        // The blocking pool does not inherit the control request's tracing
7608        // subscriber. Preserve it so slow snapshot-lock warnings stay visible.
7609        let subscriber = tracing::dispatcher::get_default(Clone::clone);
7610        let corr = frame.header.corr;
7611        tokio::task::spawn_blocking(move || {
7612            runtime.block_on(
7613                list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
7614                    .with_subscriber(subscriber),
7615            )
7616        })
7617        .await
7618        .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
7619    }
7620
7621    async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
7622        use crate::config_gate::runtime::ApprovedDaemonView;
7623        let publication = self
7624            .config_runtime
7625            .as_ref()
7626            .and_then(|owner| owner.polled_snapshot().ok());
7627        let approved = publication
7628            .as_ref()
7629            .map(|view| view.approved_daemon.clone())
7630            .unwrap_or_else(|| {
7631                if self.config_runtime.is_some() {
7632                    ApprovedDaemonView::Unavailable
7633                } else {
7634                    ApprovedDaemonView::NotStamped
7635                }
7636            });
7637        let generation = self
7638            .registry
7639            .generation()
7640            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
7641        let supervised = self.supervisor.list();
7642        let mut statuses = BTreeMap::new();
7643        let mut not_running = BTreeSet::new();
7644        for module in &supervised {
7645            let status = module.status_for_control("list").map_err(|err| {
7646                RouterError::backend(
7647                    0,
7648                    frame.header.corr,
7649                    format!("failed to read supervisor status: {err}"),
7650                )
7651            })?;
7652            if !status.enabled || !status.process_alive {
7653                not_running.insert(module.module_id().to_owned());
7654            }
7655            statuses.insert(module.module_id().to_owned(), status);
7656        }
7657        // Legacy per-process diagnostics keep their shared optional budget.
7658        // The cached tree and accounting export below does not spend it.
7659        let mut process_resources = BTreeMap::new();
7660        for module in &supervised {
7661            process_resources.insert(
7662                module.module_id().to_owned(),
7663                crate::subtree_resources::read_process(module.clone()).await,
7664            );
7665        }
7666        // Image evidence uses the same optional deadline, but an exhausted
7667        // budget must not suppress the committed resource view.
7668        let mut image_readings = BTreeMap::new();
7669        for module in &supervised {
7670            image_readings.insert(
7671                module.module_id().to_owned(),
7672                module.running_image_agreement().await,
7673            );
7674        }
7675        let readings = self.resource_tracker.list(supervised.clone());
7676        let mut modules = Vec::new();
7677        for module in supervised {
7678            let status = statuses
7679                .remove(module.module_id())
7680                .expect("each supervised module has a status snapshot");
7681            let (configured, _) = module.configuration().map_err(|err| {
7682                RouterError::backend(
7683                    0,
7684                    frame.header.corr,
7685                    format!("failed to read module configuration: {err}"),
7686                )
7687            })?;
7688            let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
7689                RouterError::backend(
7690                    0,
7691                    frame.header.corr,
7692                    format!("failed to read retry time: {err}"),
7693                )
7694            })?;
7695            let image = image_readings
7696                .remove(module.module_id())
7697                .expect("each supervised module has an image snapshot");
7698            let resources = process_resources.remove(module.module_id());
7699            let mut verdict =
7700                reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
7701            verdict.spec = Some(launch_spec_agreement(
7702                &configured,
7703                status.process_alive,
7704                status.launch_spec_digest.as_ref(),
7705            ));
7706            let pending_reload = Some(verdict);
7707            let pending_entry = match &approved {
7708                ApprovedDaemonView::Approved { config, .. } => config
7709                    .modules
7710                    .iter()
7711                    .find(|m| m.module_id == module.module_id()),
7712                _ => None,
7713            };
7714            let pending = self
7715                .pending_module_fields(
7716                    &module,
7717                    pending_entry,
7718                    false,
7719                    Self::registration_channel_in_force(
7720                        publication
7721                            .as_ref()
7722                            .and_then(|view| view.registration_declaration(module.module_id()))
7723                            .as_deref(),
7724                        pending_entry.map(|entry| entry.registration_channel),
7725                    ),
7726                )
7727                .map_err(|err| RouterError::backend(0, frame.header.corr, err))?;
7728            let script = configured.script.as_ref().map(|entry| {
7729                let approved = self
7730                    .config_runtime
7731                    .as_ref()
7732                    .and_then(|owner| owner.approved_script(&status.module_id, &entry.path));
7733                crate::supervise::script::status(entry, approved.as_ref())
7734            });
7735            let configured_enabled = match &approved {
7736                ApprovedDaemonView::Approved {
7737                    config: approved, ..
7738                } => approved
7739                    .modules
7740                    .iter()
7741                    .find(|module| module.module_id == status.module_id)
7742                    .map(|module| module.enabled),
7743                ApprovedDaemonView::NotStamped => self
7744                    .capability_evaluator
7745                    .configured_enabled(&status.module_id),
7746                ApprovedDaemonView::Unavailable => None,
7747            };
7748            let admission_refused = status
7749                .admission_refused
7750                .clone()
7751                .filter(|_| status.enabled && !status.process_alive)
7752                .filter(|_| {
7753                    publication
7754                        .as_ref()
7755                        .and_then(|view| view.registration_declaration(&status.module_id))
7756                        .as_deref()
7757                        == Some("undeclared")
7758                });
7759            // Keep the retired policy field on the wire for one release so
7760            // existing status consumers still receive the platform policy.
7761            let mut entry = SupervisorEntry::new(
7762                status.module_id,
7763                status.state.to_string(),
7764                status.enabled,
7765                status.live,
7766                status.health.status,
7767            )
7768            .with_configured_enabled(configured_enabled)
7769            .with_script(script)
7770            .with_admission_refused(admission_refused)
7771            .with_next_retry_at_ms(next_retry_at_ms)
7772            .with_launch_nonce_env(Some(!cfg!(unix)))
7773            .with_launch_nonce_source(status.launch_nonce_source)
7774            .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
7775            .with_protocol(status.protocol)
7776            .with_pending_reload(pending_reload)
7777            .with_last_probe_ms(status.health.last_probe_ms)
7778            .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
7779            .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
7780            .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
7781            .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
7782            .with_restart_count(Some(status.restart_count))
7783            .with_max_restarts(Some(status.max_restarts))
7784            .with_lifetime_restarts(Some(status.lifetime_restarts))
7785            .with_spawn_generation(Some(status.spawn_generation))
7786            .with_restart_window_secs(Some(status.restart_window.as_secs()))
7787            .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
7788            .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
7789            .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
7790            .with_resources(resources);
7791            entry.pending = pending;
7792            modules.push(entry);
7793        }
7794        for module in &mut modules {
7795            if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
7796                module.tree = tree.clone();
7797                module.cpu_last_hour = hour.clone();
7798            }
7799            module.tree_unavailable = readings
7800                .reasons
7801                .get(&module.module_id)
7802                .map(|reason| (*reason).to_owned());
7803            if module.tree.is_none() {
7804                module.tree_unavailable = Some(
7805                    if not_running.contains(&module.module_id) {
7806                        "not_running"
7807                    } else if let Some(reason) = readings.reasons.get(&module.module_id) {
7808                        reason
7809                    } else {
7810                        "not_yet_measured"
7811                    }
7812                    .to_owned(),
7813                );
7814            }
7815        }
7816        let response = ClientControlResponse::SupervisorList {
7817            generation,
7818            modules,
7819            daemon: readings.daemon,
7820            agent_commands: readings.agent_commands,
7821        };
7822        Ok(vec![control_response_body_frame(
7823            &frame,
7824            &response,
7825            "ClientControlResponse::SupervisorList",
7826        )?])
7827    }
7828
7829    fn handle_supervisor_stderr_tail(
7830        &self,
7831        frame: Frame,
7832        module_id: String,
7833        max_lines: Option<u32>,
7834        max_bytes: Option<u32>,
7835    ) -> Result<Vec<Frame>, RouterError> {
7836        let Some(module) = self.supervisor.get(&module_id) else {
7837            return Ok(vec![control_error_frame(
7838                &frame,
7839                "unknown_module",
7840                format!("module_id '{module_id}' is not supervised"),
7841            )?]);
7842        };
7843
7844        let snapshot = module.stderr_tail(
7845            max_lines.map(|value| value as usize),
7846            max_bytes.map(|value| value as usize),
7847        );
7848
7849        let response = ClientControlResponse::SupervisorStderrTail {
7850            module_id,
7851            tail: StderrTail {
7852                capture: match snapshot.capture {
7853                    CaptureState::Captured => StderrCaptureState::Captured,
7854                    CaptureState::Incomplete { reason } => {
7855                        StderrCaptureState::Incomplete { reason }
7856                    }
7857                    CaptureState::NotCaptured { reason } => {
7858                        StderrCaptureState::NotCaptured { reason }
7859                    }
7860                },
7861                entries: snapshot
7862                    .entries
7863                    .into_iter()
7864                    .map(|entry| match entry {
7865                        TailEntry::Line {
7866                            text,
7867                            truncated,
7868                            at_ms,
7869                        } => StderrTailEntry::Line {
7870                            text,
7871                            truncated,
7872                            at_ms,
7873                        },
7874                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
7875                    })
7876                    .collect(),
7877                dropped_lines: snapshot.dropped_lines,
7878            },
7879        };
7880        Ok(vec![control_response_body_frame(
7881            &frame,
7882            &response,
7883            "ClientControlResponse::SupervisorStderrTail",
7884        )?])
7885    }
7886
7887    async fn observe_control_wait<F: std::future::Future>(
7888        &self,
7889        ctx: &RouteCtx,
7890        target: Option<&str>,
7891        kind: &'static str,
7892        future: F,
7893    ) -> F::Output {
7894        self.counters
7895            .observe_reader_wait(
7896                || {
7897                    let module = self
7898                        .forwarding
7899                        .module_id_for_connection(ctx.connection_id)
7900                        .ok()
7901                        .flatten();
7902                    let principal = if module.is_some() { "module" } else { "direct" };
7903                    (
7904                        self.resolve_reader_wait_target(kind, target)
7905                            .or(module)
7906                            .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
7907                        principal.into(),
7908                    )
7909                },
7910                kind,
7911                future,
7912            )
7913            .await
7914    }
7915
7916    /// Resolve the reader-wait module ID only from supervisor or registry
7917    /// records; never attribute a wait from the request selector alone.
7918    pub(crate) fn resolve_reader_wait_target(
7919        &self,
7920        op: &str,
7921        selector: Option<&str>,
7922    ) -> Option<String> {
7923        let selector = selector?;
7924        match op {
7925            ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
7926                .supervisor
7927                .get(selector)
7928                .map(|module| module.module_id().to_owned()),
7929            ops::SUPERVISOR_HEALTH_PROBE => self
7930                .registry
7931                .get_module(selector)
7932                .ok()
7933                .flatten()
7934                .map(|registration| registration.manifest.module_id),
7935            _ => None,
7936        }
7937    }
7938
7939    async fn handle_supervisor_terminals(
7940        &self,
7941        ctx: &RouteCtx,
7942        frame: Frame,
7943        module_id: String,
7944    ) -> Result<Vec<Frame>, RouterError> {
7945        let Some(module) = self.supervisor.get(&module_id) else {
7946            return Ok(vec![control_error_frame(
7947                &frame,
7948                "unknown_module",
7949                format!("module_id '{module_id}' is not supervised"),
7950            )?]);
7951        };
7952
7953        // The journal read runs on a blocking thread: it can be megabytes of
7954        // file I/O and must not occupy a runtime worker.
7955        let terminals = self
7956            .observe_control_wait(
7957                ctx,
7958                Some(module.module_id()),
7959                ops::SUPERVISOR_TERMINALS,
7960                module.read_durable_terminal_history(),
7961            )
7962            .await
7963            .map_err(|error| {
7964                RouterError::backend(
7965                    0,
7966                    frame.header.corr,
7967                    format!("failed to read terminal history: {error}"),
7968                )
7969            })?;
7970        let response = ClientControlResponse::SupervisorTerminals {
7971            module_id,
7972            terminals,
7973        };
7974        Ok(vec![control_response_body_frame(
7975            &frame,
7976            &response,
7977            "ClientControlResponse::SupervisorTerminals",
7978        )?])
7979    }
7980
7981    fn handle_supervisor_routes(
7982        &self,
7983        frame: Frame,
7984        module_id: Option<String>,
7985    ) -> Result<Vec<Frame>, RouterError> {
7986        let modules = self
7987            .forwarding
7988            .route_census(module_id.as_deref())
7989            .map_err(RouterError::Forwarding)?
7990            .into_iter()
7991            .map(|(module_id, routes)| SupervisorRouteModule {
7992                module_id,
7993                routes: routes
7994                    .into_iter()
7995                    .map(|route| SupervisorRoute {
7996                        consumer: match route.principal {
7997                            Principal::Reserved { module_id } => {
7998                                SupervisorRouteConsumer::Reserved { module_id }
7999                            }
8000                            Principal::Direct | Principal::Unverified => {
8001                                SupervisorRouteConsumer::Direct {
8002                                    connection_id: route.goodbye_target.connection_id.get(),
8003                                }
8004                            }
8005                        },
8006                        age_ms: Instant::now()
8007                            .saturating_duration_since(route.bound_at)
8008                            .as_millis()
8009                            .try_into()
8010                            .unwrap_or(u64::MAX),
8011                        draining: route.draining,
8012                        drain_reason: route.drain_reason,
8013                    })
8014                    .collect(),
8015            })
8016            .collect();
8017        let response = ClientControlResponse::SupervisorRoutes { modules };
8018        Ok(vec![control_response_body_frame(
8019            &frame,
8020            &response,
8021            "ClientControlResponse::SupervisorRoutes",
8022        )?])
8023    }
8024
8025    fn running_registration_method(
8026        &self,
8027        module: &str,
8028    ) -> Option<subc_control::RegistrationMethod> {
8029        let registration = self.registry.get_module(module).ok()??;
8030        let connection = registration.connection_id;
8031        let channel = self.supervisor.registration_spawn(connection).is_some();
8032        let nonce = if channel {
8033            None
8034        } else {
8035            self.hello_launch_nonces
8036                .lock()
8037                .ok()?
8038                .nonce(connection)
8039                .map(str::to_owned)
8040        };
8041        if !channel && nonce.is_none() {
8042            return None;
8043        }
8044        self.supervisor
8045            .registration_snapshot_is_current_live(
8046                &self.registry,
8047                module,
8048                connection,
8049                nonce.as_deref(),
8050            )
8051            .then_some(if channel {
8052                subc_control::RegistrationMethod::Channel
8053            } else {
8054                subc_control::RegistrationMethod::Nonce
8055            })
8056    }
8057
8058    async fn handle_supervisor_provenance(
8059        &self,
8060        ctx: &RouteCtx,
8061        frame: Frame,
8062        module_id: Option<String>,
8063    ) -> Result<Vec<Frame>, RouterError> {
8064        let targeted = module_id.is_some();
8065        let selected = if let Some(module_id) = module_id {
8066            let Some(module) = self.supervisor.get(&module_id) else {
8067                return Ok(vec![control_error_frame(
8068                    &frame,
8069                    "unknown_module",
8070                    format!("module_id '{module_id}' is not supervised"),
8071                )?]);
8072            };
8073            vec![module]
8074        } else {
8075            self.supervisor.list()
8076        };
8077
8078        let target = targeted.then(|| selected[0].module_id());
8079        let declarations = if let Some(owner) = &self.config_runtime {
8080            owner
8081                .on_blocking(|owner| owner.registration_declarations())
8082                .await
8083                .ok()
8084        } else {
8085            None
8086        };
8087        self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
8088            let mut modules = Vec::with_capacity(selected.len());
8089            let mut extensions = Vec::with_capacity(selected.len());
8090            for module in &selected {
8091                let (status, observed_image) = module
8092                    .status_and_running_image_agreement()
8093                    .await
8094                    .map_err(|err| {
8095                        RouterError::backend(
8096                            0,
8097                            frame.header.corr,
8098                            format!("failed to read supervisor status: {err}"),
8099                        )
8100                    })?;
8101                let module_declared = self
8102                    .registry
8103                    .get_module(&status.module_id)
8104                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
8105                    .and_then(|registration| registration.manifest.provenance)
8106                    .map(|build| ModuleDeclaredProvenance::Reported { build })
8107                    .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
8108                #[cfg(test)]
8109                let running_image = match &self.provenance_probe_override {
8110                    Some(result) => result.clone(),
8111                    None => observed_image,
8112                };
8113                #[cfg(not(test))]
8114                let running_image = observed_image;
8115                let registration = self.running_registration_method(&status.module_id);
8116                let mut extension = subc_control::SupervisorModuleProvenanceExt::new()
8117                    .with_registration(registration);
8118                extension.registration_declaration = declarations
8119                    .as_ref()
8120                    .and_then(|entries| entries.get(&status.module_id))
8121                    .cloned();
8122                if status.process_alive && status.registration_declaration.is_some() {
8123                    extension.registration_declaration = status.registration_declaration.clone();
8124                }
8125                extensions.push(extension);
8126                modules.push(SupervisorModuleProvenance {
8127                    module_id: status.module_id,
8128                    module_declared,
8129                    daemon_observed: SupervisorObservedProcess {
8130                        launch_nonce_source: status.launch_nonce_source,
8131                        pid: status.pid,
8132                        spawned_at_ms: status.spawned_at_ms,
8133                        spawned_from: status.spawned_from,
8134                        running_image,
8135                    },
8136                });
8137            }
8138            let daemon = SupervisorDaemonProvenance {
8139                daemon_build: self.daemon_provenance.build.clone(),
8140                daemon_observed: DaemonObservedProcess {
8141                    pid: self.daemon_provenance.pid,
8142                    started_at_ms: self
8143                        .daemon_provenance
8144                        .start_clock
8145                        .map(|clock| clock.started_at_ms())
8146                        .or(self.daemon_provenance.started_at_ms),
8147                    running_image: self
8148                        .daemon_provenance
8149                        .probe
8150                        .observe(
8151                            self.daemon_provenance.pid,
8152                            self.daemon_provenance.executable_path.as_deref(),
8153                            self.daemon_provenance.executable_identity.clone(),
8154                            self.daemon_provenance.process_start_time,
8155                        )
8156                        .await,
8157                },
8158            };
8159            let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
8160            let mut response = serde_json::to_value(response)
8161                .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
8162            for (entry, extension) in response["modules"]
8163                .as_array_mut()
8164                .expect("provenance modules are an array")
8165                .iter_mut()
8166                .zip(extensions)
8167            {
8168                extension.write_to(entry).map_err(|error| {
8169                    RouterError::backend(0, frame.header.corr, error.to_string())
8170                })?;
8171            }
8172            Ok(vec![control_response_body_frame(
8173                &frame,
8174                &response,
8175                "ClientControlResponse::SupervisorProvenance",
8176            )?])
8177        })
8178        .await
8179    }
8180
8181    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
8182        self.refresh_capability_requirements();
8183        let publication = self
8184            .config_runtime
8185            .as_ref()
8186            .and_then(|owner| owner.polled_snapshot().ok());
8187        let generation = self
8188            .registry
8189            .generation()
8190            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
8191        let modules = self
8192            .supervisor
8193            .list()
8194            .into_iter()
8195            .map(|module| {
8196                let status = module.status_for_control("health").map_err(|err| {
8197                    RouterError::backend(
8198                        0,
8199                        frame.header.corr,
8200                        format!("failed to read supervisor health: {err}"),
8201                    )
8202                })?;
8203                let module_id = status.module_id;
8204                let capability_detail = self
8205                    .capability_evaluator
8206                    .required_problem_detail(&module_id);
8207                let declaration_warning = self.registry.get_module(&module_id).ok().flatten()
8208                    .filter(|registration| registration.manifest.registration_channel == Some(false))
8209                    .filter(|_| publication.as_ref().and_then(|view| view.registration_declaration(&module_id))
8210                        .as_deref().map(|next| if status.process_alive { status.registration_declaration.as_deref().unwrap_or(next) } else { next }) == Some("declared"))
8211                    .map(|_| format!("{module_id}: HELLO registration_channel is false despite the approved channel declaration"));
8212                let capability_detail = append_capability_problem_detail(capability_detail, declaration_warning);
8213                Ok(SupervisorHealthEntry {
8214                    module_id,
8215                    status: status.health.status,
8216                    detail: append_capability_problem_detail(
8217                        status.health.detail,
8218                        capability_detail,
8219                    ),
8220                    metrics: status.health.metrics,
8221                    consecutive_failures: status.health.consecutive_failures,
8222                    late_answer_count: status.health.late_answer_count,
8223                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
8224                    last_action: status.health.last_action,
8225                    last_action_ms: status.health.last_action_ms,
8226                    last_probe_ms: status.health.last_probe_ms,
8227                })
8228            })
8229            .collect::<Result<Vec<_>, RouterError>>()?;
8230        let response = ClientControlResponse::SupervisorHealth {
8231            generation,
8232            modules,
8233        };
8234        Ok(vec![control_response_body_frame(
8235            &frame,
8236            &response,
8237            "ClientControlResponse::SupervisorHealth",
8238        )?])
8239    }
8240
8241    async fn handle_supervisor_restart(
8242        &self,
8243        frame: Frame,
8244        module_id: String,
8245        drain_timeout_ms: Option<u64>,
8246        requester: Principal,
8247    ) -> Result<Vec<Frame>, RouterError> {
8248        let operation_lock = self.supervisor.operation_lock();
8249        let _operation_guard = operation_lock.lock().await;
8250        let Some(module) = self.supervisor.get(&module_id) else {
8251            return Ok(vec![control_error_frame(
8252                &frame,
8253                "unknown_module",
8254                format!("module_id '{module_id}' is not supervised"),
8255            )?]);
8256        };
8257
8258        self.route_outages.mark_operator_action(&module_id);
8259        if let Err(err) = module
8260            .restart_requested(drain_timeout_ms, Some(requester))
8261            .await
8262        {
8263            self.route_outages
8264                .operator_action_ended_unrefused(&module_id);
8265            let (code, message) = match err {
8266                _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8267                crate::supervise::SuperviseError::Disabled { .. } => {
8268                    ("module_disabled", err.to_string())
8269                }
8270                crate::supervise::SuperviseError::SwapInProgress { .. } => {
8271                    ("swap_in_progress", err.to_string())
8272                }
8273                _ => (
8274                    "target_unavailable",
8275                    format!("failed to restart module_id '{module_id}': {err}"),
8276                ),
8277            };
8278            return Ok(vec![control_error_frame(&frame, code, message)?]);
8279        }
8280
8281        let response = ClientControlResponse::SupervisorAck {
8282            module_id,
8283            applied: true,
8284        };
8285        Ok(vec![control_response_body_frame(
8286            &frame,
8287            &response,
8288            "ClientControlResponse::SupervisorAck",
8289        )?])
8290    }
8291
8292    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
8293    /// when the old process has finished draining: a caller whose own lane
8294    /// rides the old process must get its reply before that drain waits on it.
8295    async fn handle_supervisor_swap(
8296        &self,
8297        frame: Frame,
8298        module_id: String,
8299        ready_timeout_ms: Option<u64>,
8300        requester: Principal,
8301    ) -> Result<Vec<Frame>, RouterError> {
8302        // The daemon-wide operation lock is held only to resolve the handle,
8303        // not across the swap. The swap can take its whole readiness budget,
8304        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
8305        // holding it here would park an operator's stop behind the swap it is
8306        // meant to abort. A rescan or stop that reaches the module during the
8307        // swap is served by the swap itself (see `supervise_swap`).
8308        let module = {
8309            let operation_lock = self.supervisor.operation_lock();
8310            let _operation_guard = operation_lock.lock().await;
8311            self.supervisor.get(&module_id)
8312        };
8313        let Some(module) = module else {
8314            return Ok(vec![control_error_frame(
8315                &frame,
8316                "unknown_module",
8317                format!("module_id '{module_id}' is not supervised"),
8318            )?]);
8319        };
8320
8321        self.route_outages.mark_operator_action(&module_id);
8322        if let Err(err) = module
8323            .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
8324            .await
8325        {
8326            self.route_outages
8327                .operator_action_ended_unrefused(&module_id);
8328            use crate::supervise::SuperviseError;
8329            let message = err.to_string();
8330            let error = match err {
8331                SuperviseError::ConfigHeld { .. } => ErrorBody::new("config_gated", message),
8332                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
8333                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
8334                    code: "swap_refused".to_string(),
8335                    message,
8336                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
8337                },
8338                SuperviseError::SwapFailed {
8339                    arm,
8340                    candidate_exit,
8341                    ..
8342                } => ErrorBody {
8343                    code: "swap_failed".to_string(),
8344                    message,
8345                    detail: Some(serde_json::json!({
8346                        "arm": arm.as_str(),
8347                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
8348                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
8349                    })),
8350                },
8351                _ => ErrorBody::new(
8352                    "target_unavailable",
8353                    format!("failed to swap module_id '{module_id}': {message}"),
8354                ),
8355            };
8356            return Ok(vec![control_error_body_frame(&frame, error)?]);
8357        }
8358        // A completed swap kept the incumbent serving until cutover, so it
8359        // usually opened no outage; a mark left behind would make the next,
8360        // unrelated outage read as requested.
8361        self.route_outages
8362            .operator_action_ended_unrefused(&module_id);
8363
8364        let response = ClientControlResponse::SupervisorAck {
8365            module_id,
8366            applied: true,
8367        };
8368        Ok(vec![control_response_body_frame(
8369            &frame,
8370            &response,
8371            "ClientControlResponse::SupervisorAck",
8372        )?])
8373    }
8374
8375    async fn handle_supervisor_reload(
8376        &self,
8377        frame: Frame,
8378        module_id: String,
8379        requester: Principal,
8380    ) -> Result<Vec<Frame>, RouterError> {
8381        let operation_lock = self.supervisor.operation_lock();
8382        let _operation_guard = operation_lock.lock().await;
8383        let Some(module) = self.supervisor.get(&module_id) else {
8384            return Ok(vec![control_error_frame(
8385                &frame,
8386                "unknown_module",
8387                format!("module_id '{module_id}' is not supervised"),
8388            )?]);
8389        };
8390
8391        self.route_outages.mark_operator_action(&module_id);
8392        if let Err(err) = module.reload_requested(Some(requester)).await {
8393            self.route_outages
8394                .operator_action_ended_unrefused(&module_id);
8395            let (code, message) = match err {
8396                _ if err.is_config_gate_error() => ("config_gated", err.to_string()),
8397                crate::supervise::SuperviseError::Disabled { .. } => {
8398                    ("module_disabled", err.to_string())
8399                }
8400                crate::supervise::SuperviseError::SwapInProgress { .. } => {
8401                    ("swap_in_progress", err.to_string())
8402                }
8403                _ => (
8404                    "reload_failed",
8405                    format!("failed to reload module_id '{module_id}': {err}"),
8406                ),
8407            };
8408            return Ok(vec![control_error_frame(&frame, code, message)?]);
8409        }
8410
8411        let response = ClientControlResponse::SupervisorAck {
8412            module_id,
8413            applied: true,
8414        };
8415        Ok(vec![control_response_body_frame(
8416            &frame,
8417            &response,
8418            "ClientControlResponse::SupervisorAck",
8419        )?])
8420    }
8421
8422    /// `supervisor.rescan` and `supervisor.rescan_checked`. `mode` says where
8423    /// the configuration comes from and whether to apply it; every mode then
8424    /// runs the same comparison, so a preview describes the apply it previews.
8425    async fn handle_supervisor_rescan(
8426        &self,
8427        frame: Frame,
8428        mode: RescanMode,
8429        requester: Principal,
8430    ) -> Result<Vec<Frame>, RouterError> {
8431        let preview = mode.previews();
8432        let mut approved_preview = None;
8433        if matches!(mode, RescanMode::Preview) {
8434            if let Some(owner) = &self.config_runtime {
8435                match owner
8436                    .on_blocking(|owner| {
8437                        owner.observe_file_proposal()?;
8438                        owner.approved_daemon_configuration()
8439                    })
8440                    .await
8441                {
8442                    Ok(config) => approved_preview = config,
8443                    Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8444                }
8445            }
8446        }
8447        let Some(context) = self.rescan.clone() else {
8448            return Ok(vec![control_error_frame(
8449                &frame,
8450                "rescan_unavailable",
8451                "the daemon was not started with a reloadable config path".to_string(),
8452            )?]);
8453        };
8454
8455        let operation_lock = self.supervisor.operation_lock();
8456        let _operation_guard = operation_lock.lock().await;
8457        // Loaded under the operation lock, so no other rescan can change the
8458        // staged configuration between a checked apply's comparison and the
8459        // apply itself.
8460        let loaded = match self.load_rescan_config(&mode, &context) {
8461            Ok(config) => config,
8462            Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
8463        };
8464        // `load` reports a missing file as Ok(None), which is correct at boot
8465        // (no config, nothing to supervise) and catastrophic here: rescan treats
8466        // "not in the config" as "remove it", so an absent file would read as an
8467        // empty module list and retire the entire running fleet. An editor
8468        // writing via write-new-then-rename, or a half-finished edit, is enough
8469        // to open that window. Refuse instead: a config that cannot be read
8470        // carries no instruction to remove anything.
8471        let Some((config, activity_config)) = loaded else {
8472            return Ok(vec![control_error_frame(
8473                &frame,
8474                "invalid_daemon_config",
8475                format!(
8476                    "daemon config not found at {}; refusing to rescan (an absent config would \
8477                     retire every supervised module)",
8478                    context.config_path.display()
8479                ),
8480            )?]);
8481        };
8482        let (
8483            configured_port,
8484            storage_config,
8485            admission_facts_carrier_module_id,
8486            admission_facts_targets,
8487            scope_authority_owners,
8488            federated_scope_owner,
8489            privileged_operator_prompt,
8490            modules,
8491            reserved_capabilities,
8492        ) = (
8493            config.port,
8494            config.storage,
8495            config.admission_facts_carrier_module_id,
8496            config.admission_facts_targets,
8497            config.scope_authority_owners,
8498            config.federated_scope_owner,
8499            config.privileged_operator_prompt,
8500            config.modules,
8501            config.reserved_capabilities,
8502        );
8503
8504        // Collect the sections rescan cannot apply, so the REPLY carries them.
8505        //
8506        // The warning below has always been correct and has always gone only to
8507        // the journal -- addressed to whoever reads logs, while the person who
8508        // just edited the config is looking at the CLI. Naming each section
8509        // individually rather than setting a flag: "something outside modules
8510        // changed" sends the operator back to diffing their own file, which is
8511        // the work this is meant to save.
8512        let mut restart_required = Vec::new();
8513        for section in RestartRequiredSection::ALL {
8514            let changed = match section {
8515                RestartRequiredSection::Port => configured_port != context.configured_port,
8516                RestartRequiredSection::Storage => storage_config != context.storage_config,
8517                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
8518                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
8519                }
8520                RestartRequiredSection::AdmissionFactsTargets => {
8521                    admission_facts_targets != context.admission_facts_targets
8522                }
8523                RestartRequiredSection::ScopeAuthorityOwners => {
8524                    scope_authority_owners != context.scope_authority_owners
8525                }
8526                RestartRequiredSection::FederatedScopeOwner => {
8527                    federated_scope_owner != context.federated_scope_owner
8528                }
8529                RestartRequiredSection::PrivilegedOperatorPrompt => {
8530                    privileged_operator_prompt != context.privileged_operator_prompt
8531                }
8532            };
8533            if changed {
8534                restart_required.push(section.label().to_string());
8535            }
8536        }
8537        if !restart_required.is_empty() {
8538            warn!(
8539                config_path = %context.config_path.display(),
8540                sections = %restart_required.join(", "),
8541                "daemon config changed outside the modules section; restart the daemon to apply those changes"
8542            );
8543        }
8544
8545        for configured in &modules {
8546            if let Err(err) = validate_spec(&configured.module_spec()) {
8547                return Ok(vec![control_error_frame(
8548                    &frame,
8549                    "invalid_daemon_config",
8550                    format!("supervisor rescan rejected daemon config: {err}"),
8551                )?]);
8552            }
8553        }
8554
8555        let configured_capabilities = modules
8556            .iter()
8557            .map(|module| (module.module_id.clone(), module.enabled))
8558            .collect::<Vec<_>>();
8559        let preview_capability_warnings = if preview {
8560            let (_, registrations) = self.runtime_capability_snapshot()?;
8561            let current_modules = self
8562                .supervisor
8563                .list()
8564                .into_iter()
8565                .map(|module| module.module_id().to_string())
8566                .collect::<BTreeSet<_>>();
8567            let resulting_modules = configured_capabilities.clone();
8568            let removed = current_modules
8569                .into_iter()
8570                .filter(|module_id| {
8571                    !resulting_modules
8572                        .iter()
8573                        .any(|(configured_id, _)| configured_id == module_id)
8574                })
8575                .collect::<Vec<_>>();
8576            // A candidate preview also names providers the candidate disables.
8577            // `supervisor.rescan`'s own preview keeps its removal-only warnings.
8578            let disabled = match &mode {
8579                RescanMode::Candidate(_) => resulting_modules
8580                    .iter()
8581                    .filter(|(_, enabled)| !enabled)
8582                    .map(|(module_id, _)| module_id.clone())
8583                    .collect::<Vec<_>>(),
8584                _ => Vec::new(),
8585            };
8586            self.capability_evaluator.preview_warnings(
8587                resulting_modules,
8588                &removed,
8589                &disabled,
8590                &registrations,
8591            )
8592        } else {
8593            Vec::new()
8594        };
8595        let reconciled = match (&mode, approved_preview.as_ref()) {
8596            (RescanMode::Preview, Some(approved)) => Ok((
8597                config_gate::preview_approved_daemon(approved, &modules),
8598                Vec::new(),
8599            )),
8600            (RescanMode::Candidate(candidate), _) => self
8601                .preview_candidate(&context, modules, candidate)
8602                .map(|result| (result, Vec::new())),
8603            _ => {
8604                self.reconcile_supervised_modules(
8605                    &context.supervisor,
8606                    modules,
8607                    preview,
8608                    requester.clone(),
8609                )
8610                .await
8611            }
8612        };
8613        let (result, changed) = match reconciled {
8614            Ok(result) => result,
8615            Err(message) => {
8616                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
8617            }
8618        };
8619        if !preview {
8620            if let Some(ledger) = &self.activity {
8621                ledger.module_rescanned(&result.added, &result.removed, requester.clone());
8622                ledger.config_changed(&result.added, &result.removed, &changed, requester);
8623            }
8624            if let Err(code) = self.configure_activity(activity_config).await {
8625                warn!(
8626                    code,
8627                    "activity retention could not be applied during rescan"
8628                );
8629            }
8630            self.capability_evaluator
8631                .configure(configured_capabilities, reserved_capabilities);
8632            self.capability_evaluator.wake_deadline_loop();
8633            self.refresh_capability_requirements();
8634        }
8635        let mut result = result;
8636        result.restart_required = restart_required;
8637        result.capability_warnings = preview_capability_warnings;
8638        let response = ClientControlResponse::SupervisorRescan { result };
8639        Ok(vec![control_response_body_frame(
8640            &frame,
8641            &response,
8642            "ClientControlResponse::SupervisorRescan",
8643        )?])
8644    }
8645
8646    async fn handle_supervisor_release_reserved(
8647        &self,
8648        frame: Frame,
8649        module_id: String,
8650    ) -> Result<Vec<Frame>, RouterError> {
8651        let Some(context) = self.rescan.clone() else {
8652            return Ok(vec![control_error_frame(
8653                &frame,
8654                "release_unavailable",
8655                "reserved-id release requires a daemon started with a reloadable config path",
8656            )?]);
8657        };
8658        let operation_lock = self.supervisor.operation_lock();
8659        let _operation_guard = operation_lock.lock().await;
8660        let approved = if let Some(owner) = &self.config_runtime {
8661            // Observe the file as a proposal, but decide membership solely from
8662            // approved bytes. A proposed re-add must not retain a removed id.
8663            match owner
8664                .on_blocking(|owner| {
8665                    owner.observe_file_proposal()?;
8666                    owner.approved_daemon_configuration()
8667                })
8668                .await
8669            {
8670                Ok(config) => config,
8671                Err(error) => return Ok(vec![config_gate::config_error_frame(&frame, error)?]),
8672            }
8673        } else {
8674            None
8675        };
8676        let loaded = if let Some(approved) = approved {
8677            Ok(Some(approved))
8678        } else {
8679            crate::daemon_config::load(&context.config_path)
8680        };
8681        let loaded = match loaded {
8682            Ok(Some(config)) => config,
8683            Ok(None) => {
8684                return Ok(vec![control_error_frame(
8685                    &frame,
8686                    "invalid_daemon_config",
8687                    format!(
8688                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
8689                        context.config_path.display()
8690                    ),
8691                )?])
8692            }
8693            Err(err) => {
8694                return Ok(vec![control_error_frame(
8695                    &frame,
8696                    "invalid_daemon_config",
8697                    format!("unable to verify reserved-id release against daemon config: {err}"),
8698                )?])
8699            }
8700        };
8701        if loaded
8702            .modules
8703            .iter()
8704            .any(|configured| configured.module_id == module_id)
8705        {
8706            return Ok(vec![control_error_frame(
8707                &frame,
8708                "reserved_module_configured",
8709                format!(
8710                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
8711                ),
8712            )?]);
8713        }
8714        if !self.supervisor.release_retained_reserved_gate(&module_id) {
8715            return Ok(vec![control_error_frame(
8716                &frame,
8717                "reserved_gate_not_retained",
8718                format!(
8719                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
8720                ),
8721            )?]);
8722        }
8723
8724        let response = ClientControlResponse::SupervisorAck {
8725            module_id,
8726            applied: true,
8727        };
8728        Ok(vec![control_response_body_frame(
8729            &frame,
8730            &response,
8731            "ClientControlResponse::SupervisorAck",
8732        )?])
8733    }
8734
8735    /// Compare the configured module set with the supervised one, changing
8736    /// nothing. The staged spec each supervised module holds is what the
8737    /// configured entry is compared with.
8738    fn plan_rescan(
8739        &self,
8740        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8741    ) -> Result<RescanPlan, String> {
8742        let mut current = BTreeMap::new();
8743        for module in self.supervisor.list() {
8744            let (spec, health) = module.configuration().map_err(|err| {
8745                format!(
8746                    "failed to read configuration for module_id '{}': {err}",
8747                    module.module_id()
8748                )
8749            })?;
8750            let enabled = module
8751                .status()
8752                .map_err(|err| {
8753                    format!(
8754                        "failed to read status for module_id '{}': {err}",
8755                        module.module_id()
8756                    )
8757                })?
8758                .enabled;
8759            current.insert(
8760                module.module_id().to_string(),
8761                (module, spec, health, enabled),
8762            );
8763        }
8764        let configured = configured_modules
8765            .into_iter()
8766            .map(|module| (module.module_id.clone(), module))
8767            .collect::<BTreeMap<_, _>>();
8768
8769        let added = configured
8770            .keys()
8771            .filter(|module_id| !current.contains_key(*module_id))
8772            .cloned()
8773            .collect::<Vec<_>>();
8774        let removed = current
8775            .keys()
8776            .filter(|module_id| !configured.contains_key(*module_id))
8777            .cloned()
8778            .collect::<Vec<_>>();
8779        let mut changed_pending_reload = Vec::new();
8780        let mut changed_pending = Vec::new();
8781        let mut configuration_changes = BTreeSet::new();
8782        let mut enabled_changes = BTreeSet::new();
8783        let mut unchanged = 0_u32;
8784        let publication = self
8785            .config_runtime
8786            .as_ref()
8787            .and_then(|owner| owner.polled_snapshot().ok());
8788
8789        for (module_id, configured_module) in &configured {
8790            let Some((module, current_spec, current_health, current_enabled)) =
8791                current.get(module_id)
8792            else {
8793                continue;
8794            };
8795            // Compare the whole launch spec so a future launch field cannot
8796            // accidentally become a live-only policy change. Health is stored
8797            // separately and applies live without replacing the process.
8798            let launch_changed = *current_spec != configured_module.module_spec();
8799            let configuration_changed =
8800                launch_changed || *current_health != configured_module.health;
8801            let pending = self.pending_module_fields(
8802                module,
8803                Some(configured_module),
8804                configuration_changed,
8805                Self::registration_channel_in_force(
8806                    publication
8807                        .as_ref()
8808                        .and_then(|view| view.registration_declaration(module_id))
8809                        .as_deref(),
8810                    publication
8811                        .as_ref()
8812                        .and_then(|view| match &view.approved_daemon {
8813                            crate::config_gate::runtime::ApprovedDaemonView::Approved {
8814                                config,
8815                                ..
8816                            } => config
8817                                .modules
8818                                .iter()
8819                                .find(|entry| entry.module_id == *module_id)
8820                                .map(|entry| entry.registration_channel),
8821                            _ => None,
8822                        }),
8823                ),
8824            )?;
8825            let has_pending = !pending.is_empty();
8826            if has_pending {
8827                changed_pending.push(subc_control::SupervisorPendingModule {
8828                    module_id: module_id.clone(),
8829                    fields: pending
8830                        .into_iter()
8831                        .map(|mut field| {
8832                            field.value = None;
8833                            field
8834                        })
8835                        .collect(),
8836                });
8837            }
8838            // Applying a rescan also replaces the recorded config value, even
8839            // when an operator already put the runtime in the requested state.
8840            let enabled_changed = *current_enabled != configured_module.enabled
8841                || self
8842                    .capability_evaluator
8843                    .configured_enabled(module_id)
8844                    .is_some_and(|enabled| enabled != configured_module.enabled);
8845            if configuration_changed {
8846                configuration_changes.insert(module_id.clone());
8847            }
8848            if launch_changed {
8849                changed_pending_reload.push(module_id.clone());
8850            }
8851            if enabled_changed {
8852                enabled_changes.insert(module_id.clone());
8853            }
8854            if !configuration_changed && !enabled_changed && !has_pending {
8855                unchanged = unchanged.saturating_add(1);
8856            }
8857        }
8858        Ok(RescanPlan {
8859            current,
8860            configured,
8861            added,
8862            removed,
8863            changed_pending_reload,
8864            changed_pending,
8865            configuration_changes,
8866            enabled_changes,
8867            unchanged,
8868        })
8869    }
8870
8871    /// Reconcile the running module set against the configured one.
8872    ///
8873    /// With `preview` set, the diff is computed and returned WITHOUT applying any
8874    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
8875    /// deliberately shares this function with the executing path rather than
8876    /// computing the same diff somewhere else -- two implementations of one
8877    /// decision agree until they do not, and the whole value of a preview is that
8878    /// it describes the operation that will actually run.
8879    async fn reconcile_supervised_modules(
8880        &self,
8881        supervisor: &Supervisor,
8882        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
8883        preview: bool,
8884        requester: Principal,
8885    ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
8886        let plan = self.plan_rescan(configured_modules)?;
8887
8888        // Everything above this point is pure computation over two snapshots.
8889        // Everything below MUTATES. The preview returns here so the boundary is a
8890        // single early return rather than a condition repeated at each mutation
8891        // site, where one missed guard would apply part of a change the caller was
8892        // told would not happen.
8893        if preview {
8894            return Ok((plan.preview_result(), Vec::new()));
8895        }
8896        let changed = plan
8897            .configuration_changes
8898            .union(&plan.enabled_changes)
8899            .cloned()
8900            .collect::<Vec<_>>();
8901        let RescanPlan {
8902            current,
8903            configured,
8904            added,
8905            removed,
8906            changed_pending_reload,
8907            changed_pending,
8908            configuration_changes,
8909            enabled_changes,
8910            unchanged,
8911        } = plan;
8912        // The staged generation moves before the first change rather than
8913        // after the last, so a rescan that fails partway still invalidates
8914        // every earlier `supervisor.rescan_checked` preview: part of its diff
8915        // may have run.
8916        if !(added.is_empty()
8917            && removed.is_empty()
8918            && configuration_changes.is_empty()
8919            && changed_pending.is_empty()
8920            && enabled_changes.is_empty())
8921        {
8922            if let Some(context) = &self.rescan {
8923                context
8924                    .generation
8925                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
8926            }
8927        }
8928
8929        for module_id in &removed {
8930            let module = &current
8931                .get(module_id)
8932                .expect("removed module came from current supervisor state")
8933                .0;
8934            module
8935                .retire_requested(Some(requester.clone()))
8936                .await
8937                .map_err(|err| {
8938                    format!("failed to retire module_id '{module_id}' during rescan: {err}")
8939                })?;
8940            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
8941            //
8942            // `handle_route_open` resolves an absent module in three steps:
8943            // registry, then supervisor status, then tombstone. Retiring first
8944            // opens a window where ALL THREE ARE ABSENT -- the registry entry
8945            // went with the teardown above, the supervisor entry went with
8946            // `retire`, and the tombstone does not exist yet -- so a route.open
8947            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
8948            // it") for a module that was deliberately removed and whose caller
8949            // should get `module_removed` (TERMINAL, carrying a removal age).
8950            //
8951            // Writing the tombstone first closes it: during the window the
8952            // supervisor entry still answers, so the caller gets
8953            // `target_unavailable` -- retryable, and TRUE, because the module
8954            // is mid-teardown. After both statements it is `module_removed`.
8955            // No instant remains where a removed module reads as one that
8956            // never existed.
8957            //
8958            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
8959            // the absence of a test beside a fix invites deletion: these are
8960            // two sync statements with no await between them, so reaching the
8961            // window needs a second worker thread to land exactly between them
8962            // and there is no hook to force it. MEASURED: the 25 daemon_config
8963            // tests pass identically with the old order and the new one, so
8964            // the existing suite cannot see this and a green run is not
8965            // evidence either way. What the suite does hold is the
8966            // post-condition -- a removed module answers `module_removed` --
8967            // which this preserves.
8968            //
8969            // Found by an Athena panel reading the shipped tree against a
8970            // design note (2026-09-19), as the one concrete instance of that
8971            // note's class that survived contact with source. Direction is
8972            // benign: retryable where terminal was intended, never the reverse.
8973            self.supervisor.record_rescan_removal(module_id);
8974            self.supervisor.retire(module_id);
8975            self.route_outages.forget(module_id);
8976        }
8977
8978        for module_id in configured.keys() {
8979            let Some((module, _, _, current_enabled)) = current.get(module_id) else {
8980                continue;
8981            };
8982            let configured_module = configured
8983                .get(module_id)
8984                .expect("configured module id came from configured map");
8985            if configuration_changes.contains(module_id) {
8986                module
8987                    .update_configuration(
8988                        configured_module.module_spec(),
8989                        configured_module.health.clone(),
8990                        configured_module.drain_timeout_ms,
8991                    )
8992                    .await
8993                    .map_err(|err| {
8994                        format!(
8995                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
8996                        )
8997                    })?;
8998            }
8999            // Config-only drift is applied by the evaluator's config update;
9000            // do not turn it into a runtime start or stop.
9001            if *current_enabled != configured_module.enabled {
9002                // A rescan that starts or stops a module applies an operator's
9003                // edit to the config, so the resulting outage was asked for.
9004                self.route_outages.mark_operator_action(module_id);
9005                module
9006                    .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
9007                    .await
9008                    .map_err(|err| {
9009                        self.route_outages.operator_action_ended_unrefused(module_id);
9010                        format!(
9011                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
9012                            configured_module.enabled
9013                        )
9014                    })?;
9015            }
9016        }
9017
9018        for module_id in &added {
9019            let configured_module = configured
9020                .get(module_id)
9021                .expect("added module id came from configured map");
9022            supervisor
9023                .supervise_configured_requested_async(
9024                    configured_module.module_spec(),
9025                    configured_module.enabled,
9026                    configured_module.health.clone(),
9027                    configured_module.drain_timeout_ms,
9028                    configured_module.restart,
9029                    configured_module.register_timeout,
9030                    Some(requester.clone()),
9031                )
9032                .await
9033                .map_err(|err| {
9034                    format!("failed to add module_id '{module_id}' during rescan: {err}")
9035                })?;
9036        }
9037
9038        if let Some(context) = &self.rescan {
9039            let mut entries = context
9040                .module_entries
9041                .lock()
9042                .unwrap_or_else(|e| e.into_inner());
9043            entries.record(&configured, &added, &removed);
9044        }
9045
9046        Ok((
9047            SupervisorRescanResult {
9048                added,
9049                removed,
9050                changed_pending_reload,
9051                changed_pending,
9052                enabled_changes: enabled_changes.iter().cloned().collect(),
9053                unchanged,
9054                preview: false,
9055                // Filled by the caller, which is the only layer that can see the
9056                // previous config to diff against.
9057                restart_required: Vec::new(),
9058                capability_warnings: Vec::new(),
9059                module_actions: Vec::new(),
9060                earlier_pending: Vec::new(),
9061                expect: None,
9062            },
9063            changed,
9064        ))
9065    }
9066
9067    async fn handle_supervisor_set_enabled(
9068        &self,
9069        frame: Frame,
9070        module_id: String,
9071        enabled: bool,
9072        requester: Principal,
9073    ) -> Result<Vec<Frame>, RouterError> {
9074        let operation_lock = self.supervisor.operation_lock();
9075        let _operation_guard = operation_lock.lock().await;
9076        let Some(module) = self.supervisor.get(&module_id) else {
9077            return Ok(vec![control_error_frame(
9078                &frame,
9079                "unknown_module",
9080                format!("module_id '{module_id}' is not supervised"),
9081            )?]);
9082        };
9083
9084        // Enabling counts as well as disabling: a module an operator starts
9085        // is refused until it registers, and that wait was asked for.
9086        self.route_outages.mark_operator_action(&module_id);
9087        let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
9088            Ok(applied) => applied,
9089            Err(err) => {
9090                self.route_outages
9091                    .operator_action_ended_unrefused(&module_id);
9092                return Ok(vec![control_error_frame(
9093                    &frame,
9094                    if err.is_config_gate_error() {
9095                        "config_gated"
9096                    } else {
9097                        "target_unavailable"
9098                    },
9099                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
9100                )?]);
9101            }
9102        };
9103        if !applied {
9104            // Already in the requested state: nothing was made unavailable,
9105            // so the mark must not outlive this request.
9106            self.route_outages
9107                .operator_action_ended_unrefused(&module_id);
9108        }
9109
9110        self.capability_evaluator.wake_deadline_loop();
9111        self.refresh_capability_requirements();
9112        let response = ClientControlResponse::SupervisorAck { module_id, applied };
9113        Ok(vec![control_response_body_frame(
9114            &frame,
9115            &response,
9116            "ClientControlResponse::SupervisorAck",
9117        )?])
9118    }
9119
9120    async fn handle_supervisor_health_probe(
9121        &self,
9122        ctx: &RouteCtx,
9123        frame: Frame,
9124        module_id: String,
9125    ) -> Result<Vec<Frame>, RouterError> {
9126        self.refresh_capability_requirements();
9127        let Some(registration) = self
9128            .registry
9129            .get_module(&module_id)
9130            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
9131        else {
9132            return Ok(vec![control_error_frame(
9133                &frame,
9134                "unknown_module",
9135                format!("module_id '{module_id}' is not registered"),
9136            )?]);
9137        };
9138
9139        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
9140        // named for it. Making `module_registration_grants_op` return false
9141        // unconditionally reddens five tests, and every one is named for something
9142        // else -- capability relay, probe/bind demultiplexing, supervision-only
9143        // probing. They exercise a successful advertisement check on the way to their
9144        // own subject.
9145        //
9146        // Real protection, fragile in a specific way: narrowing any of those tests to
9147        // focus on its stated subject would silently remove coverage nobody knows
9148        // they are carrying. Recorded here rather than as a sixth test, because the
9149        // useful fact is WHICH tests hold the guard up -- a new test would add
9150        // coverage without telling the next person what the existing ones quietly do.
9151        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
9152        {
9153            return Ok(vec![control_error_frame(
9154                &frame,
9155                "health_not_advertised",
9156                format!("module_id '{module_id}' did not advertise health.check"),
9157            )?]);
9158        }
9159
9160        self.observe_control_wait(
9161            ctx,
9162            Some(&registration.manifest.module_id),
9163            ops::SUPERVISOR_HEALTH_PROBE,
9164            async {
9165                let deadline = Instant::now() + self.health_probe_timeout;
9166                let pending = match self.forwarding.begin_module_control_rpc_for(
9167                    &module_id,
9168                    MODULE_CONTROL_OP_HEALTH_CHECK,
9169                    deadline,
9170                ) {
9171                    Ok(pending) => pending,
9172                    Err(err) => {
9173                        return Ok(vec![control_error_frame(
9174                            &frame,
9175                            forwarding_error_code(&err),
9176                            err.to_string(),
9177                        )?])
9178                    }
9179                };
9180
9181                let PendingModuleControlRpc {
9182                    endpoint,
9183                    module_sink,
9184                    negotiated_ver,
9185                    corr: probe_corr,
9186                    receiver,
9187                } = pending;
9188                let mut guard =
9189                    ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
9190                let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
9191                    .map_err(|err| {
9192                        RouterError::backend(
9193                            0,
9194                            frame.header.corr,
9195                            format!("failed to encode health.check request: {err}"),
9196                        )
9197                    })?;
9198                let probe_frame = Frame::build_with_version(
9199                    negotiated_ver,
9200                    FrameType::Request,
9201                    control_flags(),
9202                    0,
9203                    0,
9204                    probe_corr,
9205                    probe_body,
9206                )
9207                .map_err(RouterError::FrameBuild)?;
9208
9209                // Queue admission and the reply share one deadline, so a module that
9210                // stops reading cannot park the requesting connection's reader forever.
9211                match timeout_at(deadline, module_sink.send(probe_frame)).await {
9212                    Ok(Ok(())) => {}
9213                    Ok(Err(err)) => {
9214                        return Ok(vec![control_error_frame(
9215                            &frame,
9216                            "target_unavailable",
9217                            err.to_string(),
9218                        )?]);
9219                    }
9220                    Err(_) => {
9221                        return Ok(vec![control_error_frame(
9222                            &frame,
9223                            "module_timeout",
9224                            format!(
9225                                "module_id '{module_id}' did not answer health.check within {:?}",
9226                                self.health_probe_timeout
9227                            ),
9228                        )?]);
9229                    }
9230                }
9231
9232                match timeout_at(deadline, receiver).await {
9233                    Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => {
9234                        Ok(vec![control_error_frame(
9235                            &frame,
9236                            "invalid_control_body",
9237                            "health.check returned an unexpected config response",
9238                        )?])
9239                    }
9240                    Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
9241                        guard.disarm();
9242                        let Some(report) = response.health_report() else {
9243                            return Ok(vec![control_error_frame(
9244                                &frame,
9245                                "invalid_control_body",
9246                                "health.check RPC returned a non-health response",
9247                            )?]);
9248                        };
9249                        // Metrics go out whole here. The supervisor's cached snapshot
9250                        // caps this blob (see truncate_health_metrics), and this path
9251                        // exists precisely to answer without that cap -- so applying it
9252                        // here would leave no way to see what the cached view drops.
9253                        let HealthReport {
9254                            status,
9255                            detail,
9256                            metrics,
9257                        } = report;
9258                        let capability_detail = self
9259                            .capability_evaluator
9260                            .required_problem_detail(&module_id);
9261                        let response = ClientControlResponse::SupervisorHealthProbe {
9262                            module_id,
9263                            status,
9264                            detail: append_capability_problem_detail(detail, capability_detail),
9265                            metrics,
9266                        };
9267                        Ok(vec![control_response_body_frame(
9268                            &frame,
9269                            &response,
9270                            "ClientControlResponse::SupervisorHealthProbe",
9271                        )?])
9272                    }
9273                    Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
9274                        guard.disarm();
9275                        // The module refused the health probe with its own body, so
9276                        // the refusal is relayed without the daemon-origin flag
9277                        // (bit 6): a client must not read it as the daemon's own
9278                        // refusal, which it may treat as never forwarded.
9279                        Ok(vec![control_relayed_error_body_frame(&frame, body)?])
9280                    }
9281                    Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
9282                        guard.disarm();
9283                        Ok(vec![control_error_frame(
9284                            &frame,
9285                            "target_unavailable",
9286                            message,
9287                        )?])
9288                    }
9289                    Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
9290                        guard.disarm();
9291                        Ok(vec![control_error_frame(
9292                            &frame,
9293                            "invalid_control_body",
9294                            message,
9295                        )?])
9296                    }
9297                    Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
9298                        guard.disarm();
9299                        Ok(vec![control_error_frame(
9300                            &frame,
9301                            "invalid_control_body",
9302                            format!("expected module-control op '{expected}', got '{actual}'"),
9303                        )?])
9304                    }
9305                    Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
9306                        guard.disarm();
9307                        Ok(vec![control_error_frame(
9308                            &frame,
9309                            "module_timeout",
9310                            format!(
9311                                "module_id '{module_id}' answered health.check after {:?}",
9312                                self.health_probe_timeout
9313                            ),
9314                        )?])
9315                    }
9316                    Ok(Err(_)) => Ok(vec![control_error_frame(
9317                        &frame,
9318                        "target_unavailable",
9319                        "health.check waiter was canceled before the module responded",
9320                    )?]),
9321                    Err(_) => Ok(vec![control_error_frame(
9322                        &frame,
9323                        "module_timeout",
9324                        format!(
9325                            "module_id '{module_id}' did not answer health.check within {:?}",
9326                            self.health_probe_timeout
9327                        ),
9328                    )?]),
9329                }
9330            },
9331        )
9332        .await
9333    }
9334
9335    fn supervisor_status(
9336        &self,
9337        module_id: &str,
9338        corr: u64,
9339    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
9340        self.supervisor
9341            .get(module_id)
9342            .map(|module| {
9343                let warming = module.is_warming_for_control("status").map_err(|err| {
9344                    RouterError::backend(
9345                        0,
9346                        corr,
9347                        format!(
9348                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
9349                        ),
9350                    )
9351                })?;
9352                module.status_for_control("status").map_err(|err| {
9353                    RouterError::backend(
9354                        0,
9355                        corr,
9356                        format!(
9357                            "failed to read supervisor status for module_id '{module_id}': {err}"
9358                        ),
9359                    )
9360                }).map(|status| (status, warming))
9361            })
9362            .transpose()
9363    }
9364
9365    fn guard_module_control_op(
9366        &self,
9367        frame: &Frame,
9368        module_id: &str,
9369        op: &str,
9370    ) -> Result<Option<Frame>, RouterError> {
9371        if self.module_grants_op(module_id, op, frame.header.corr)? {
9372            return Ok(None);
9373        }
9374
9375        Ok(Some(control_error_frame(
9376            frame,
9377            "op_not_allowed",
9378            format!("module_id '{module_id}' did not grant control op '{op}'"),
9379        )?))
9380    }
9381
9382    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
9383        let Some(registration) = self
9384            .registry
9385            .get_module(module_id)
9386            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
9387        else {
9388            return Ok(false);
9389        };
9390        Ok(module_registration_grants_op(&registration.control_ops, op))
9391    }
9392
9393    fn handle_status_update(
9394        &self,
9395        endpoint: ModuleEndpointId,
9396        frame: Frame,
9397    ) -> Result<Vec<Frame>, RouterError> {
9398        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
9399            Ok(update) => update,
9400            Err(err) => {
9401                // Forward-compat: a newer module may push a channel-0 op this subc
9402                // version doesn't know. The control contract says unknown push ops
9403                // are IGNORED, never answered with an error. Only a malformed body
9404                // for an op we DO know is a real error worth surfacing.
9405                if is_known_module_push_op(&frame.body) {
9406                    return Ok(vec![control_error_frame(
9407                        &frame,
9408                        "invalid_control_body",
9409                        format!("malformed module control push body: {err}"),
9410                    )?]);
9411                }
9412                return Ok(Vec::new());
9413            }
9414        };
9415
9416        match update {
9417            ModuleControlPush::RouteStatus {
9418                route_channel,
9419                route_epoch,
9420                status,
9421            } => {
9422                self.forwarding
9423                    .cache_status(endpoint, route_channel, route_epoch, status)
9424                    .map_err(RouterError::Forwarding)?;
9425            }
9426        }
9427        Ok(Vec::new())
9428    }
9429
9430    fn handle_route_poll(
9431        &self,
9432        ctx: &RouteCtx,
9433        frame: Frame,
9434        route_channel: u16,
9435        route_epoch: u32,
9436        kind: PollKind,
9437    ) -> Result<Vec<Frame>, RouterError> {
9438        let snapshot = self
9439            .forwarding
9440            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
9441            .map_err(RouterError::Forwarding)?;
9442        let response = match (kind, snapshot) {
9443            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
9444                ClientControlResponse::RoutePoll {
9445                    route_channel,
9446                    route_epoch,
9447                    status,
9448                    live: None,
9449                }
9450            }
9451            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9452                route_channel,
9453                route_epoch,
9454                status: None,
9455                live: None,
9456            },
9457            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
9458                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
9459                // is what makes reporting `true` correct rather than a
9460                // confident guess. `process_live` returns None only when the
9461                // module id has no supervisor snapshot at all -- an
9462                // externally-started module the daemon did not spawn -- and
9463                // for those the supervisor has no opinion to offer, ever. It
9464                // is never None for a supervised module in an unknown state:
9465                // a supervised module always has a snapshot, and the answer
9466                // comes from `state == Running && process_alive`.
9467                //
9468                // The route is Bound, so the module completed a HELLO on a
9469                // live connection; "the process this route points at is
9470                // running" is therefore attested by the binding rather than
9471                // assumed. Reporting `false` for an unsupervised module would
9472                // be the actual lie -- it would tell a client its healthy
9473                // route is dead because the daemon does not manage the
9474                // process.
9475                //
9476                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
9477                // module whose liveness is genuinely unknown, e.g. a snapshot
9478                // that has not been populated yet -- THIS DEFAULT BECOMES
9479                // WRONG and must split: unsupervised stays true, unknown
9480                // becomes null so the client can tell the two apart. The
9481                // response field is already `Option<bool>`, so the wire can
9482                // carry that distinction today.
9483                let live = self
9484                    .process_liveness
9485                    .as_ref()
9486                    .and_then(|source| source.process_live(&module_id))
9487                    .unwrap_or(true);
9488                ClientControlResponse::RoutePoll {
9489                    route_channel,
9490                    route_epoch,
9491                    status: None,
9492                    live: Some(live),
9493                }
9494            }
9495            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
9496                route_channel,
9497                route_epoch,
9498                status: None,
9499                live: Some(false),
9500            },
9501        };
9502
9503        Ok(vec![control_response_body_frame(
9504            &frame,
9505            &response,
9506            "ClientControlResponse::RoutePoll",
9507        )?])
9508    }
9509
9510    pub(crate) fn observe_module_control_completion(
9511        &self,
9512        completion: ModuleControlRpcCompletion,
9513    ) -> bool {
9514        match completion {
9515            ModuleControlRpcCompletion::Unknown => false,
9516            ModuleControlRpcCompletion::Settled => true,
9517            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
9518                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
9519                info!(
9520                    module_id = %module_id,
9521                    latency_ms,
9522                    "late health.check answer proves the module is alive"
9523                );
9524                match self
9525                    .supervisor
9526                    .record_late_health_answer(&module_id, latency_ms)
9527                {
9528                    Ok(true) => {}
9529                    Ok(false) => debug!(
9530                        module_id = %module_id,
9531                        latency_ms,
9532                        "late health.check answer has no active supervisor snapshot"
9533                    ),
9534                    Err(err) => warn!(
9535                        module_id = %module_id,
9536                        latency_ms,
9537                        error = %err,
9538                        "failed to record late health.check answer"
9539                    ),
9540                }
9541                true
9542            }
9543        }
9544    }
9545
9546    /// Decide whether a failure while settling a relayed `route.bind` belongs to
9547    /// the module connection whose frame is being handled, or to the client that
9548    /// relay was opened for.
9549    ///
9550    /// This runs on the MODULE connection's frame handler, where returning `Err`
9551    /// ends that connection -- and a module connection carries every client's
9552    /// routes to that module, so ending it costs the whole fleet its tools.
9553    /// `ConnectionClosing` carries the id of the connection that is closing, and
9554    /// when that id is a CLIENT's, the condition is entirely about that one
9555    /// client's route.open. A client-scoped condition has no authority over a
9556    /// shared module connection, so it is logged and the single relay is dropped:
9557    /// the client is going away, and `complete_pending_relay` already removed the
9558    /// relay before failing, so there is nothing left to settle. Anything that
9559    /// relay still reserved is released by that client's own connection teardown,
9560    /// which is already under way -- that is what "closing" means.
9561    ///
9562    /// Every other failure is a statement about THIS connection and stays fatal:
9563    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
9564    /// id in `ConnectionClosing` all mean this connection cannot keep serving
9565    /// frames correctly.
9566    fn refuse_to_end_module_connection_for_a_client(
9567        &self,
9568        module_connection_id: ConnectionId,
9569        corr: u64,
9570        err: ForwardingError,
9571    ) -> Result<(), RouterError> {
9572        if let ForwardingError::ConnectionClosing { connection_id } = err {
9573            if connection_id != module_connection_id {
9574                warn!(
9575                    module_connection_id = module_connection_id.get(),
9576                    client_connection_id = connection_id.get(),
9577                    corr,
9578                    "dropping a route.bind response for a closing client; the module connection keeps serving"
9579                );
9580                return Ok(());
9581            }
9582        }
9583        Err(RouterError::Forwarding(err))
9584    }
9585
9586    fn handle_module_relay_response(
9587        &self,
9588        connection_id: ConnectionId,
9589        frame: Frame,
9590    ) -> Result<Vec<Frame>, RouterError> {
9591        let mut secondary_error = None;
9592        let outcome = match frame.header.ty {
9593            FrameType::Response => match module_control_response_probe(&frame.body) {
9594                Ok(probe) if probe.op == "route.bind" => {
9595                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9596                        Ok(ModuleControlResponse::RouteBindAck {}) => {
9597                            RouteBindRelayOutcome::Accepted
9598                        }
9599                        Ok(other) => {
9600                            let message =
9601                                format!("route.bind response carried unexpected body: {other:?}");
9602                            secondary_error = Some(control_error_frame(
9603                                &frame,
9604                                "invalid_control_body",
9605                                message.clone(),
9606                            )?);
9607                            RouteBindRelayOutcome::ModuleGone(message)
9608                        }
9609                        Err(err) => {
9610                            let message = format!("malformed route.bind response body: {err}");
9611                            secondary_error = Some(control_error_frame(
9612                                &frame,
9613                                "invalid_control_body",
9614                                message.clone(),
9615                            )?);
9616                            RouteBindRelayOutcome::ModuleGone(message)
9617                        }
9618                    }
9619                }
9620                Ok(probe) => {
9621                    let outcome = if matches!(
9622                        probe.op.as_str(),
9623                        "config.review" | "consent/v1" | "projects.enumerate"
9624                    ) {
9625                        ModuleControlRpcOutcome::RawResponse(frame.body.clone())
9626                    } else {
9627                        match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
9628                            Ok(response) => ModuleControlRpcOutcome::Response(response),
9629                            Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9630                                "malformed {} response body: {err}",
9631                                probe.op
9632                            )),
9633                        }
9634                    };
9635                    let completion = self
9636                        .forwarding
9637                        .complete_module_control_rpc(
9638                            connection_id,
9639                            frame.header.corr,
9640                            Some(&probe.op),
9641                            outcome,
9642                        )
9643                        .map_err(RouterError::Forwarding)?;
9644                    if !self.observe_module_control_completion(completion) {
9645                        debug!(
9646                            connection_id = connection_id.get(),
9647                            corr = frame.header.corr,
9648                            op = %probe.op,
9649                            "dropping late or unknown module-control RPC response"
9650                        );
9651                    }
9652                    return Ok(Vec::new());
9653                }
9654                Err(err) => {
9655                    if let Some(expected_op) = self
9656                        .forwarding
9657                        .pending_module_control_op(connection_id, frame.header.corr)
9658                        .map_err(RouterError::Forwarding)?
9659                    {
9660                        let completion = self
9661                            .forwarding
9662                            .complete_module_control_rpc(
9663                                connection_id,
9664                                frame.header.corr,
9665                                None,
9666                                ModuleControlRpcOutcome::MalformedResponse(format!(
9667                                    "malformed {expected_op} response body: {err}"
9668                                )),
9669                            )
9670                            .map_err(RouterError::Forwarding)?;
9671                        if !self.observe_module_control_completion(completion) {
9672                            debug!(
9673                                connection_id = connection_id.get(),
9674                                corr = frame.header.corr,
9675                                "dropping late malformed module-control RPC response"
9676                            );
9677                        }
9678                        return Ok(Vec::new());
9679                    }
9680                    let message = format!("malformed route.bind response body: {err}");
9681                    secondary_error = Some(control_error_frame(
9682                        &frame,
9683                        "invalid_control_body",
9684                        message.clone(),
9685                    )?);
9686                    RouteBindRelayOutcome::ModuleGone(message)
9687                }
9688            },
9689            FrameType::Error => {
9690                if self
9691                    .forwarding
9692                    .pending_module_control_op(connection_id, frame.header.corr)
9693                    .map_err(RouterError::Forwarding)?
9694                    .is_some()
9695                {
9696                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
9697                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
9698                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
9699                            "malformed module-control ERROR body: {err}"
9700                        )),
9701                    };
9702                    let completion = self
9703                        .forwarding
9704                        .complete_module_control_rpc(
9705                            connection_id,
9706                            frame.header.corr,
9707                            None,
9708                            outcome,
9709                        )
9710                        .map_err(RouterError::Forwarding)?;
9711                    if !self.observe_module_control_completion(completion) {
9712                        debug!(
9713                            connection_id = connection_id.get(),
9714                            corr = frame.header.corr,
9715                            "dropping late or unknown module-control RPC error"
9716                        );
9717                    }
9718                    return Ok(Vec::new());
9719                }
9720                match serde_json::from_slice::<ErrorBody>(&frame.body) {
9721                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
9722                    Err(err) => {
9723                        let message = format!("malformed route.bind ERROR body: {err}");
9724                        secondary_error = Some(control_error_frame(
9725                            &frame,
9726                            "invalid_control_body",
9727                            message.clone(),
9728                        )?);
9729                        RouteBindRelayOutcome::ModuleGone(message)
9730                    }
9731                }
9732            }
9733            ty => {
9734                return Ok(vec![control_error_frame(
9735                    &frame,
9736                    "unsupported_control_frame",
9737                    format!("unsupported module channel-0 frame {ty:?}"),
9738                )?])
9739            }
9740        };
9741
9742        let settled =
9743            self.forwarding
9744                .complete_pending_relay(connection_id, frame.header.corr, outcome);
9745        let completion = match settled {
9746            Ok(completion) => completion,
9747            Err(err) => {
9748                self.refuse_to_end_module_connection_for_a_client(
9749                    connection_id,
9750                    frame.header.corr,
9751                    err,
9752                )?;
9753                return Ok(secondary_error.into_iter().collect());
9754            }
9755        };
9756        if let Some(target) = completion.abandoned.as_ref() {
9757            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
9758        }
9759        if !completion.settled {
9760            debug!(
9761                connection_id = connection_id.get(),
9762                corr = frame.header.corr,
9763                frame_type = ?frame.header.ty,
9764                "dropping late or unknown route.bind relay response"
9765            );
9766        }
9767        Ok(secondary_error.into_iter().collect())
9768    }
9769
9770    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
9771        debug!(connection_id = connection_id.get(), "handling GOODBYE");
9772        // GOODBYE ends the connection's logical session even when its socket
9773        // stays open. Use disconnect teardown so verdicts, client notices and
9774        // scope authority are released at the same lifecycle boundary.
9775        self.cleanup_connection_with_end_reason(
9776            connection_id,
9777            RegistrationEndReason::ExplicitGoodbye,
9778        )
9779        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
9780        Ok(Vec::new())
9781    }
9782}
9783
9784impl Default for ControlHandler {
9785    fn default() -> Self {
9786        Self::new(Arc::new(Registry::default()))
9787    }
9788}
9789
9790impl crate::supervise::SwapPromotionObserver for ControlHandler {
9791    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
9792        self.apply_registration_capabilities(registration);
9793    }
9794}
9795
9796fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
9797    CapabilityRequirementStatus {
9798        consumer: status.consumer,
9799        capability: status.capability,
9800        need: match status.need {
9801            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
9802            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
9803        },
9804        verdict: status.verdict.as_str().to_string(),
9805        episode_seq: status.episode_seq,
9806        config_satisfiable: status.config_satisfiable,
9807        runtime_available: status.runtime_available,
9808        detail: status.detail,
9809    }
9810}
9811
9812fn append_capability_problem_detail(
9813    detail: Option<String>,
9814    capability_detail: Option<String>,
9815) -> Option<String> {
9816    match (detail, capability_detail) {
9817        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
9818        (Some(detail), None) => Some(detail),
9819        (None, Some(capability_detail)) => Some(capability_detail),
9820        (None, None) => None,
9821    }
9822}
9823
9824fn subc_ops() -> Vec<String> {
9825    SUBC_CONTROL_OPS
9826        .iter()
9827        .map(|op| (*op).to_string())
9828        .collect()
9829}
9830
9831fn module_subc_ops() -> Vec<String> {
9832    SUBC_CONTROL_OPS
9833        .iter()
9834        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
9835        .map(|op| (*op).to_string())
9836        .collect()
9837}
9838
9839#[cfg(test)]
9840fn module_baseline_control_ops() -> Vec<String> {
9841    MODULE_BASELINE_CONTROL_OPS
9842        .iter()
9843        .map(|op| (*op).to_string())
9844        .collect()
9845}
9846
9847fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
9848    let mut seen = HashSet::new();
9849    let mut effective = Vec::new();
9850    for op in MODULE_BASELINE_CONTROL_OPS {
9851        if seen.insert((*op).to_string()) {
9852            effective.push((*op).to_string());
9853        }
9854    }
9855    for op in declared.unwrap_or_default() {
9856        if seen.insert(op.clone()) {
9857            effective.push(op);
9858        }
9859    }
9860    effective
9861}
9862
9863fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
9864    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
9865}
9866
9867fn target_module_id(target: &RouteTarget) -> &str {
9868    match target {
9869        RouteTarget::ToolProvider { module_id }
9870        | RouteTarget::ManagementSurface { module_id }
9871        | RouteTarget::InternalService { module_id, .. } => module_id,
9872    }
9873}
9874
9875fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
9876    roles.iter().any(|role| match (target, role) {
9877        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
9878        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
9879        (
9880            RouteTarget::InternalService { service_id, .. },
9881            ProviderRole::InternalService {
9882                service_id: provided,
9883                ..
9884            },
9885        ) => service_id == provided,
9886        _ => false,
9887    })
9888}
9889
9890fn is_routable_role(role: &ProviderRole) -> bool {
9891    matches!(
9892        role,
9893        ProviderRole::ToolProvider { .. }
9894            | ProviderRole::ManagementSurface { .. }
9895            | ProviderRole::InternalService { .. }
9896    )
9897}
9898
9899#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9900enum ControlRequestBodyError {
9901    UnknownOp,
9902    InvalidBody,
9903}
9904
9905#[derive(Debug, Deserialize)]
9906struct ControlOpProbe {
9907    op: String,
9908}
9909
9910fn module_control_response_probe(body: &[u8]) -> Result<ControlOpProbe, serde_json::Error> {
9911    serde_json::from_slice(body).or_else(|original| {
9912        #[derive(Deserialize)]
9913        struct ConsentProbe {
9914            protocol: String,
9915        }
9916        match serde_json::from_slice::<ConsentProbe>(body) {
9917            Ok(probe) if probe.protocol == subc_protocol::consent::CONSENT_V1 => {
9918                Ok(ControlOpProbe { op: probe.protocol })
9919            }
9920            _ => Err(original),
9921        }
9922    })
9923}
9924
9925/// Channel-0 push ops this subc version understands. A push whose `op` is not in
9926/// this set is treated as a forward-compat unknown and ignored rather than errored.
9927const MODULE_PUSH_OPS: &[&str] = &["route.status"];
9928
9929fn is_known_module_push_op(body: &[u8]) -> bool {
9930    serde_json::from_slice::<ControlOpProbe>(body)
9931        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
9932        .unwrap_or(false)
9933}
9934
9935fn is_known_module_request_op(body: &[u8]) -> bool {
9936    serde_json::from_slice::<ControlOpProbe>(body)
9937        .map(|probe| is_module_to_subc_op(&probe.op))
9938        .unwrap_or(false)
9939}
9940
9941fn is_module_to_subc_op(op: &str) -> bool {
9942    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
9943}
9944
9945fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
9946    debug!(
9947        op = %op,
9948        connection_id = connection_id.get(),
9949        corr,
9950        "control dispatch"
9951    );
9952}
9953
9954fn log_slow_control_dispatch(
9955    dispatch_started_at: Option<StdInstant>,
9956    op: &'static str,
9957    connection_id: ConnectionId,
9958    corr: u64,
9959    route_open_timings: Option<&RouteOpenTimings>,
9960) {
9961    let Some(dispatch_started_at) = dispatch_started_at else {
9962        return;
9963    };
9964    // Use the same monotonic clock as the phase timers and relay deadline.
9965    let elapsed = Instant::from_std(dispatch_started_at).elapsed();
9966    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
9967        if let Some(timings) = route_open_timings {
9968            warn!(
9969                op = %op,
9970                connection_id = connection_id.get(),
9971                corr,
9972                elapsed_ms = elapsed.as_millis() as u64,
9973                module_id = ?timings.module_id,
9974                principal = %timings.principal,
9975                admission_ms = timings.admission.as_millis() as u64,
9976                project_root_ms = timings.project_root.as_millis() as u64,
9977                egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
9978                bind_send_ms = timings.bind_send.as_millis() as u64,
9979                bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
9980                "slow control dispatch"
9981            );
9982            return;
9983        }
9984        warn!(
9985            op = %op,
9986            connection_id = connection_id.get(),
9987            corr,
9988            elapsed_ms = elapsed.as_millis() as u64,
9989            "slow control dispatch"
9990        );
9991    }
9992}
9993
9994/// A test stand-in for the OS caller lookup, given the recorded endpoints.
9995#[cfg(test)]
9996type TestCallerLookup =
9997    Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
9998
9999/// Every new client operation needs an explicit privilege decision here.
10000fn privileged_operation(request: &impl PrivilegedRequest) -> Option<PrivilegedOperation<'_>> {
10001    request.privileged_operation()
10002}
10003
10004trait PrivilegedRequest {
10005    fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>>;
10006}
10007
10008impl PrivilegedRequest for ClientControlRequest {
10009    fn privileged_operation(&self) -> Option<PrivilegedOperation<'_>> {
10010        let request = self;
10011        match request {
10012        ClientControlRequest::SupervisorRestart { module_id, .. } => {
10013            Some(PrivilegedOperation::Restart { module_id })
10014        }
10015        ClientControlRequest::SupervisorSwap { module_id, .. } => {
10016            Some(PrivilegedOperation::Swap { module_id })
10017        }
10018        ClientControlRequest::SupervisorReload { module_id } => {
10019            Some(PrivilegedOperation::Reload { module_id })
10020        }
10021        ClientControlRequest::SupervisorRescan { preview } => {
10022            if *preview {
10023                None
10024            } else {
10025                Some(PrivilegedOperation::Rescan)
10026            }
10027        }
10028        // Only the well-formed apply asks for approval. A preview changes
10029        // nothing, and any other combination is refused before it touches
10030        // anything, so prompting the operator for it would ask about a request
10031        // that cannot run.
10032        ClientControlRequest::SupervisorRescanChecked {
10033            preview,
10034            candidate,
10035            expect,
10036        } => (!*preview && candidate.is_none() && expect.is_some())
10037            .then_some(PrivilegedOperation::Rescan),
10038        ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
10039            Some(PrivilegedOperation::SetEnabled {
10040                module_id,
10041                enabled: *enabled,
10042            })
10043        }
10044        ClientControlRequest::SupervisorReleaseReserved { module_id } => {
10045            Some(PrivilegedOperation::ReleaseReserved { module_id })
10046        }
10047        ClientControlRequest::ServerDescribe {}
10048        // The announcement uses the same client admission as supervisor.restart,
10049        // but cannot stop anything itself and needs no operator confirmation.
10050        | ClientControlRequest::DaemonShutdownIntent { .. }
10051        | ClientControlRequest::CatalogList { .. }
10052        | ClientControlRequest::RouteOpen { .. }
10053        | ClientControlRequest::RoutePoll { .. }
10054        | ClientControlRequest::SupervisorList {}
10055        | ClientControlRequest::SupervisorHealth {}
10056        | ClientControlRequest::SupervisorHealthProbe { .. }
10057        | ClientControlRequest::SupervisorRoutes { .. }
10058        | ClientControlRequest::SupervisorProvenance { .. }
10059        | ClientControlRequest::SupervisorStderrTail { .. }
10060        | ClientControlRequest::SupervisorTerminals { .. }
10061        | ClientControlRequest::SupervisorSpawnSnapshot {}
10062        | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
10063    }
10064    }
10065}
10066
10067fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
10068    match request {
10069        ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
10070        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
10071        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
10072        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
10073        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
10074        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
10075        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
10076        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
10077        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
10078        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
10079        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
10080        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
10081        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
10082        ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
10083        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
10084        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
10085        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
10086        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
10087        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
10088        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
10089        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
10090    }
10091}
10092
10093/// The caller of one `route.open`, as its `route.open refused` line names it.
10094///
10095/// A connection id alone cannot name a caller after the fact: ids restart at
10096/// every daemon restart, so the same number belongs to different modules
10097/// before and after one. These three fields are what the refused line needs
10098/// to be attributable on its own, and they are rendered with the same helpers
10099/// as the `route.open accepted` line so both directions grep alike.
10100struct RouteOpenCaller {
10101    /// The attested principal (`direct`, `reserved:<module>`), or `unattested`
10102    /// when the request presented a consumer identity whose launch nonce did
10103    /// not verify. The claimed module id is never printed as a principal.
10104    principal: String,
10105    /// `none` for an unscoped open; the admitted stamp's label once scope
10106    /// admission issued one; before that, the requested selector's label.
10107    scope: String,
10108    /// The bind identity's harness, Debug-quoted and bounded like a scope ref,
10109    /// because it is a string the caller chose.
10110    harness: String,
10111}
10112
10113impl RouteOpenCaller {
10114    fn new(
10115        principal: Option<&Principal>,
10116        scope: Option<&ScopeSelector>,
10117        identity: &BindIdentity,
10118    ) -> Self {
10119        Self {
10120            principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
10121            scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
10122            harness: bounded_log_text(&identity.harness),
10123        }
10124    }
10125
10126    /// For a refusal that has no parsed request at hand: names whatever the
10127    /// frame still says, and `unknown` where it says nothing parseable.
10128    fn unknown() -> Self {
10129        Self {
10130            principal: "unknown".to_string(),
10131            scope: "unknown".to_string(),
10132            harness: "unknown".to_string(),
10133        }
10134    }
10135}
10136
10137/// Longest client-chosen text (a scope ref, a harness) a route.open log line
10138/// prints before cutting it.
10139const MAX_LOGGED_TEXT_CHARS: usize = 64;
10140
10141/// `value` Debug-quoted, so control characters land escaped and cannot forge
10142/// a log line, and cut to [`MAX_LOGGED_TEXT_CHARS`] characters with a trailing
10143/// `…` inside the quotes so one request cannot flood the line.
10144fn bounded_log_text(value: &str) -> String {
10145    let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
10146    if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
10147        bounded.push('…');
10148    }
10149    format!("{bounded:?}")
10150}
10151
10152/// The `scope=` field of the `route.open accepted` line, from the stamp that
10153/// admission issued: `<kind>/<owner>/<ref>@<epoch>`. The ref is Debug-quoted
10154/// so a ref holding control characters cannot forge a log line, and cut to 64
10155/// characters with a trailing `…` to bound the line. The bind proof and the
10156/// scope's attributes never appear.
10157fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
10158    use subc_protocol::scope::ScopeKind;
10159    let kind = match stamp.kind {
10160        ScopeKind::Head => "head",
10161        ScopeKind::Worker => "worker",
10162        ScopeKind::Ephemeral => "ephemeral",
10163        ScopeKind::Federated => "federated",
10164        _ => "unknown",
10165    };
10166    format!(
10167        "{kind}/{}/{}@{}",
10168        principal_label(&stamp.owner),
10169        bounded_log_text(&stamp.scope_ref),
10170        stamp.scope_epoch
10171    )
10172}
10173
10174/// The `scope=` field of a `route.open refused` line for an open refused
10175/// before scope admission issued a stamp: `requested/<owner>/<ref>@<epoch>`.
10176/// The kind is not known yet, so the literal `requested` stands in its place.
10177/// The owner is Debug-quoted as well as the ref, because unlike a stamp's
10178/// owner it has not been checked against the configured modules: a quoted
10179/// owner means the value was never validated. The bind proof never appears.
10180fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
10181    let epoch = selector
10182        .scope_epoch
10183        .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
10184    format!(
10185        "requested/{}/{}@{epoch}",
10186        bounded_log_text(&principal_label(&selector.owner)),
10187        bounded_log_text(&selector.scope_ref),
10188    )
10189}
10190
10191fn principal_label(principal: &Principal) -> String {
10192    match principal {
10193        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
10194        Principal::Direct => "direct".to_string(),
10195        other => format!("{other:?}"),
10196    }
10197}
10198
10199fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
10200    match request {
10201        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
10202        ModuleControlRequestFromModule::CatalogUpdateRoleDeclarations { .. } => {
10203            MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS
10204        }
10205        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
10206        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
10207        ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
10208        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
10209        ModuleControlRequestFromModule::RouteAuthorize { .. } => {
10210            subc_protocol::scope::ROUTE_AUTHORIZE_OP
10211        }
10212        ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
10213        _ => "unsupported",
10214    }
10215}
10216
10217fn parse_client_control_request(
10218    body: &[u8],
10219) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
10220    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
10221        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10222            Ok(probe)
10223                if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
10224                    || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
10225            {
10226                ControlRequestBodyError::InvalidBody
10227            }
10228            Ok(_) => ControlRequestBodyError::UnknownOp,
10229            Err(_) => ControlRequestBodyError::InvalidBody,
10230        };
10231        (err, classification)
10232    })
10233}
10234
10235fn parse_module_control_request_from_module(
10236    body: &[u8],
10237) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
10238    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
10239        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
10240            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
10241            Ok(_) => ControlRequestBodyError::UnknownOp,
10242            Err(_) => ControlRequestBodyError::InvalidBody,
10243        };
10244        (err, classification)
10245    })
10246}
10247
10248#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
10249enum ProviderRoleKind {
10250    ToolProvider,
10251    PipelineStage,
10252    ManagementSurface,
10253    InternalService,
10254}
10255
10256fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
10257    match role {
10258        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
10259        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
10260        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
10261        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
10262    }
10263}
10264
10265fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
10266    roles.iter().map(provider_role_kind).collect()
10267}
10268
10269/// Most refused scope records named individually in the log per sync; the
10270/// `refused` count on the accepted line is always complete.
10271const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
10272
10273/// Per-outcome counts of one accepted `scope.sync`, for its log line.
10274#[derive(Debug, Default, PartialEq, Eq)]
10275struct ScopeOutcomeCounts {
10276    created: usize,
10277    replaced: usize,
10278    updated: usize,
10279    unchanged: usize,
10280    refused: usize,
10281}
10282
10283impl ScopeOutcomeCounts {
10284    fn of(results: &[ScopeRecordResult]) -> Self {
10285        let mut counts = Self::default();
10286        for result in results {
10287            let slot = match result.outcome {
10288                ScopeRecordOutcome::Created => &mut counts.created,
10289                ScopeRecordOutcome::Replaced => &mut counts.replaced,
10290                ScopeRecordOutcome::Updated => &mut counts.updated,
10291                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
10292                ScopeRecordOutcome::Refused => &mut counts.refused,
10293            };
10294            *slot += 1;
10295        }
10296        counts
10297    }
10298}
10299
10300#[cfg(test)]
10301mod scope_outcome_count_tests {
10302    use super::*;
10303
10304    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
10305        ScopeRecordResult {
10306            scope_ref: "r".to_string(),
10307            scope_epoch: 1,
10308            outcome,
10309            code: None,
10310            message: None,
10311            version: None,
10312            parent_state: None,
10313        }
10314    }
10315
10316    /// Each outcome lands in its own count, so a refused record can never be
10317    /// hidden inside the total the log already printed.
10318    #[test]
10319    fn every_outcome_is_counted_in_its_own_field() {
10320        let results = [
10321            result(ScopeRecordOutcome::Created),
10322            result(ScopeRecordOutcome::Created),
10323            result(ScopeRecordOutcome::Replaced),
10324            result(ScopeRecordOutcome::Updated),
10325            result(ScopeRecordOutcome::Unchanged),
10326            result(ScopeRecordOutcome::Refused),
10327            result(ScopeRecordOutcome::Refused),
10328            result(ScopeRecordOutcome::Refused),
10329        ];
10330        assert_eq!(
10331            ScopeOutcomeCounts::of(&results),
10332            ScopeOutcomeCounts {
10333                created: 2,
10334                replaced: 1,
10335                updated: 1,
10336                unchanged: 1,
10337                refused: 3,
10338            }
10339        );
10340    }
10341}
10342
10343/// Return whether a catalog change can create a newly violating live route.
10344/// Removing an attested claim is intentionally excluded: it makes fewer routes
10345/// forbidden and therefore must leave the existing route census untouched.
10346fn capability_census_trigger(
10347    old: Option<&CapabilityDeclarations>,
10348    new: Option<&CapabilityDeclarations>,
10349) -> bool {
10350    let old_provides = old
10351        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
10352        .unwrap_or_default();
10353    let old_denies = old
10354        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
10355        .unwrap_or_default();
10356    let new = new.cloned().unwrap_or(CapabilityDeclarations {
10357        provides: Vec::new(),
10358        requires: Vec::new(),
10359        must_never_reach: Vec::new(),
10360    });
10361
10362    new.provides
10363        .iter()
10364        .any(|capability| !old_provides.contains(capability))
10365        || new
10366            .must_never_reach
10367            .iter()
10368            .any(|capability| !old_denies.contains(capability))
10369}
10370
10371fn catalog_update_frozen_field_message(
10372    registered: &ModuleManifest,
10373    provides: &[ProviderRole],
10374) -> Option<String> {
10375    let old_has_provides = !registered.provides.is_empty();
10376    let new_has_provides = !provides.is_empty();
10377    if old_has_provides != new_has_provides {
10378        return Some(format!(
10379            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
10380            registered.module_id
10381        ));
10382    }
10383
10384    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
10385        return Some(format!(
10386            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
10387            registered.module_id
10388        ));
10389    }
10390
10391    let registered_concurrency = manifest_concurrency(registered);
10392    let mut candidate = registered.clone();
10393    candidate.provides = provides.to_vec();
10394    let candidate_concurrency = manifest_concurrency(&candidate);
10395    if candidate_concurrency != registered_concurrency {
10396        return Some(format!(
10397            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
10398            registered.module_id, registered_concurrency, candidate_concurrency
10399        ));
10400    }
10401
10402    // control_ops live beside the manifest in the HELLO body, not inside
10403    // ModuleManifest, so a provides-only catalog.update cannot change them.
10404    None
10405}
10406
10407fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
10408    manifest.provides.iter().any(is_routable_role)
10409}
10410
10411/// Returns the routable-provider concurrency subc should enforce for this manifest.
10412///
10413/// ToolProvider and ManagementSurface store their delivery concurrency directly.
10414/// InternalService has no role-specific concurrency field, so it retains the
10415/// existing ModuleManaged default for backward compatibility.
10416fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
10417    manifest
10418        .provides
10419        .iter()
10420        .find_map(|provider| match provider {
10421            ProviderRole::ToolProvider { concurrency, .. }
10422            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
10423            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
10424        })
10425        .unwrap_or(Concurrency::ModuleManaged)
10426}
10427
10428/// True when the manifest carries a ManagementSurface role whose concurrency
10429/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
10430/// bytes because the typed manifest deliberately erases that distinction: the
10431/// default exists for wire compatibility, and this probe exists so the default
10432/// stays observable. Any parse irregularity returns false -- the caller only
10433/// logs, and a malformed body already failed registration upstream.
10434fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
10435    let has_management_surface = manifest
10436        .provides
10437        .iter()
10438        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
10439    if !has_management_surface {
10440        return false;
10441    }
10442    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
10443        return false;
10444    };
10445    let Some(provides) = raw
10446        .get("manifest")
10447        .and_then(|manifest| manifest.get("provides"))
10448        .and_then(serde_json::Value::as_array)
10449    else {
10450        return false;
10451    };
10452    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
10453    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
10454    // against the management_surface_manifest_without_concurrency golden, not
10455    // recalled (the externally-tagged guess was this function's first bug).
10456    provides.iter().any(|role| {
10457        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
10458            && role.get("concurrency").is_none()
10459    })
10460}
10461
10462fn negotiate_version(peer_version: u8) -> Result<u8, String> {
10463    if peer_version != PROTOCOL_VERSION {
10464        return Err(format!(
10465            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
10466        ));
10467    }
10468    Ok(PROTOCOL_VERSION)
10469}
10470
10471fn pong(frame: &Frame) -> Result<Frame, RouterError> {
10472    Frame::build_with_version(
10473        response_version(frame),
10474        FrameType::Pong,
10475        frame.header.flags,
10476        0,
10477        0,
10478        frame.header.corr,
10479        Vec::new(),
10480    )
10481    .map_err(RouterError::FrameBuild)
10482}
10483
10484pub(crate) fn control_error_frame(
10485    frame: &Frame,
10486    code: &'static str,
10487    message: impl Into<String>,
10488) -> Result<Frame, RouterError> {
10489    control_error_body_frame(
10490        frame,
10491        ErrorBody {
10492            code: code.to_string(),
10493            message: message.into(),
10494            detail: None,
10495        },
10496    )
10497}
10498
10499/// A channel-0 ERROR for a refusal the daemon decided itself: every
10500/// route.open refusal, the drain `module_reloading` and scope-change/end
10501/// bodies, spawn-subscription and supervisor refusals. Carries the
10502/// daemon-origin bit, which is what lets a client treat `module_reloading`
10503/// as proof that its request never reached a module. Response, push,
10504/// goodbye and relay-request frames keep the plain control flags in their
10505/// own builders.
10506pub(crate) fn control_error_body_frame(
10507    frame: &Frame,
10508    error: ErrorBody,
10509) -> Result<Frame, RouterError> {
10510    control_error_frame_with_flags(frame, error, daemon_error_flags())
10511}
10512
10513/// A channel-0 ERROR relaying a module's own rejection body unchanged: a
10514/// refused route.bind answered as a route.open refusal, or a refused
10515/// module-control RPC. Plain control flags — the code and message are
10516/// module-chosen, so the frame must not claim daemon authorship, even when
10517/// the module picks a daemon code like `module_reloading`.
10518fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
10519    control_error_frame_with_flags(frame, error, control_flags())
10520}
10521
10522fn control_error_frame_with_flags(
10523    frame: &Frame,
10524    error: ErrorBody,
10525    flags: Flags,
10526) -> Result<Frame, RouterError> {
10527    let body = serde_json::to_vec(&error).map_err(|err| {
10528        RouterError::backend(
10529            0,
10530            frame.header.corr,
10531            format!("failed to encode control ERROR: {err}"),
10532        )
10533    })?;
10534
10535    Frame::build_with_version(
10536        response_version(frame),
10537        FrameType::Error,
10538        flags,
10539        0,
10540        0,
10541        frame.header.corr,
10542        body,
10543    )
10544    .map_err(RouterError::FrameBuild)
10545}
10546
10547pub(crate) fn control_response_body_frame<T: Serialize>(
10548    frame: &Frame,
10549    reply: &T,
10550    label: &'static str,
10551) -> Result<Frame, RouterError> {
10552    let body = serde_json::to_vec(reply).map_err(|err| {
10553        RouterError::backend(
10554            0,
10555            frame.header.corr,
10556            format!("failed to encode {label}: {err}"),
10557        )
10558    })?;
10559
10560    Frame::build_with_version(
10561        response_version(frame),
10562        FrameType::Response,
10563        control_flags(),
10564        0,
10565        0,
10566        frame.header.corr,
10567        body,
10568    )
10569    .map_err(RouterError::FrameBuild)
10570}
10571
10572/// Map a forwarding failure to the wire code a client sees.
10573///
10574/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
10575/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
10576/// chosen here decides whether a caller retries or gives up.
10577///
10578/// That makes attribution the load-bearing property, not merely having a code. A
10579/// permanent fault published as a retryable one produces a fleet-wide retry storm
10580/// against something that can never recover; a transient fault published as
10581/// permanent gives up on work that would have succeeded. Both look correct in a
10582/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
10583/// the mapping per variant rather than merely asserting that some code exists.
10584///
10585/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
10586/// two codes on the same side of the boundary passes it. Measured rather than
10587/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
10588/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
10589/// a test named for something else that happens to assert the string.
10590///
10591/// That accidental coverage is deliberately left alone rather than promoted to a
10592/// named test, because it guards a property this function does not promise.
10593/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
10594/// specific code within a class, so identity is free to change and only the
10595/// partition is a contract. Splitting it out would assert a guarantee nothing
10596/// depends on — and a suite that promises more than the code does is the harder
10597/// thing to correct later, because the next reader cannot tell which assertions
10598/// are load-bearing.
10599///
10600/// Pin identity here the moment a consumer branches on a specific code.
10601/// The spawn a HELLO would register as and the provider declarations it
10602/// requests, carried from the HELLO's checks to the point the registry
10603/// accepts it.
10604struct HelloAuthority {
10605    record: Arc<crate::spawn_authority::SpawnAuthorityRecord>,
10606    requested: ProviderDeclarations,
10607    /// Whether admitting this HELLO froze the spawn's requested declarations,
10608    /// so a refusal after admission undoes that freeze and no other.
10609    froze: std::sync::atomic::AtomicBool,
10610}
10611
10612fn forwarding_error_code(err: &ForwardingError) -> &'static str {
10613    match err {
10614        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
10615        ForwardingError::NoModuleConnection => "target_unavailable",
10616        ForwardingError::ModuleReloading { .. } => "module_reloading",
10617        ForwardingError::ClientRouteChannelExhausted { .. }
10618        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
10619        ForwardingError::StaleModuleEndpoint
10620        | ForwardingError::UnknownReservation { .. }
10621        | ForwardingError::ConnectionClosing { .. }
10622        | ForwardingError::ClientEgressClosed { .. }
10623        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
10624        // Only a swap candidate's registration can produce this, and it means
10625        // exactly what a second active HELLO for a live id means.
10626        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
10627        ForwardingError::RelayCorrelationExhausted
10628        | ForwardingError::RouteOpenBuild(_)
10629        | ForwardingError::Poisoned => "forwarding_error",
10630    }
10631}
10632
10633fn response_version(frame: &Frame) -> u8 {
10634    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
10635        frame.header.ver
10636    } else {
10637        PROTOCOL_VERSION
10638    }
10639}
10640
10641pub(crate) fn control_flags() -> Flags {
10642    Flags::new(false, Priority::Passive, false)
10643}
10644
10645/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
10646/// channel. The target is the module (a client never saw the route), so this
10647/// takes the module path: delivered late rather than dropped when the module's
10648/// queue is momentarily full, and never closing its connection.
10649fn send_goodbye_target_best_effort(
10650    counters: &DaemonCounters,
10651    target: &GoodbyeTarget,
10652    context: &'static str,
10653) {
10654    let Ok(frame) = Frame::build_with_version(
10655        target.negotiated_ver,
10656        FrameType::Goodbye,
10657        control_flags(),
10658        target.channel,
10659        target.epoch,
10660        0,
10661        Vec::new(),
10662    ) else {
10663        return;
10664    };
10665    crate::forwarding::send_module_route_goodbye(
10666        counters,
10667        &target.sink,
10668        frame,
10669        target.module_id.as_deref(),
10670        context,
10671    );
10672}
10673
10674fn route_close_reason_for_opener(
10675    reason: RouteCloseReason,
10676    accepts_close_reasons: Option<&[String]>,
10677) -> RouteCloseReason {
10678    if reason == RouteCloseReason::AuthorityChanged
10679        && !accepts_close_reasons
10680            .is_some_and(|reasons| reasons.iter().any(|reason| reason == "authority_changed"))
10681    {
10682        // Older readers forbid reopening for unknown reasons. Reload already
10683        // permits reopening and, like an authority change, requires a new bind.
10684        RouteCloseReason::Reload
10685    } else {
10686        reason
10687    }
10688}
10689
10690pub(crate) fn send_route_control_pushes(
10691    forwarding: &ForwardingTable,
10692    routes: Vec<EndpointRoute>,
10693    push: ClientControlPush,
10694) {
10695    let reason = match &push {
10696        ClientControlPush::RouteClosing { reason, .. }
10697        | ClientControlPush::RouteClosed { reason, .. } => *reason,
10698    };
10699    let mut targets: Vec<(GoodbyeTarget, RouteCloseReason, Vec<u16>)> = Vec::new();
10700    for route in routes {
10701        let recipient_reason =
10702            route_close_reason_for_opener(reason, route.accepts_close_reasons.as_deref());
10703        let target = route.goodbye_target;
10704        if let Some((existing, _, channels)) = targets.iter_mut().find(|(existing, reason, _)| {
10705            existing.connection_id == target.connection_id && *reason == recipient_reason
10706        }) {
10707            debug_assert_eq!(
10708                existing.negotiated_ver, target.negotiated_ver,
10709                "one connection cannot negotiate multiple frame versions"
10710            );
10711            if !channels.contains(&target.channel) {
10712                channels.push(target.channel);
10713            }
10714            continue;
10715        }
10716        let channel = target.channel;
10717        targets.push((target, recipient_reason, vec![channel]));
10718    }
10719    for (target, recipient_reason, mut channels) in targets {
10720        channels.sort_unstable();
10721        let mut push = push.clone();
10722        match &mut push {
10723            ClientControlPush::RouteClosing {
10724                channels: covered,
10725                reason,
10726                ..
10727            }
10728            | ClientControlPush::RouteClosed {
10729                channels: covered,
10730                reason,
10731                ..
10732            } => {
10733                *covered = channels;
10734                *reason = recipient_reason;
10735            }
10736        }
10737        let body = match serde_json::to_vec(&push) {
10738            Ok(body) => body,
10739            Err(err) => {
10740                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
10741                continue;
10742            }
10743        };
10744        let frame = match Frame::build_with_version(
10745            target.negotiated_ver,
10746            FrameType::Push,
10747            control_flags(),
10748            0,
10749            0,
10750            0,
10751            body.clone(),
10752        ) {
10753            Ok(frame) => frame,
10754            Err(err) => {
10755                warn!(
10756                    route_channel = target.channel,
10757                    error = %err,
10758                    "failed to build route lifecycle control PUSH frame"
10759                );
10760                continue;
10761            }
10762        };
10763        if let Err(err) = target.sink.try_send(frame) {
10764            if target.close_on_delivery_failure() {
10765                warn!(
10766                    target_connection_id = target.connection_id.get(),
10767                    route_channel = target.channel,
10768                    error = %err,
10769                    "route lifecycle control PUSH was not delivered to client; closing target connection"
10770                );
10771                let _ = forwarding.escalate_client_delivery_failure(
10772                    target.connection_id,
10773                    target.channel,
10774                    target.epoch,
10775                    CloseReason::new(
10776                        "route_lifecycle_push_delivery_failed",
10777                        format!(
10778                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
10779                            target.channel
10780                        ),
10781                    ),
10782                    crate::forwarding::UndeliveredFrame {
10783                        module_id: target.module_id.as_deref(),
10784                        sink: &target.sink,
10785                    },
10786                );
10787            }
10788        }
10789    }
10790}
10791
10792#[cfg(test)]
10793mod tests {
10794    include!("control/agent_commands_tests.rs");
10795    include!("control/resource_revision_tests.rs");
10796    include!("control/route_open_recovery_tests.rs");
10797    #[tokio::test]
10798    async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
10799        let registry = Arc::new(Registry::default());
10800        let handler = ControlHandler::new(registry);
10801        assert_eq!(
10802            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10803            None
10804        );
10805        let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
10806        hello_via_sink(
10807            &handler,
10808            &ctx,
10809            &mut rx,
10810            hello_frame("daemon", PROTOCOL_VERSION, 1),
10811        )
10812        .await;
10813        assert_eq!(
10814            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
10815            Some("daemon".into())
10816        );
10817        assert_eq!(
10818            handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
10819            None
10820        );
10821        assert_eq!(
10822            handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
10823            None,
10824            "a registered module is not necessarily supervised"
10825        );
10826        assert_eq!(
10827            handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
10828            None,
10829            "aggregate provenance has no single target"
10830        );
10831        assert_eq!(
10832            handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
10833            None
10834        );
10835    }
10836
10837    mod peer_manifest {
10838        include!("peer_manifest_tests.rs");
10839    }
10840    mod catalog_declarations {
10841        include!("control/catalog_tests.rs");
10842    }
10843    mod activity_read {
10844        include!("activity_read_tests.rs");
10845    }
10846
10847    mod lifecycle_activity {
10848        include!("control_lifecycle_tests.rs");
10849    }
10850    #[cfg(unix)]
10851    #[tokio::test]
10852    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
10853        let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
10854        let path = dir.join("subc.jsonc");
10855        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
10856            "program":"/bin/sleep","args":["60"],"protocol":"none",
10857            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
10858        }}}).to_string()).unwrap();
10859        let mut configured = crate::daemon_config::load(&path)
10860            .unwrap()
10861            .unwrap()
10862            .modules
10863            .pop()
10864            .unwrap();
10865        let registry = std::sync::Arc::new(crate::Registry::default());
10866        let handle = crate::SupervisorHandle::new();
10867        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
10868            .with_handle(handle.clone());
10869        let module = supervisor
10870            .supervise_configured_with_health(
10871                configured.module_spec(),
10872                true,
10873                configured.health.clone(),
10874                None,
10875                configured.restart,
10876            )
10877            .unwrap();
10878        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
10879        let before = module.status().unwrap().pid;
10880        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
10881        configured.health.cadence = std::time::Duration::from_secs(3600);
10882        let health_only = handler
10883            .reconcile_supervised_modules(
10884                &supervisor,
10885                vec![configured.clone()],
10886                false,
10887                Principal::Direct,
10888            )
10889            .await
10890            .unwrap();
10891        assert!(
10892            health_only.0.changed_pending_reload.is_empty(),
10893            "health policy is already applied live"
10894        );
10895        assert_eq!(module.status().unwrap().pid, before);
10896        assert_eq!(
10897            module.configuration().unwrap().1.http,
10898            configured.health.http
10899        );
10900        configured.args = vec!["61".into()];
10901        let launch = handler
10902            .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
10903            .await
10904            .unwrap();
10905        assert_eq!(launch.0.changed_pending_reload, ["stock"]);
10906        assert_eq!(
10907            module.status().unwrap().pid,
10908            before,
10909            "a launch edit is stored until reload"
10910        );
10911        module.drain().await.unwrap();
10912    }
10913    use cortexkit_test_support::ScratchDir;
10914    use std::{
10915        collections::BTreeMap,
10916        fmt,
10917        path::PathBuf,
10918        sync::{Arc, Mutex},
10919        time::Duration,
10920    };
10921
10922    use serde_json::{json, Value};
10923    use subc_protocol::{
10924        manifest::{
10925            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
10926            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
10927        },
10928        session::HealthStatus,
10929        FrameType,
10930    };
10931
10932    use super::*;
10933    use crate::{
10934        forwarding::{DataRoute, DataRouteState},
10935        registry::ChannelState,
10936        router::FrameSink,
10937        stderr_tail::DEFAULT_MAX_LINE_BYTES,
10938        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
10939        RouteCtx, Router,
10940    };
10941    use tokio::{
10942        sync::mpsc,
10943        time::{sleep, Instant},
10944    };
10945    use tracing::{
10946        field::{Field, Visit},
10947        Event, Subscriber,
10948    };
10949    use tracing_subscriber::{layer::Context, prelude::*, Layer};
10950
10951    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
10952    ///
10953    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
10954    /// is only populated for `tests/*.rs` integration test binaries -- this file
10955    /// compiles as part of the library target, which gets neither. The binary
10956    /// lives in the profile folder this test was built into,
10957    /// `<target-dir>/<profile>/fake-aft-stub`, which `cargo_profile_bin` finds
10958    /// from the test's own path.
10959    ///
10960    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
10961    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
10962    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
10963    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
10964    /// `NotFound`, which reads as a broken test rather than an unbuilt
10965    /// dependency -- so state the cause and the remedy instead. Deliberately a
10966    /// panic and not a silent skip: a test that quietly passes when it could not
10967    /// run is worse than one that fails, because it reports health it never
10968    /// verified.
10969    fn fake_aft_stub_path() -> PathBuf {
10970        let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
10971        assert!(
10972            path.exists(),
10973            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
10974             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
10975            path.display()
10976        );
10977        path
10978    }
10979
10980    /// Whether clients retry `code` in place: the predicate itself, never a copy
10981    /// of its set. A copied list breaks silently when a code is added to or
10982    /// removed from the real one, and a stale copy here would let exactly the
10983    /// failure this test exists to catch pass.
10984    fn client_retries(code: &str) -> bool {
10985        subc_protocol::error_codes::is_retryable_route_open(code)
10986    }
10987
10988    /// A code is not a label — clients branch on it, so publishing the wrong KIND
10989    /// of failure is worse than publishing none. A permanent fault dressed as
10990    /// retryable makes every client in the fleet retry forever against something
10991    /// that cannot recover; a transient fault dressed as permanent abandons work
10992    /// that would have succeeded.
10993    ///
10994    /// Asserting "a code exists" cannot catch either, because the string is free
10995    /// to say anything. This enumerates every variant and pins which side of the
10996    /// retry boundary it lands on, so a new variant must be classified here
10997    /// deliberately rather than inheriting whichever arm it was appended to.
10998    #[test]
10999    fn retryability_of_forwarding_codes_matches_the_failure() {
11000        // Transient by nature: the target is booting, reloading, or its endpoint
11001        // was swapped mid-flight. Retrying is how these resolve.
11002        let transient = [
11003            ForwardingError::NoModuleConnection,
11004            ForwardingError::ModuleReloading {
11005                module_id: "m".into(),
11006            },
11007            ForwardingError::StaleModuleEndpoint,
11008            ForwardingError::UnknownReservation {
11009                client_channel: 1,
11010                module_channel: 1,
11011            },
11012            ForwardingError::ConnectionClosing {
11013                connection_id: ConnectionId::new(1),
11014            },
11015            ForwardingError::ClientEgressClosed {
11016                connection_id: ConnectionId::new(1),
11017            },
11018            ForwardingError::ModuleEgressUnavailable {
11019                connection_id: ConnectionId::new(1),
11020            },
11021        ];
11022        for err in transient {
11023            let code = forwarding_error_code(&err);
11024            assert!(
11025                client_retries(code),
11026                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
11027            );
11028        }
11029
11030        // Not fixed by retrying. Channel and correlation exhaustion need the
11031        // caller to close routes, and a poisoned lock is a daemon that cannot
11032        // recover at all — the worst thing to advertise as retryable, since every
11033        // client would storm a daemon that will never answer.
11034        let permanent = [
11035            ForwardingError::ConnectionRoleConflict {
11036                connection_id: ConnectionId::new(1),
11037            },
11038            ForwardingError::ClientRouteChannelExhausted {
11039                connection_id: ConnectionId::new(1),
11040            },
11041            ForwardingError::ModuleRouteChannelExhausted {
11042                endpoint: ModuleEndpointId {
11043                    connection_id: ConnectionId::new(1),
11044                    generation: 1,
11045                },
11046            },
11047            ForwardingError::RelayCorrelationExhausted,
11048            ForwardingError::RouteOpenBuild("x".into()),
11049            ForwardingError::Poisoned,
11050        ];
11051        for err in permanent {
11052            let code = forwarding_error_code(&err);
11053            assert!(
11054                !client_retries(code),
11055                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
11056            );
11057        }
11058    }
11059
11060    /// The principal is the daemon's answer to "who is calling", and modules
11061    /// branch on it: aft gates bash on it, cerebellum gates browser control,
11062    /// plexus gates connector invocation. So a stamp is an authorization input in
11063    /// another process, not a label — and both possible answers SUCCEED, which is
11064    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
11065    /// first-party capability to something that never proved it; a supervised one
11066    /// stamped `Direct` silently strips a module of capability it is entitled to.
11067    ///
11068    /// Neither shows up in a test that only checks the bind succeeded. Before this
11069    /// test the only coverage was accidental —
11070    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
11071    /// stamped principal on its way past, so narrowing that wire-shape test to its
11072    /// stated subject would have deleted the last assertion on this value. It
11073    /// still asserts the stamp, which is now redundancy rather than the only
11074    /// guard: both fail under the same mutation, and this one names the reason.
11075    /// SCOPE: this handler's supervisor has spawned nothing, so
11076    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
11077    /// is unreachable here. Both assertions below are refusals, and a mutant that
11078    /// refuses everything would satisfy them.
11079    ///
11080    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
11081    /// spawns a supervised consumer, reads its live nonce, and asserts the module
11082    /// observed `principal.kind == "reserved"` carrying that module_id — verified
11083    /// at source rather than assumed, since a citation is a claim about another
11084    /// file and ages like one. Recorded because a harness that structurally
11085    /// cannot reach an arm reports "none" for that arm identically to one that
11086    /// covers it and found nothing.
11087    #[tokio::test]
11088    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
11089        let handler = ControlHandler::default();
11090        let frame =
11091            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
11092
11093        // Absent consumer_identity is the ordinary case: a human at a terminal, or
11094        // any process holding the connection file. Nothing was proved, so nothing
11095        // may be granted beyond the unattested floor.
11096        let stamped = handler
11097            .route_open_principal(ConnectionId::new(1), &frame, None)
11098            .unwrap()
11099            .unwrap();
11100        assert_eq!(
11101            stamped,
11102            Principal::Direct,
11103            "a caller that proved nothing must not be stamped as a supervised module"
11104        );
11105
11106        // A claimed module_id with a nonce no supervised child was given is a
11107        // forgery attempt, not a weaker caller: it must be REFUSED rather than
11108        // quietly demoted to Direct, or an impersonation attempt looks identical
11109        // to an ordinary unattested connection.
11110        let forged = handler
11111            .route_open_principal(
11112                ConnectionId::new(1),
11113                &frame,
11114                Some(ConsumerIdentity {
11115                    module_id: "aft".to_string(),
11116                    launch_nonce: "not-a-real-nonce".to_string(),
11117                }),
11118            )
11119            .unwrap();
11120        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
11121        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
11122    }
11123
11124    /// The test above hands `route_open_principal` an identity it built itself,
11125    /// which proves the stamping rule and nothing about where the identity comes
11126    /// from. The real producer is a wire body, and the two are joined by a serde
11127    /// field name that nothing else asserts.
11128    ///
11129    /// That join fails quietly in one specific way: an unrecognised key is simply
11130    /// absent after parsing, so a renamed or misspelled `consumer_identity`
11131    /// yields `None` and every supervised module silently drops to `Direct`.
11132    /// Capability-wise that is the safe direction, but it surfaces far from its
11133    /// cause — as a module mysteriously refused bash — and it would pass every
11134    /// test that builds its own input.
11135    ///
11136    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
11137    /// would break every client the moment the daemon gains a field, trading a
11138    /// quiet demotion for a hard refusal on additive change. Asserting the join
11139    /// instead means a rename breaks a test here rather than the fleet.
11140    #[test]
11141    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
11142        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
11143        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
11144        let ClientControlRequest::RouteOpen {
11145            consumer_identity, ..
11146        } = parsed
11147        else {
11148            panic!("route.open body must parse as RouteOpen");
11149        };
11150        assert_eq!(
11151            consumer_identity,
11152            Some(ConsumerIdentity {
11153                module_id: "aft".to_string(),
11154                launch_nonce: "n".to_string(),
11155            }),
11156            "the wire field name must reach the value route_open_principal reads"
11157        );
11158    }
11159
11160    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
11161        ModuleManifest::builder(module_id, "0.1.0")
11162            .protocol_ver(protocol_ver)
11163            .provides(vec![ProviderRole::ToolProvider {
11164                tools: vec![Tool {
11165                    name: "read".to_string(),
11166                    description: None,
11167                    execution_mode: ExecutionMode::Pure,
11168                    schema: json!({"type": "object"}),
11169                }],
11170                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
11171                concurrency: Concurrency::ModuleManaged,
11172                emits_push: true,
11173                sub_supervises: true,
11174            }])
11175            .build()
11176    }
11177
11178    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
11179        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
11180    }
11181
11182    fn hello_frame_with_control_ops(
11183        module_id: &str,
11184        protocol_ver: u8,
11185        corr: u64,
11186        control_ops: Option<Vec<String>>,
11187    ) -> Frame {
11188        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
11189    }
11190
11191    fn hello_frame_with_nonce(
11192        module_id: &str,
11193        protocol_ver: u8,
11194        corr: u64,
11195        launch_nonce: Option<&str>,
11196    ) -> Frame {
11197        hello_frame_full(
11198            module_id,
11199            protocol_ver,
11200            corr,
11201            None,
11202            launch_nonce.map(ToOwned::to_owned),
11203        )
11204    }
11205
11206    fn hello_frame_full(
11207        module_id: &str,
11208        protocol_ver: u8,
11209        corr: u64,
11210        control_ops: Option<Vec<String>>,
11211        launch_nonce: Option<String>,
11212    ) -> Frame {
11213        let body = serde_json::to_vec(&ModuleHelloBody {
11214            manifest: manifest(module_id, protocol_ver),
11215            protocol_ver,
11216            control_ops,
11217            launch_nonce,
11218        })
11219        .unwrap();
11220        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11221    }
11222
11223    fn hello_frame_with_rich_ui(
11224        module_id: &str,
11225        protocol_ver: u8,
11226        corr: u64,
11227        launch_nonce: Option<&str>,
11228        rich_ui: Value,
11229    ) -> Frame {
11230        let mut body = serde_json::to_value(ModuleHelloBody {
11231            manifest: manifest(module_id, protocol_ver),
11232            protocol_ver,
11233            control_ops: None,
11234            launch_nonce: launch_nonce.map(str::to_string),
11235        })
11236        .unwrap();
11237        body["manifest"]["rich_ui"] = rich_ui;
11238        Frame::build(
11239            FrameType::Hello,
11240            control_flags(),
11241            0,
11242            0,
11243            corr,
11244            serde_json::to_vec(&body).unwrap(),
11245        )
11246        .unwrap()
11247    }
11248
11249    fn non_routable_hello_frame_with_control_ops(
11250        module_id: &str,
11251        corr: u64,
11252        control_ops: Option<Vec<String>>,
11253    ) -> Frame {
11254        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11255        manifest.provides.clear();
11256        let body = serde_json::to_vec(&ModuleHelloBody {
11257            manifest,
11258            protocol_ver: PROTOCOL_VERSION,
11259            control_ops,
11260            launch_nonce: None,
11261        })
11262        .unwrap();
11263        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
11264    }
11265
11266    fn capability_grammar_hello_frame(
11267        capabilities: Value,
11268        runtime_computed: Option<Value>,
11269        corr: u64,
11270    ) -> Frame {
11271        let mut body = serde_json::to_value(ModuleHelloBody {
11272            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
11273            protocol_ver: PROTOCOL_VERSION,
11274            control_ops: None,
11275            launch_nonce: None,
11276        })
11277        .expect("HELLO body serializes");
11278        body["manifest"]["capabilities"] = capabilities;
11279        if let Some(runtime_computed) = runtime_computed {
11280            body["runtime_computed"] = runtime_computed;
11281        }
11282        Frame::build(
11283            FrameType::Hello,
11284            control_flags(),
11285            0,
11286            0,
11287            corr,
11288            serde_json::to_vec(&body).expect("HELLO body reserializes"),
11289        )
11290        .expect("HELLO frame builds")
11291    }
11292
11293    fn channel_request(channel: u16, corr: u64) -> Frame {
11294        Frame::build(
11295            FrameType::Request,
11296            Flags::new(true, Priority::Interactive, false),
11297            channel,
11298            0,
11299            corr,
11300            b"opaque".to_vec(),
11301        )
11302        .unwrap()
11303    }
11304
11305    fn route_ctx(
11306        connection_id: ConnectionId,
11307    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11308        let (tx, rx) = mpsc::channel(8);
11309        (
11310            RouteCtx {
11311                connection_id,
11312                egress: FrameSink::new(tx),
11313            },
11314            rx,
11315        )
11316    }
11317
11318    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
11319        serde_json::from_slice(&frame.body).unwrap()
11320    }
11321
11322    /// Register a module over a connection that has a sink and return the
11323    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
11324    /// module's own sink rather than returning it as a reply, so the ack is
11325    /// taken off `rx` here and whatever the test reads next is what followed it.
11326    async fn hello_via_sink(
11327        handler: &ControlHandler,
11328        ctx: &RouteCtx,
11329        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11330        hello: Frame,
11331    ) -> Frame {
11332        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
11333        assert!(
11334            replies.is_empty(),
11335            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
11336        );
11337        let ack = rx
11338            .try_recv()
11339            .expect("HELLO_ACK is queued on the module sink")
11340            .frame;
11341        assert_eq!(ack.header.ty, FrameType::HelloAck);
11342        ack
11343    }
11344
11345    fn parse_error(frame: &Frame) -> Value {
11346        serde_json::from_slice(&frame.body).unwrap()
11347    }
11348
11349    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
11350        serde_json::from_slice(&frame.body).unwrap()
11351    }
11352
11353    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
11354        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
11355            route_channel,
11356            route_epoch: 0,
11357            kind,
11358        })
11359        .unwrap();
11360        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11361    }
11362
11363    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
11364        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
11365            module_id: module_id.to_string(),
11366        })
11367        .unwrap();
11368        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11369    }
11370
11371    fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
11372        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
11373    }
11374
11375    // Exercise retired opener fields without putting them back in BindIdentity.
11376    fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
11377        let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
11378        let mut frame = route_open_frame(corr, module_id, root);
11379        let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
11380        body["identity"]["project_root"] = serde_json::json!(path.as_path());
11381        frame.body = serde_json::to_vec(&body).unwrap();
11382        frame
11383    }
11384
11385    fn route_open_frame_with_consumer_capabilities(
11386        corr: u64,
11387        module_id: &str,
11388        _project_root: ScratchDir,
11389        consumer_capabilities: Option<Vec<String>>,
11390    ) -> Frame {
11391        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11392            busy_refusals: None,
11393            target: RouteTarget::ToolProvider {
11394                module_id: module_id.to_string(),
11395            },
11396            identity: BindIdentity::new("unit".to_string(), "session".to_string()),
11397            consumer_identity: None,
11398            consumer_capabilities,
11399            role_versions: None,
11400            admission_facts: None,
11401            scope: None,
11402        })
11403        .unwrap();
11404        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11405    }
11406
11407    fn route_open_frame_with_role_versions(
11408        corr: u64,
11409        module_id: &str,
11410        _project_root: ScratchDir,
11411        role_versions: Option<BTreeMap<String, String>>,
11412    ) -> Frame {
11413        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11414            busy_refusals: None,
11415            target: RouteTarget::ToolProvider {
11416                module_id: module_id.to_string(),
11417            },
11418            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11419            consumer_identity: None,
11420            consumer_capabilities: None,
11421            role_versions,
11422            admission_facts: None,
11423            scope: None,
11424        })
11425        .unwrap();
11426        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11427    }
11428
11429    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
11430        entries
11431            .iter()
11432            .map(|(role, version)| (role.to_string(), version.to_string()))
11433            .collect()
11434    }
11435
11436    fn route_open_frame_with_admission_facts(
11437        corr: u64,
11438        module_id: &str,
11439        _project_root: ScratchDir,
11440        consumer_identity: Option<subc_control::ConsumerIdentity>,
11441        facts: Option<Value>,
11442    ) -> Frame {
11443        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11444            busy_refusals: None,
11445            target: RouteTarget::ToolProvider {
11446                module_id: module_id.to_string(),
11447            },
11448            identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
11449            consumer_identity,
11450            consumer_capabilities: None,
11451            role_versions: None,
11452            admission_facts: facts,
11453            scope: None,
11454        })
11455        .unwrap();
11456        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11457    }
11458
11459    #[derive(Clone, Default)]
11460    struct EventCapture {
11461        events: Arc<Mutex<Vec<CapturedEvent>>>,
11462    }
11463
11464    #[derive(Clone, Debug)]
11465    struct CapturedEvent {
11466        target: String,
11467        level: tracing::Level,
11468        fields: BTreeMap<String, String>,
11469    }
11470
11471    impl EventCapture {
11472        fn events(&self) -> Vec<CapturedEvent> {
11473            self.events.lock().unwrap().clone()
11474        }
11475    }
11476
11477    impl<S> Layer<S> for EventCapture
11478    where
11479        S: Subscriber,
11480    {
11481        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
11482            let mut visitor = EventFieldVisitor::default();
11483            event.record(&mut visitor);
11484            self.events.lock().unwrap().push(CapturedEvent {
11485                target: event.metadata().target().to_string(),
11486                level: *event.metadata().level(),
11487                fields: visitor.fields,
11488            });
11489        }
11490    }
11491
11492    #[derive(Default)]
11493    struct EventFieldVisitor {
11494        fields: BTreeMap<String, String>,
11495    }
11496
11497    impl Visit for EventFieldVisitor {
11498        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
11499            self.fields
11500                .insert(field.name().to_string(), format!("{value:?}"));
11501        }
11502    }
11503
11504    fn health_response(corr: u64, status: HealthStatus) -> Frame {
11505        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
11506            status,
11507            detail: Some("warming".to_string()),
11508            metrics: Some(json!({"queue_depth": 3})),
11509        })
11510        .unwrap();
11511        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11512    }
11513
11514    fn route_bind_ack(corr: u64) -> Frame {
11515        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
11516        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
11517    }
11518
11519    fn unique_project_root(label: &str) -> ScratchDir {
11520        ScratchDir::new(label)
11521    }
11522
11523    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
11524        match parse_route_poll(frame) {
11525            ClientControlResponse::RoutePoll {
11526                status: None,
11527                live: Some(live),
11528                ..
11529            } => assert_eq!(live, expected_live),
11530            other => panic!("unexpected route.poll response: {other:?}"),
11531        }
11532    }
11533
11534    fn bind_liveness_route(
11535        registry: &Registry,
11536        forwarding: &ForwardingTable,
11537        module_id: &str,
11538    ) -> (RouteCtx, u16, u32) {
11539        let module_connection = ConnectionId::new(101);
11540        let client_connection = ConnectionId::new(202);
11541        let registration = registry
11542            .register_with_control_ops(
11543                manifest(module_id, PROTOCOL_VERSION),
11544                PROTOCOL_VERSION,
11545                module_connection,
11546                module_baseline_control_ops(),
11547            )
11548            .unwrap();
11549        let (module_tx, _module_rx) = mpsc::channel(8);
11550        let endpoint = forwarding
11551            .register_module_connection(
11552                module_connection,
11553                module_id.to_string(),
11554                PROTOCOL_VERSION,
11555                manifest_concurrency(&registration.manifest),
11556                FrameSink::new(module_tx),
11557            )
11558            .unwrap();
11559        let (client_ctx, _client_rx) = route_ctx(client_connection);
11560        let pending = forwarding
11561            .begin_route_bind_relay_for_test(
11562                client_connection,
11563                client_ctx.egress.clone(),
11564                1,
11565                module_id,
11566            )
11567            .unwrap();
11568        assert_eq!(pending.endpoint, endpoint);
11569        let route_channel = pending.client_channel;
11570        let route_epoch = pending.client_epoch;
11571        forwarding
11572            .complete_pending_relay(
11573                module_connection,
11574                pending.corr,
11575                RouteBindRelayOutcome::Accepted,
11576            )
11577            .unwrap();
11578        (client_ctx, route_channel, route_epoch)
11579    }
11580
11581    struct FakeProcessLiveness {
11582        live: Option<bool>,
11583    }
11584
11585    impl ModuleProcessLiveness for FakeProcessLiveness {
11586        fn process_live(&self, _module_id: &str) -> Option<bool> {
11587            self.live
11588        }
11589    }
11590
11591    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11592    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
11593    {
11594        let registry = Arc::new(Registry::default());
11595        let supervisor_handle = SupervisorHandle::new();
11596        let supervisor = Supervisor::new_for_test(
11597            Arc::clone(&registry),
11598            RestartPolicy::new(1, Duration::from_millis(10)),
11599        )
11600        .with_handle(supervisor_handle.clone());
11601        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
11602        let module = supervisor
11603            .spawn(ModuleSpec {
11604                script: None,
11605                carry_containment: false,
11606                module_id: "stderr-tail-wire".to_string(),
11607                program: fake_aft_stub_path(),
11608                args: Vec::new(),
11609                env: vec![
11610                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
11611                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
11612                ],
11613                reserved: false,
11614                reserved_prefixes: Vec::new(),
11615                protocol: ModuleProtocol::Subc,
11616                overlap: Default::default(),
11617            })
11618            .unwrap();
11619
11620        let deadline = Instant::now() + Duration::from_secs(5);
11621        loop {
11622            let tail = module.stderr_tail(None, None);
11623            if tail
11624                .entries
11625                .iter()
11626                .any(|entry| matches!(entry, TailEntry::ProcessStart))
11627                && tail.entries.iter().any(|entry| {
11628                    matches!(
11629                        entry,
11630                        TailEntry::Line {
11631                            truncated: true,
11632                            ..
11633                        }
11634                    )
11635                })
11636            {
11637                break;
11638            }
11639            assert!(
11640                Instant::now() < deadline,
11641                "module did not produce a truncated line and restart boundary: {tail:?}"
11642            );
11643            sleep(Duration::from_millis(10)).await;
11644        }
11645
11646        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11647        let request = ClientControlRequest::SupervisorStderrTail {
11648            module_id: "stderr-tail-wire".to_string(),
11649            max_lines: None,
11650            max_bytes: None,
11651        };
11652        let frame = Frame::build(
11653            FrameType::Request,
11654            control_flags(),
11655            0,
11656            0,
11657            1,
11658            serde_json::to_vec(&request).unwrap(),
11659        )
11660        .unwrap();
11661        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11662        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11663        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
11664            serde_json::from_slice(&responses[0].body).unwrap()
11665        else {
11666            panic!("expected supervisor.stderr_tail response");
11667        };
11668
11669        assert!(
11670            tail.entries
11671                .iter()
11672                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
11673            "the control response lost the restart boundary"
11674        );
11675        let Some(StderrTailEntry::Line {
11676            text,
11677            truncated,
11678            at_ms,
11679        }) = tail.entries.iter().find(|entry| {
11680            matches!(
11681                entry,
11682                StderrTailEntry::Line {
11683                    truncated: true,
11684                    ..
11685                }
11686            )
11687        })
11688        else {
11689            panic!("the control response lost the truncated line");
11690        };
11691        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
11692        assert!(*truncated);
11693        assert!(
11694            at_ms.is_some(),
11695            "the control response lost the line's capture time"
11696        );
11697    }
11698
11699    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
11700    /// read done on the worker thread would stall every other task until it
11701    /// finished; the read must run off the worker so this test's own task keeps
11702    /// running while the read is paused.
11703    #[tokio::test(flavor = "current_thread")]
11704    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
11705        let dir = ScratchDir::new("terminals-off-worker");
11706        let journal_path = dir.join("terminals.jsonl");
11707        let registry = Arc::new(Registry::default());
11708        let supervisor_handle = SupervisorHandle::new();
11709        let supervisor =
11710            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
11711                .with_handle(supervisor_handle.clone())
11712                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
11713        let module = supervisor
11714            .spawn(ModuleSpec {
11715                script: None,
11716                carry_containment: false,
11717                module_id: "terminal-off-worker".to_string(),
11718                program: fake_aft_stub_path(),
11719                args: Vec::new(),
11720                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11721                reserved: false,
11722                reserved_prefixes: Vec::new(),
11723                protocol: ModuleProtocol::Subc,
11724                overlap: Default::default(),
11725            })
11726            .unwrap();
11727        let deadline = Instant::now() + Duration::from_secs(5);
11728        while module.terminal_history().entries.len() != 2 {
11729            assert!(Instant::now() < deadline, "module did not record two exits");
11730            sleep(Duration::from_millis(10)).await;
11731        }
11732
11733        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
11734        let handler =
11735            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
11736        let frame = Frame::build(
11737            FrameType::Request,
11738            control_flags(),
11739            0,
11740            0,
11741            1,
11742            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
11743                module_id: "terminal-off-worker".to_string(),
11744            })
11745            .unwrap(),
11746        )
11747        .unwrap();
11748        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11749        let spawned_at = std::time::Instant::now();
11750        let read = tokio::spawn({
11751            let handler = Arc::clone(&handler);
11752            async move { handler.handle_control_frame(&ctx, frame).await }
11753        });
11754        // Waiting for the pause from a blocking thread keeps this task pending,
11755        // so the runtime's single worker is free to run the read task.
11756        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
11757            .await
11758            .unwrap()
11759            .expect("the history read reached its pause");
11760        let elapsed = spawned_at.elapsed();
11761        assert!(
11762            elapsed < Duration::from_secs(2) && !read.is_finished(),
11763            "this task could not run while the history read was paused \
11764             (resumed after {elapsed:?}, read finished: {})",
11765            read.is_finished()
11766        );
11767
11768        drop(release);
11769        let responses = read.await.unwrap().unwrap();
11770        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11771        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
11772            panic!("expected supervisor.terminals response");
11773        };
11774        assert_eq!(terminals.entries.len(), 2);
11775        assert_eq!(terminals.journal_skipped_lines, 0);
11776        assert_eq!(terminals.journal_read_errors, 0);
11777    }
11778
11779    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11780    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
11781        let registry = Arc::new(Registry::default());
11782        let supervisor_handle = SupervisorHandle::new();
11783        let supervisor =
11784            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
11785                .with_handle(supervisor_handle.clone());
11786        let module = supervisor
11787            .spawn(ModuleSpec {
11788                script: None,
11789                carry_containment: false,
11790                module_id: "terminal-golden".to_string(),
11791                program: fake_aft_stub_path(),
11792                args: Vec::new(),
11793                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11794                reserved: false,
11795                reserved_prefixes: Vec::new(),
11796                protocol: ModuleProtocol::Subc,
11797                overlap: Default::default(),
11798            })
11799            .unwrap();
11800
11801        let deadline = Instant::now() + Duration::from_secs(5);
11802        while module.terminal_history().entries.len() != 2 {
11803            assert!(
11804                Instant::now() < deadline,
11805                "module did not retain two terminal exits: {:?}",
11806                module.terminal_history()
11807            );
11808            sleep(Duration::from_millis(10)).await;
11809        }
11810
11811        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11812        let request = ClientControlRequest::SupervisorTerminals {
11813            module_id: "terminal-golden".to_string(),
11814        };
11815        let frame = Frame::build(
11816            FrameType::Request,
11817            control_flags(),
11818            0,
11819            0,
11820            1,
11821            serde_json::to_vec(&request).unwrap(),
11822        )
11823        .unwrap();
11824        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
11825        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
11826        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
11827        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
11828            panic!("expected supervisor.terminals response");
11829        };
11830        assert_eq!(terminals.entries.len(), 2);
11831        assert_eq!(terminals.dropped, 0);
11832
11833        let mut rendered = serde_json::to_value(response).unwrap();
11834        // Wall-clock fields are the observation contract, but not stable fixture
11835        // bytes; normalize only them after the real handler has shaped the response.
11836        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
11837        for (index, entry) in rendered["entries"]
11838            .as_array_mut()
11839            .expect("terminal response entries array")
11840            .iter_mut()
11841            .enumerate()
11842        {
11843            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
11844        }
11845
11846        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11847            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
11848        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
11849        if std::env::var_os("UPDATE_GOLDEN").is_some() {
11850            std::fs::write(&golden_path, &serialized).unwrap();
11851        }
11852        let expected: Value =
11853            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
11854        assert_eq!(rendered, expected);
11855    }
11856
11857    #[test]
11858    fn hello_registers_manifest_and_returns_ack() {
11859        let registry = Arc::new(Registry::default());
11860        let handler = ControlHandler::new(Arc::clone(&registry));
11861        let conn = ConnectionId::new(1);
11862
11863        let responses = handler
11864            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
11865            .unwrap();
11866
11867        assert_eq!(responses.len(), 1);
11868        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11869        assert_eq!(responses[0].header.channel, 0);
11870        assert_eq!(responses[0].header.corr, 7);
11871        let ack = parse_ack(&responses[0]);
11872        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11873        assert!(ack
11874            .subc_capabilities
11875            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
11876        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
11877        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
11878        assert!(ack
11879            .subc_ops
11880            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
11881        assert!(ack
11882            .subc_ops
11883            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
11884        assert!(ack
11885            .subc_ops
11886            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE_ROLE_DECLARATIONS.to_string()));
11887
11888        let registration = registry.get_module("aft").unwrap().unwrap();
11889        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
11890        assert_eq!(registration.state, ChannelState::Active);
11891        assert_eq!(registration.connection_id, conn);
11892        assert_eq!(registration.control_ops, module_baseline_control_ops());
11893        assert!(registration.manifest.display_name.is_none());
11894    }
11895
11896    #[test]
11897    fn hello_display_name_refuses_invalid_manifest_by_field() {
11898        let registry = Arc::new(Registry::default());
11899        let handler = ControlHandler::new(Arc::clone(&registry));
11900        for (index, invalid) in [
11901            json!(""),
11902            json!("a".repeat(41)),
11903            json!(" Routing"),
11904            json!("a\nb"),
11905            json!("a\u{202e}b"),
11906            json!(7),
11907        ]
11908        .into_iter()
11909        .enumerate()
11910        {
11911            let module_id = format!("bad-display-name-{index}");
11912            let mut body = serde_json::to_value(ModuleHelloBody {
11913                manifest: manifest(&module_id, PROTOCOL_VERSION),
11914                protocol_ver: PROTOCOL_VERSION,
11915                control_ops: None,
11916                launch_nonce: None,
11917            })
11918            .unwrap();
11919            body["manifest"]["display_name"] = invalid;
11920            let hello = Frame::build(
11921                FrameType::Hello,
11922                control_flags(),
11923                0,
11924                0,
11925                7,
11926                serde_json::to_vec(&body).unwrap(),
11927            )
11928            .unwrap();
11929            let response = handler
11930                .handle_control(ConnectionId::new(100 + index as u64), hello)
11931                .unwrap();
11932            assert_eq!(response[0].header.ty, FrameType::Error);
11933            let error = parse_error(&response[0]);
11934            assert_eq!(error["code"], "invalid_manifest");
11935            assert!(error["message"].as_str().unwrap().contains("display_name"));
11936            assert!(registry.get_module(&module_id).unwrap().is_none());
11937        }
11938    }
11939
11940    #[test]
11941    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
11942        let invalid_identifiers = [
11943            ("case_change", "credentials-Provider/v1"),
11944            ("leading_zero", "credentials-provider/v01"),
11945            ("trailing_hyphen", "credentials-provider-/v1"),
11946            ("consecutive_hyphens", "credentials--provider/v1"),
11947            ("uppercase", "Credentials-provider/v1"),
11948            ("missing_v", "credentials-provider/1"),
11949            ("whitespace", "credentials provider/v1"),
11950            ("zero_version", "credentials-provider/v0"),
11951            ("out_of_range_version", "credentials-provider/v4294967296"),
11952            (
11953                "overlength_name",
11954                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
11955            ),
11956        ];
11957        let mut cases = invalid_identifiers
11958            .into_iter()
11959            .map(|(name, identifier)| {
11960                (
11961                    format!("identifier_{name}"),
11962                    "capabilities.provides[0]".to_string(),
11963                    identifier.to_string(),
11964                    json!({ "provides": [identifier] }),
11965                    None,
11966                )
11967            })
11968            .collect::<Vec<_>>();
11969        cases.extend([
11970            (
11971                "unknown_need".to_string(),
11972                "capabilities.requires[0].need".to_string(),
11973                "deferred".to_string(),
11974                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
11975                None,
11976            ),
11977            (
11978                "duplicate_provides".to_string(),
11979                "capabilities.provides[1]".to_string(),
11980                "credentials-provider/v1".to_string(),
11981                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
11982                None,
11983            ),
11984            (
11985                "duplicate_must_never_reach".to_string(),
11986                "capabilities.must_never_reach[1]".to_string(),
11987                "credentials-provider/v1".to_string(),
11988                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
11989                None,
11990            ),
11991            (
11992                "duplicate_requires_same_need".to_string(),
11993                "capabilities.requires[1]".to_string(),
11994                "credentials-provider/v1".to_string(),
11995                json!({ "requires": [
11996                    { "capability": "credentials-provider/v1", "need": "required" },
11997                    { "capability": "credentials-provider/v1", "need": "required" }
11998                ] }),
11999                None,
12000            ),
12001            (
12002                "duplicate_requires_conflicting_need".to_string(),
12003                "capabilities.requires[1]".to_string(),
12004                "credentials-provider/v1".to_string(),
12005                json!({ "requires": [
12006                    { "capability": "credentials-provider/v1", "need": "required" },
12007                    { "capability": "credentials-provider/v1", "need": "optional" }
12008                ] }),
12009                None,
12010            ),
12011            (
12012                "capabilities_root_pointer".to_string(),
12013                "runtime_computed[0]".to_string(),
12014                "/capabilities".to_string(),
12015                json!({}),
12016                Some(json!(["/capabilities"])),
12017            ),
12018            (
12019                "capabilities_descendant_pointer".to_string(),
12020                "runtime_computed[0]".to_string(),
12021                "/capabilities/provides".to_string(),
12022                json!({}),
12023                Some(json!(["/capabilities/provides"])),
12024            ),
12025            (
12026                "malformed_pointer_without_leading_slash".to_string(),
12027                "runtime_computed[0]".to_string(),
12028                "capabilities".to_string(),
12029                json!({}),
12030                Some(json!(["capabilities"])),
12031            ),
12032            (
12033                "malformed_pointer_escape".to_string(),
12034                "runtime_computed[0]".to_string(),
12035                "/roles/~2/tools".to_string(),
12036                json!({}),
12037                Some(json!(["/roles/~2/tools"])),
12038            ),
12039            (
12040                "unknown_capabilities_field".to_string(),
12041                "capabilities.future".to_string(),
12042                "<array>".to_string(),
12043                json!({ "future": [] }),
12044                None,
12045            ),
12046        ]);
12047
12048        for (index, (name, field, value, capabilities, runtime_computed)) in
12049            cases.into_iter().enumerate()
12050        {
12051            let registry = Arc::new(Registry::default());
12052            let handler = ControlHandler::new(Arc::clone(&registry));
12053            let response = handler
12054                .handle_control(
12055                    ConnectionId::new((index + 1) as u64),
12056                    capability_grammar_hello_frame(
12057                        capabilities,
12058                        runtime_computed,
12059                        index as u64 + 1,
12060                    ),
12061                )
12062                .expect("invalid HELLO returns a refusal");
12063
12064            assert_eq!(response.len(), 1, "{name} must emit one refusal");
12065            let error = parse_error(&response[0]);
12066            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
12067            let message = error["message"]
12068                .as_str()
12069                .expect("error message is a string");
12070            assert!(
12071                message.contains(&field),
12072                "{name}: field missing from {message}"
12073            );
12074            assert!(
12075                message.contains(&value),
12076                "{name}: value missing from {message}"
12077            );
12078            assert_eq!(
12079                registry
12080                    .active_registration_count()
12081                    .expect("registry reads"),
12082                0,
12083                "{name}: refused HELLO must not create a catalog entry"
12084            );
12085        }
12086    }
12087
12088    #[test]
12089    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
12090        let registry = Arc::new(Registry::default());
12091        let handler = ControlHandler::new(Arc::clone(&registry));
12092        let capabilities = json!({
12093            "provides": ["credentials-provider/v1"],
12094            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
12095            "must_never_reach": ["federation-transport/v1"]
12096        });
12097        let response = handler
12098            .handle_control(
12099                ConnectionId::new(99),
12100                capability_grammar_hello_frame(
12101                    capabilities.clone(),
12102                    Some(json!(["/roles/0/tools"])),
12103                    99,
12104                ),
12105            )
12106            .expect("valid HELLO registers");
12107        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12108
12109        let request = Frame::build(
12110            FrameType::Request,
12111            control_flags(),
12112            0,
12113            0,
12114            100,
12115            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12116                .expect("catalog request serializes"),
12117        )
12118        .expect("catalog request frame builds");
12119        let response = handler
12120            .handle_catalog_list(request, None)
12121            .expect("catalog list succeeds");
12122        let ClientControlResponse::CatalogList { modules, .. } =
12123            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
12124        else {
12125            panic!("catalog request must return catalog.list");
12126        };
12127        assert_eq!(modules.len(), 1);
12128        assert_eq!(
12129            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
12130            capabilities
12131        );
12132    }
12133
12134    #[test]
12135    fn catalog_list_mirrors_management_operation_description() {
12136        let registry = Arc::new(Registry::default());
12137        let handler = ControlHandler::new(Arc::clone(&registry));
12138        let description = "List managed records and return their identifiers and metadata.";
12139        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
12140        manifest.provides = vec![ProviderRole::ManagementSurface {
12141            operations: vec![ManagementOperation {
12142                name: "records.list".to_string(),
12143                kind: ManagementOperationKind::Query,
12144                description: Some(description.to_string()),
12145            }],
12146            config_schema: json!({"type": "object"}),
12147            observability: vec![ObservabilitySurface {
12148                name: "records.stats".to_string(),
12149                kind: ObservabilityKind::Snapshot,
12150            }],
12151            identity_scope: vec![IdentityScope::Project],
12152            concurrency: Concurrency::ModuleManaged,
12153        }];
12154        registry
12155            .register_with_control_ops(
12156                manifest,
12157                PROTOCOL_VERSION,
12158                ConnectionId::new(99),
12159                Vec::new(),
12160            )
12161            .expect("described management manifest registers");
12162
12163        let request = Frame::build(
12164            FrameType::Request,
12165            control_flags(),
12166            0,
12167            0,
12168            100,
12169            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
12170                .expect("catalog request serializes"),
12171        )
12172        .expect("catalog request frame builds");
12173        let response = handler
12174            .handle_catalog_list(request, None)
12175            .expect("catalog list succeeds");
12176        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
12177        assert_eq!(
12178            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
12179            "catalog.list must preserve the declared operation description verbatim"
12180        );
12181    }
12182
12183    #[test]
12184    fn catalog_list_mirrors_registered_operation_delivery() {
12185        let registry = Arc::new(Registry::default());
12186        let handler = ControlHandler::new(Arc::clone(&registry));
12187        let delivery =
12188            json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
12189        let mut body = serde_json::to_value(ModuleHelloBody {
12190            manifest: manifest("watch-management", PROTOCOL_VERSION),
12191            protocol_ver: PROTOCOL_VERSION,
12192            control_ops: None,
12193            launch_nonce: None,
12194        })
12195        .unwrap();
12196        body["manifest"]["provides"] = json!([{
12197            "role": "management_surface",
12198            "operations": [
12199                {"name": "status", "kind": "query"},
12200                {"name": "future", "kind": "query"},
12201                {"name": "reset", "kind": "mutate"}
12202            ],
12203            "config_schema": {}, "observability": [], "identity_scope": []
12204        }]);
12205        body["manifest"]["operation_delivery"] = delivery.clone();
12206        let hello = Frame::build(
12207            FrameType::Hello,
12208            control_flags(),
12209            0,
12210            0,
12211            99,
12212            serde_json::to_vec(&body).unwrap(),
12213        )
12214        .unwrap();
12215        let response = handler
12216            .handle_control(ConnectionId::new(99), hello)
12217            .unwrap();
12218        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12219
12220        let (_, registrations) = registry.list_modules().unwrap();
12221        let registered = &registrations[0].manifest;
12222        assert!(registered.is_watch_operation("status"));
12223        assert!(!registered.is_watch_operation("future"));
12224        assert!(!registered.is_watch_operation("reset"));
12225        assert!(!registered.is_watch_operation("missing"));
12226
12227        let request = Frame::build(
12228            FrameType::Request,
12229            control_flags(),
12230            0,
12231            0,
12232            100,
12233            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12234        )
12235        .unwrap();
12236        let response = handler.handle_catalog_list(request, None).unwrap();
12237        let body: Value = serde_json::from_slice(&response[0].body).unwrap();
12238        assert_eq!(body["modules"][0]["operation_delivery"], delivery);
12239        let ClientControlResponse::CatalogList { modules, .. } =
12240            serde_json::from_slice(&response[0].body).unwrap()
12241        else {
12242            panic!("expected catalog.list");
12243        };
12244        assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
12245    }
12246
12247    #[test]
12248    fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
12249        let registry = Arc::new(Registry::default());
12250        let supervisor = SupervisorHandle::new();
12251        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12252        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
12253        let descriptor = json!({
12254            "schema": "ckios.richui.manifest/1",
12255            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12256            "bytes": 4096,
12257            "chunks": 4
12258        });
12259        let connection = ConnectionId::new(201);
12260        let hello = handler
12261            .handle_control(
12262                connection,
12263                hello_frame_with_rich_ui(
12264                    "vault",
12265                    PROTOCOL_VERSION,
12266                    201,
12267                    Some("the-real-nonce"),
12268                    descriptor.clone(),
12269                ),
12270            )
12271            .unwrap();
12272        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12273
12274        let request =
12275            Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
12276        let provides = registry
12277            .get_module_by_connection(connection)
12278            .unwrap()
12279            .unwrap()
12280            .manifest
12281            .provides;
12282        handler
12283            .handle_catalog_update(connection, request, provides, None, Some(false))
12284            .unwrap();
12285
12286        let request = Frame::build(
12287            FrameType::Request,
12288            control_flags(),
12289            0,
12290            0,
12291            203,
12292            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12293        )
12294        .unwrap();
12295        let response = handler.handle_catalog_list(request, None).unwrap();
12296        let ClientControlResponse::CatalogList { modules, .. } =
12297            serde_json::from_slice(&response[0].body).unwrap()
12298        else {
12299            panic!("expected catalog.list");
12300        };
12301        assert_eq!(
12302            serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
12303            descriptor
12304        );
12305        assert!(modules[0].nonce_verified);
12306    }
12307
12308    fn hello_with_role_declarations(declarations: Value) -> Frame {
12309        let mut body = serde_json::to_value(ModuleHelloBody {
12310            manifest: manifest("role-test", PROTOCOL_VERSION),
12311            protocol_ver: PROTOCOL_VERSION,
12312            control_ops: None,
12313            launch_nonce: None,
12314        })
12315        .unwrap();
12316        body["manifest"]["role_declarations"] = declarations;
12317        Frame::build(
12318            FrameType::Hello,
12319            control_flags(),
12320            0,
12321            0,
12322            301,
12323            serde_json::to_vec(&body).unwrap(),
12324        )
12325        .unwrap()
12326    }
12327
12328    fn update_role_declarations_frame(declarations: Value) -> Frame {
12329        Frame::build(
12330            FrameType::Request,
12331            control_flags(),
12332            0,
12333            0,
12334            303,
12335            serde_json::to_vec(&json!({
12336                "op": "catalog.update_role_declarations",
12337                "role_declarations": declarations,
12338            }))
12339            .unwrap(),
12340        )
12341        .unwrap()
12342    }
12343
12344    #[tokio::test]
12345    async fn role_declarations_update_replaces_then_clears_catalog() {
12346        let registry = Arc::new(Registry::default());
12347        let handler = ControlHandler::new(Arc::clone(&registry));
12348        let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12349        let initial = json!({"media-provider/v1": {"engines": ["old"]}, "old/v1": {}});
12350        hello_via_sink(
12351            &handler,
12352            &ctx,
12353            &mut rx,
12354            hello_with_role_declarations(initial.clone()),
12355        )
12356        .await;
12357        assert_eq!(
12358            role_declarations_catalog(&handler).0["role_declarations"],
12359            initial
12360        );
12361        let before = registry.get_module("role-test").unwrap().unwrap();
12362        for declarations in [
12363            json!({"media-provider/v1": {"engines": ["new"]}}),
12364            json!({}),
12365        ] {
12366            let response = handler
12367                .handle_control_frame(&ctx, update_role_declarations_frame(declarations.clone()))
12368                .await
12369                .unwrap();
12370            assert_eq!(response[0].header.ty, FrameType::Response);
12371            assert_eq!(
12372                serde_json::from_slice::<Value>(&response[0].body).unwrap(),
12373                json!({"op": "catalog.update_role_declarations"})
12374            );
12375            let (raw, entry) = role_declarations_catalog(&handler);
12376            assert_eq!(
12377                serde_json::to_value(entry.role_declarations).unwrap(),
12378                declarations
12379            );
12380            if declarations.as_object().unwrap().is_empty() {
12381                assert!(raw.get("role_declarations").is_none());
12382            } else {
12383                assert_eq!(raw["role_declarations"], declarations);
12384            }
12385            let after = registry.get_module("role-test").unwrap().unwrap();
12386            let mut expected = before.manifest.clone();
12387            expected.role_declarations = serde_json::from_value(declarations).unwrap();
12388            assert_eq!(after.manifest, expected);
12389            assert_eq!(after.ready, before.ready);
12390        }
12391    }
12392
12393    #[tokio::test]
12394    async fn role_declarations_update_refuses_invalid_maps_without_changing_catalog() {
12395        let registry = Arc::new(Registry::default());
12396        let handler = ControlHandler::new(registry);
12397        let (ctx, mut rx) = route_ctx(ConnectionId::new(301));
12398        let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12399        hello_via_sink(
12400            &handler,
12401            &ctx,
12402            &mut rx,
12403            hello_with_role_declarations(initial.clone()),
12404        )
12405        .await;
12406        for (invalid, reason) in [
12407            (json!(null), "role_declarations must be an object"),
12408            (json!([]), "role_declarations must be an object"),
12409            (
12410                json!({"bad key": {}}),
12411                "role_declarations key must use capability-name grammar",
12412            ),
12413            (
12414                json!({"media-provider/v1": []}),
12415                "role_declarations values must be objects",
12416            ),
12417            (
12418                json!({"media-provider/v1": {"padding": "x".repeat(16384)}}),
12419                "role_declarations exceeds 16384 compact JSON bytes",
12420            ),
12421        ] {
12422            let response = handler
12423                .handle_control_frame(&ctx, update_role_declarations_frame(invalid))
12424                .await
12425                .unwrap();
12426            let error = parse_error(&response[0]);
12427            assert_eq!(error["code"], "invalid_role_declarations");
12428            assert_eq!(error["message"], reason);
12429            assert_eq!(
12430                role_declarations_catalog(&handler).0["role_declarations"],
12431                initial
12432            );
12433        }
12434    }
12435
12436    #[tokio::test]
12437    async fn role_declarations_update_is_owned_by_the_registered_connection() {
12438        let registry = Arc::new(Registry::default());
12439        let handler = ControlHandler::new(Arc::clone(&registry));
12440        let (owner, mut owner_rx) = route_ctx(ConnectionId::new(301));
12441        let initial = json!({"media-provider/v1": {"engines": ["old"]}});
12442        hello_via_sink(
12443            &handler,
12444            &owner,
12445            &mut owner_rx,
12446            hello_with_role_declarations(initial.clone()),
12447        )
12448        .await;
12449        let (other, mut other_rx) = route_ctx(ConnectionId::new(302));
12450        let replacement = json!({"media-provider/v1": {"engines": ["other"]}});
12451        let response = handler
12452            .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12453            .await
12454            .unwrap();
12455        assert_eq!(parse_error(&response[0])["code"], "not_registered");
12456        hello_via_sink(
12457            &handler,
12458            &other,
12459            &mut other_rx,
12460            hello_frame("other", PROTOCOL_VERSION, 304),
12461        )
12462        .await;
12463        let response = handler
12464            .handle_control_frame(&other, update_role_declarations_frame(replacement.clone()))
12465            .await
12466            .unwrap();
12467        assert_eq!(response[0].header.ty, FrameType::Response);
12468        assert_eq!(
12469            serde_json::to_value(
12470                registry
12471                    .get_module("role-test")
12472                    .unwrap()
12473                    .unwrap()
12474                    .manifest
12475                    .role_declarations
12476            )
12477            .unwrap(),
12478            initial
12479        );
12480        assert_eq!(
12481            serde_json::to_value(
12482                registry
12483                    .get_module("other")
12484                    .unwrap()
12485                    .unwrap()
12486                    .manifest
12487                    .role_declarations
12488            )
12489            .unwrap(),
12490            replacement
12491        );
12492    }
12493
12494    fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
12495        let request = Frame::build(
12496            FrameType::Request,
12497            control_flags(),
12498            0,
12499            0,
12500            302,
12501            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12502        )
12503        .unwrap();
12504        let response = handler.handle_catalog_list(request, None).unwrap();
12505        let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
12506        let ClientControlResponse::CatalogList { mut modules, .. } =
12507            serde_json::from_slice(&response[0].body).unwrap()
12508        else {
12509            panic!("expected catalog.list");
12510        };
12511        (raw["modules"][0].clone(), modules.remove(0))
12512    }
12513
12514    #[test]
12515    fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
12516        let registry = Arc::new(Registry::default());
12517        let handler = ControlHandler::new(Arc::clone(&registry));
12518        let declarations = json!({
12519            "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
12520            "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
12521        });
12522        let connection = ConnectionId::new(301);
12523        let response = handler
12524            .handle_control(
12525                connection,
12526                hello_with_role_declarations(declarations.clone()),
12527            )
12528            .unwrap();
12529        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12530        let registered = registry
12531            .get_module_by_connection(connection)
12532            .unwrap()
12533            .unwrap()
12534            .manifest;
12535        // Compared as JSON values, not bytes: object key order is not part of
12536        // the relay's promise, and workspace builds unify serde_json's
12537        // preserve_order feature, which changes the order a literal keeps.
12538        assert_eq!(
12539            serde_json::to_value(&registered.role_declarations).unwrap(),
12540            declarations
12541        );
12542        let (raw, entry) = role_declarations_catalog(&handler);
12543        assert_eq!(raw["role_declarations"], declarations);
12544        assert_eq!(entry.role_declarations, registered.role_declarations);
12545
12546        let request =
12547            Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
12548        handler
12549            .handle_catalog_update(connection, request, registered.provides, None, Some(false))
12550            .unwrap();
12551        let (raw, entry) = role_declarations_catalog(&handler);
12552        assert_eq!(raw["role_declarations"], declarations);
12553        assert_eq!(entry.role_declarations, registered.role_declarations);
12554    }
12555
12556    fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
12557        let registry = Arc::new(Registry::default());
12558        let handler = ControlHandler::new(Arc::clone(&registry));
12559        let capture = EventCapture::default();
12560        let _subscriber =
12561            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12562        let response = handler
12563            .handle_control(
12564                ConnectionId::new(301),
12565                hello_with_role_declarations(declarations),
12566            )
12567            .unwrap();
12568        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12569        assert!(registry
12570            .get_module_by_connection(ConnectionId::new(301))
12571            .unwrap()
12572            .unwrap()
12573            .manifest
12574            .role_declarations
12575            .is_empty());
12576        let (raw, entry) = role_declarations_catalog(&handler);
12577        assert!(raw.get("role_declarations").is_none());
12578        assert!(entry.role_declarations.is_empty());
12579        let warnings: Vec<_> = capture
12580            .events()
12581            .into_iter()
12582            .filter(|event| {
12583                event.fields.get("message").map(String::as_str)
12584                    == Some("dropping whole role_declarations map from HELLO")
12585            })
12586            .collect();
12587        assert_eq!(warnings.len(), 1);
12588        assert_eq!(warnings[0].level, tracing::Level::WARN);
12589        assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
12590        assert_eq!(warnings[0].fields["reason"], reason);
12591    }
12592
12593    #[test]
12594    fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
12595        for value in [
12596            json!(null),
12597            json!(false),
12598            json!(0),
12599            json!("opaque"),
12600            json!([]),
12601        ] {
12602            assert_hello_drops_role_declarations_with_warning(
12603                json!({"routing/v1": {}, "broken/v1": value}),
12604                "role_declarations values must be objects",
12605            );
12606        }
12607        assert_hello_drops_role_declarations_with_warning(
12608            json!([]),
12609            "role_declarations must be an object",
12610        );
12611    }
12612
12613    #[test]
12614    fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
12615        assert_hello_drops_role_declarations_with_warning(
12616            json!({"routing/v1": {}, "bad key": {}}),
12617            "role_declarations key must use capability-name grammar",
12618        );
12619    }
12620
12621    #[test]
12622    fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
12623        assert_hello_drops_role_declarations_with_warning(
12624            json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
12625            "role_declarations exceeds 16384 compact JSON bytes",
12626        );
12627    }
12628
12629    fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
12630        let mut body = serde_json::to_value(ModuleHelloBody {
12631            manifest: manifest("flow-ops", PROTOCOL_VERSION),
12632            protocol_ver: PROTOCOL_VERSION,
12633            control_ops: None,
12634            launch_nonce: None,
12635        })
12636        .unwrap();
12637        body["manifest"]["provides"] = json!([{
12638            "role": "management_surface",
12639            "operations": [
12640                {"name": "diagnostics.findings", "kind": "query"},
12641                {"name": "reset", "kind": "mutate"}
12642            ],
12643            "config_schema": {}, "observability": [], "identity_scope": []
12644        }]);
12645        body["manifest"]["capabilities"] = json!({ "provides": capabilities });
12646        if let Some(flow_operations) = flow_operations {
12647            body["manifest"]["flow_operations"] = flow_operations;
12648        }
12649        Frame::build(
12650            FrameType::Hello,
12651            control_flags(),
12652            0,
12653            0,
12654            401,
12655            serde_json::to_vec(&body).unwrap(),
12656        )
12657        .unwrap()
12658    }
12659
12660    #[test]
12661    fn catalog_list_mirrors_flow_operations_unchanged() {
12662        // Order is kept as declared (not sorted) and an empty list stays an
12663        // affirmative "none", distinct from an absent declaration.
12664        for (capability, flow_operations) in [
12665            ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
12666            ("call-modes/v1", json!(["diagnostics.findings"])),
12667            ("flow-scopes/v1", json!([])),
12668        ] {
12669            let registry = Arc::new(Registry::default());
12670            let handler = ControlHandler::new(Arc::clone(&registry));
12671            let connection = ConnectionId::new(401);
12672            let response = handler
12673                .handle_control(
12674                    connection,
12675                    hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
12676                )
12677                .unwrap();
12678            assert_eq!(response[0].header.ty, FrameType::HelloAck);
12679            let registered = registry
12680                .get_module_by_connection(connection)
12681                .unwrap()
12682                .unwrap()
12683                .manifest;
12684            assert_eq!(
12685                serde_json::to_value(&registered.flow_operations).unwrap(),
12686                flow_operations
12687            );
12688            let (raw, entry) = role_declarations_catalog(&handler);
12689            assert_eq!(raw["flow_operations"], flow_operations);
12690            assert_eq!(entry.flow_operations, registered.flow_operations);
12691        }
12692
12693        let registry = Arc::new(Registry::default());
12694        let handler = ControlHandler::new(Arc::clone(&registry));
12695        let response = handler
12696            .handle_control(
12697                ConnectionId::new(401),
12698                hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
12699            )
12700            .unwrap();
12701        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12702        let (raw, entry) = role_declarations_catalog(&handler);
12703        assert!(raw.get("flow_operations").is_none());
12704        assert_eq!(entry.flow_operations, None);
12705    }
12706
12707    #[test]
12708    fn operation_deferral_hello_accepts_valid_and_invalid_maps_and_catalog_relays_only_valid() {
12709        let oversized: serde_json::Map<String, Value> = (0..257)
12710            .map(|n| (format!("tool:t{n}"), json!(true)))
12711            .collect();
12712        for (capabilities, map, valid) in [
12713            (
12714                json!(["scope-federated/v1"]),
12715                json!({"tool:read":true,"management:reset":true}),
12716                true,
12717            ),
12718            (json!(["scope-federated/v1"]), json!({}), true),
12719            (json!([]), json!({"tool:read":true}), false),
12720            (
12721                json!(["scope-federated/v1"]),
12722                json!({"tool:valid":true,"tool:bad":false}),
12723                false,
12724            ),
12725            (
12726                json!(["scope-federated/v1"]),
12727                json!({"tool:valid":true,"management:missing":true}),
12728                false,
12729            ),
12730            (
12731                json!(["scope-federated/v1"]),
12732                Value::Object(oversized),
12733                false,
12734            ),
12735        ] {
12736            let capture = EventCapture::default();
12737            let _subscriber = tracing::subscriber::set_default(
12738                tracing_subscriber::registry().with(capture.clone()),
12739            );
12740            let registry = Arc::new(Registry::default());
12741            let handler = ControlHandler::new(Arc::clone(&registry));
12742            let hello = hello_with_flow_operations(capabilities, None);
12743            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12744            body["manifest"]["operation_deferral"] = map.clone();
12745            let hello = Frame::build(
12746                FrameType::Hello,
12747                control_flags(),
12748                0,
12749                0,
12750                401,
12751                serde_json::to_vec(&body).unwrap(),
12752            )
12753            .unwrap();
12754            let response = handler
12755                .handle_control(ConnectionId::new(401), hello)
12756                .unwrap();
12757            assert_eq!(response[0].header.ty, FrameType::HelloAck);
12758            let (raw, entry) = role_declarations_catalog(&handler);
12759            if valid {
12760                assert_eq!(raw["operation_deferral"], map);
12761                assert_eq!(
12762                    serde_json::to_value(entry.operation_deferral.unwrap()).unwrap(),
12763                    map
12764                );
12765            } else {
12766                assert!(entry.operation_deferral.is_none());
12767                assert!(raw.get("operation_deferral").is_none());
12768                assert_eq!(
12769                    capture
12770                        .events()
12771                        .iter()
12772                        .filter(|event| event.fields.get("message").map(String::as_str)
12773                            == Some("dropping operation_deferral from HELLO"))
12774                        .count(),
12775                    1
12776                );
12777            }
12778        }
12779    }
12780
12781    fn assert_hello_drops_flow_operations_with_warning(
12782        capabilities: Value,
12783        flow_operations: Value,
12784        reason: &str,
12785    ) {
12786        let registry = Arc::new(Registry::default());
12787        let handler = ControlHandler::new(Arc::clone(&registry));
12788        let capture = EventCapture::default();
12789        let _subscriber =
12790            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12791        let response = handler
12792            .handle_control(
12793                ConnectionId::new(401),
12794                hello_with_flow_operations(capabilities, Some(flow_operations)),
12795            )
12796            .unwrap();
12797        assert_eq!(response[0].header.ty, FrameType::HelloAck);
12798        let registered = registry
12799            .get_module_by_connection(ConnectionId::new(401))
12800            .unwrap()
12801            .expect("the module still registers")
12802            .manifest;
12803        assert_eq!(registered.module_id, "flow-ops");
12804        assert_eq!(registered.flow_operations, None);
12805        let (raw, entry) = role_declarations_catalog(&handler);
12806        assert_eq!(raw["module_id"], "flow-ops");
12807        assert!(raw.get("flow_operations").is_none());
12808        assert_eq!(entry.flow_operations, None);
12809        let warnings: Vec<_> = capture
12810            .events()
12811            .into_iter()
12812            .filter(|event| {
12813                event.fields.get("message").map(String::as_str)
12814                    == Some("dropping flow_operations from HELLO")
12815            })
12816            .collect();
12817        assert_eq!(warnings.len(), 1);
12818        assert_eq!(warnings[0].level, tracing::Level::WARN);
12819        assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
12820        assert_eq!(warnings[0].fields["reason"], reason);
12821    }
12822
12823    #[test]
12824    fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
12825        assert_hello_drops_flow_operations_with_warning(
12826            json!(["flow-scopes/v1"]),
12827            json!(["diagnostics.findings", "missing"]),
12828            "flow_operations names 'missing', which is not a declared management operation",
12829        );
12830    }
12831
12832    #[test]
12833    fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
12834        assert_hello_drops_flow_operations_with_warning(
12835            json!([]),
12836            json!(["diagnostics.findings"]),
12837            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12838        );
12839        // A run-only capability does not admit flows either.
12840        assert_hello_drops_flow_operations_with_warning(
12841            json!(["agent-run-scopes/v1"]),
12842            json!([]),
12843            "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
12844        );
12845    }
12846
12847    #[test]
12848    fn hello_malformed_flow_operations_is_dropped_with_warning() {
12849        assert_hello_drops_flow_operations_with_warning(
12850            json!(["flow-scopes/v1"]),
12851            json!(["reset", "reset"]),
12852            "flow_operations names 'reset' more than once",
12853        );
12854        assert_hello_drops_flow_operations_with_warning(
12855            json!(["flow-scopes/v1"]),
12856            json!(vec!["reset"; 257]),
12857            "flow_operations lists 257 names, more than 256",
12858        );
12859        assert_hello_drops_flow_operations_with_warning(
12860            json!(["flow-scopes/v1"]),
12861            json!("reset"),
12862            "flow_operations must be an array of operation names",
12863        );
12864    }
12865
12866    #[test]
12867    fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
12868        let registry = Arc::new(Registry::default());
12869        let handler = ControlHandler::new(registry.clone());
12870        let descriptor = json!({
12871            "schema": "ckios.richui.manifest/1",
12872            "manifest_digest": format!("sha256:{}", "a".repeat(64)),
12873            "bytes": 128,
12874            "chunks": 2
12875        });
12876        let malformed = [
12877            ("schema", json!("")),
12878            ("schema", json!("a".repeat(65))),
12879            ("schema", json!("Bad schema")),
12880            (
12881                "manifest_digest",
12882                json!(format!("sha256:{}", "A".repeat(64))),
12883            ),
12884            ("manifest_digest", json!("sha256:abc")),
12885            ("bytes", json!(0)),
12886            ("bytes", json!(16_777_217)),
12887            ("bytes", json!("128")),
12888            ("chunks", json!(0)),
12889            ("chunks", json!(1025)),
12890            ("chunks", json!("2")),
12891        ];
12892        for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
12893            let mut invalid = descriptor.clone();
12894            invalid[field] = invalid_value;
12895            let response = handler
12896                .handle_control(
12897                    ConnectionId::new(210 + index as u64),
12898                    hello_frame_with_rich_ui(
12899                        &format!("bad-rich-ui-{index}"),
12900                        PROTOCOL_VERSION,
12901                        210 + index as u64,
12902                        None,
12903                        invalid,
12904                    ),
12905                )
12906                .unwrap();
12907            assert_eq!(response[0].header.ty, FrameType::Error);
12908            let error = parse_error(&response[0]);
12909            assert_eq!(error["code"], "invalid_manifest");
12910            assert!(error["message"]
12911                .as_str()
12912                .unwrap()
12913                .contains(&format!("rich_ui.{field}")));
12914            assert!(registry
12915                .get_module(&format!("bad-rich-ui-{index}"))
12916                .unwrap()
12917                .is_none());
12918        }
12919    }
12920
12921    #[test]
12922    fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
12923        let registry = Arc::new(Registry::default());
12924        let handler = ControlHandler::new(Arc::clone(&registry));
12925        let connection = ConnectionId::new(219);
12926        registry
12927            .register_with_control_ops(
12928                ModuleManifest::builder("invalid-rich-ui", "0.1.0")
12929                    .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
12930                        "bad schema",
12931                        format!("sha256:{}", "a".repeat(64)),
12932                        128,
12933                        2,
12934                    )))
12935                    .build(),
12936                PROTOCOL_VERSION,
12937                connection,
12938                Vec::new(),
12939            )
12940            .unwrap();
12941        let request =
12942            Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
12943        let response = handler
12944            .handle_catalog_update(connection, request, Vec::new(), None, None)
12945            .unwrap();
12946        assert_eq!(response[0].header.ty, FrameType::Error);
12947        let error = parse_error(&response[0]);
12948        assert_eq!(error["code"], "invalid_manifest");
12949        assert!(error["message"]
12950            .as_str()
12951            .unwrap()
12952            .contains("rich_ui.schema"));
12953    }
12954
12955    #[test]
12956    fn swap_candidate_hello_is_nonce_verified_after_promotion() {
12957        let registry = Arc::new(Registry::default());
12958        let supervisor = SupervisorHandle::new();
12959        supervisor.open_swap("provider", "candidate-nonce".to_string());
12960        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
12961        let connection = ConnectionId::new(218);
12962        let hello = handler
12963            .handle_control(
12964                connection,
12965                hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
12966            )
12967            .unwrap();
12968        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
12969        assert!(registry.list_modules().unwrap().1.is_empty());
12970
12971        registry.promote_candidate("provider").unwrap().unwrap();
12972        let request = Frame::build(
12973            FrameType::Request,
12974            control_flags(),
12975            0,
12976            0,
12977            219,
12978            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
12979        )
12980        .unwrap();
12981        let response = handler.handle_catalog_list(request, None).unwrap();
12982        let ClientControlResponse::CatalogList { modules, .. } =
12983            serde_json::from_slice(&response[0].body).unwrap()
12984        else {
12985            panic!("expected catalog.list");
12986        };
12987        assert_eq!(modules.len(), 1);
12988        assert!(modules[0].nonce_verified);
12989    }
12990
12991    #[test]
12992    fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
12993        for provenance_json in [
12994            r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
12995            r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
12996        ] {
12997            let registry = Arc::new(Registry::default());
12998            let handler = ControlHandler::new(registry);
12999            let hello = Frame::build(
13000                FrameType::Hello,
13001                control_flags(),
13002                0,
13003                0,
13004                220,
13005                serde_json::to_vec(&ModuleHelloBody {
13006                    manifest: ModuleManifest::builder("built", "0.1.0")
13007                        .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
13008                        .build(),
13009                    protocol_ver: PROTOCOL_VERSION,
13010                    control_ops: None,
13011                    launch_nonce: None,
13012                })
13013                .unwrap(),
13014            )
13015            .unwrap();
13016            assert_eq!(
13017                handler
13018                    .handle_control(ConnectionId::new(220), hello)
13019                    .unwrap()[0]
13020                    .header
13021                    .ty,
13022                FrameType::HelloAck
13023            );
13024            let request = Frame::build(
13025                FrameType::Request,
13026                control_flags(),
13027                0,
13028                0,
13029                221,
13030                serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13031            )
13032            .unwrap();
13033            let response = handler.handle_catalog_list(request, None).unwrap();
13034            let ClientControlResponse::CatalogList { modules, .. } =
13035                serde_json::from_slice(&response[0].body).unwrap()
13036            else {
13037                panic!("expected catalog.list");
13038            };
13039            assert_eq!(modules.len(), 1);
13040            assert_eq!(
13041                serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
13042                format!(
13043                    r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
13044                )
13045                .as_bytes()
13046            );
13047        }
13048    }
13049
13050    #[test]
13051    fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
13052        let registry = Arc::new(Registry::default());
13053        let handler = ControlHandler::new(Arc::clone(&registry));
13054        let hello = Frame::build(
13055            FrameType::Hello,
13056            control_flags(),
13057            0,
13058            0,
13059            220,
13060            serde_json::to_vec(&ModuleHelloBody {
13061                manifest: ModuleManifest::builder("plain", "0.1.0").build(),
13062                protocol_ver: PROTOCOL_VERSION,
13063                control_ops: None,
13064                launch_nonce: Some("unverified".to_string()),
13065            })
13066            .unwrap(),
13067        )
13068        .unwrap();
13069        assert_eq!(
13070            handler
13071                .handle_control(ConnectionId::new(220), hello)
13072                .unwrap()[0]
13073                .header
13074                .ty,
13075            FrameType::HelloAck
13076        );
13077        let request = Frame::build(
13078            FrameType::Request,
13079            control_flags(),
13080            0,
13081            0,
13082            221,
13083            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
13084        )
13085        .unwrap();
13086        let response = handler.handle_catalog_list(request, None).unwrap();
13087        let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
13088        assert!(wire["modules"][0].get("provenance").is_none());
13089        let ClientControlResponse::CatalogList { modules, .. } =
13090            serde_json::from_slice(&response[0].body).unwrap()
13091        else {
13092            panic!("expected catalog.list");
13093        };
13094        assert_eq!(
13095            serde_json::to_vec(&catalog_legacy_projection(modules[0].clone())).unwrap(),
13096            br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
13097        );
13098        assert!(!modules[0].nonce_verified);
13099        assert!(modules[0].provenance.is_none());
13100    }
13101
13102    // Preserve the legacy-field byte pins while discovery adds optional facts.
13103    fn catalog_legacy_projection(mut entry: CatalogEntry) -> CatalogEntry {
13104        assert_eq!(entry.first_party, Some(false));
13105        entry.incarnation = None;
13106        entry.spawn = None;
13107        entry.first_party = None;
13108        entry.origin = None;
13109        entry.requested_declarations_digest = None;
13110        entry.approved_declarations_digest = None;
13111        entry
13112    }
13113
13114    #[test]
13115    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
13116        let registry = Arc::new(Registry::default());
13117        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
13118            [("vault".to_string(), true), ("squatter".to_string(), true)],
13119            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13120        );
13121        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
13122        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13123            provides: vec!["credentials-provider/v1".to_string()],
13124            requires: Vec::new(),
13125            must_never_reach: Vec::new(),
13126        });
13127        let frame = Frame::build(
13128            FrameType::Hello,
13129            control_flags(),
13130            0,
13131            0,
13132            77,
13133            serde_json::to_vec(&ModuleHelloBody {
13134                manifest: squatter,
13135                protocol_ver: PROTOCOL_VERSION,
13136                control_ops: None,
13137                launch_nonce: None,
13138            })
13139            .expect("HELLO serializes"),
13140        )
13141        .expect("HELLO frame builds");
13142        let response = handler
13143            .handle_control(ConnectionId::new(77), frame)
13144            .expect("reserved claim receives a typed refusal");
13145        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13146        assert_eq!(
13147            registry
13148                .active_registration_count()
13149                .expect("registry reads"),
13150            0,
13151            "a reserved capability refusal must not leave a catalog entry"
13152        );
13153    }
13154
13155    #[test]
13156    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
13157        for settlement in ["timeout", "inconclusive", "drop"] {
13158            let breakers = RouteBindBreakers::default();
13159            let RouteBindAdmission::Admitted {
13160                guard: mut old,
13161                probe: false,
13162            } = breakers.admit("prov")
13163            else {
13164                panic!("ordinary relay admitted")
13165            };
13166            let RouteBindAdmission::Admitted {
13167                guard: mut opener, ..
13168            } = breakers.admit("prov")
13169            else {
13170                panic!("second relay admitted")
13171            };
13172            assert!(
13173                !opener
13174                    .record_timeout(1, Duration::ZERO)
13175                    .unwrap()
13176                    .reopened_after_probe
13177            );
13178            let RouteBindAdmission::Admitted {
13179                guard: mut probe,
13180                probe: true,
13181            } = breakers.admit("prov")
13182            else {
13183                panic!("one cooldown probe admitted")
13184            };
13185            match settlement {
13186                "timeout" => assert!(
13187                    !old.record_timeout(1, Duration::ZERO)
13188                        .unwrap()
13189                        .reopened_after_probe
13190                ),
13191                "inconclusive" => old.record_inconclusive(),
13192                "drop" => drop(old),
13193                _ => unreachable!(),
13194            }
13195            assert!(
13196                matches!(
13197                    breakers.admit("prov"),
13198                    RouteBindAdmission::Refused {
13199                        probe_in_flight: true,
13200                        ..
13201                    }
13202                ),
13203                "{settlement} of a pre-open relay cannot release the real probe"
13204            );
13205            assert!(
13206                probe
13207                    .record_timeout(1, Duration::ZERO)
13208                    .unwrap()
13209                    .reopened_after_probe
13210            );
13211            assert!(matches!(
13212                breakers.admit("prov"),
13213                RouteBindAdmission::Admitted { probe: true, .. }
13214            ));
13215        }
13216        let breakers = RouteBindBreakers::default();
13217        let admit = || match breakers.admit("prov") {
13218            RouteBindAdmission::Admitted { guard, .. } => guard,
13219            _ => panic!("relay admitted"),
13220        };
13221        admit().record_timeout(1, Duration::ZERO);
13222        let mut old_probe = admit();
13223        breakers.reset_for_new_module_connection("prov");
13224        admit().record_timeout(1, Duration::ZERO);
13225        let _new_probe = admit();
13226        old_probe.record_inconclusive();
13227        assert!(matches!(
13228            breakers.admit("prov"),
13229            RouteBindAdmission::Refused {
13230                probe_in_flight: true,
13231                ..
13232            }
13233        ));
13234    }
13235
13236    #[tokio::test]
13237    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
13238        for candidate in [false, true] {
13239            let registry = Arc::new(Registry::default());
13240            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
13241                [("vault".to_string(), true), ("squatter".to_string(), true)],
13242                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
13243            );
13244            let conn = ConnectionId::new(77);
13245            let (ctx, mut rx) = route_ctx(conn);
13246            let initial = capability_manifest("squatter", &[], &[]);
13247            if candidate {
13248                registry
13249                    .register_candidate_with_control_ops(
13250                        initial.clone(),
13251                        PROTOCOL_VERSION,
13252                        conn,
13253                        module_baseline_control_ops(),
13254                    )
13255                    .unwrap();
13256                handler
13257                    .forwarding
13258                    .register_candidate_module_connection(
13259                        conn,
13260                        "squatter".to_string(),
13261                        PROTOCOL_VERSION,
13262                        manifest_concurrency(&initial),
13263                        ctx.egress.clone(),
13264                    )
13265                    .unwrap();
13266            } else {
13267                hello_via_sink(
13268                    &handler,
13269                    &ctx,
13270                    &mut rx,
13271                    hello_frame_with_manifest(initial.clone(), 1),
13272                )
13273                .await;
13274            }
13275            let response = handler
13276                .handle_control_frame(
13277                    &ctx,
13278                    catalog_update_with_capabilities_frame(
13279                        2,
13280                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
13281                            .capabilities
13282                            .unwrap(),
13283                    ),
13284                )
13285                .await
13286                .unwrap();
13287            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
13288            assert_eq!(
13289                registry
13290                    .get_module_by_connection(conn)
13291                    .unwrap()
13292                    .unwrap()
13293                    .manifest,
13294                initial
13295            );
13296        }
13297    }
13298
13299    #[test]
13300    #[cfg(not(feature = "dev-desktop-approval"))]
13301    fn server_describe_default_build_advertises_no_config_capability() {
13302        let assert_default = |handler: &ControlHandler| {
13303            let request = Frame::build(
13304                FrameType::Request,
13305                control_flags(),
13306                0,
13307                0,
13308                79,
13309                serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13310            )
13311            .unwrap();
13312            let responses = handler.handle_server_describe(request).unwrap();
13313            assert_eq!(responses.len(), 1);
13314            let response: Value = serde_json::from_slice(&responses[0].body).unwrap();
13315            assert_eq!(response["op"], "server.describe");
13316            let capabilities = response["capabilities"]
13317                .as_array()
13318                .expect("capability array");
13319            assert!(
13320                !capabilities.is_empty(),
13321                "exercise an actual describe response, not an empty fixture"
13322            );
13323            assert!(
13324                capabilities
13325                    .iter()
13326                    .all(|capability| !capability.as_str().unwrap().starts_with("config.")),
13327                "a default build must not announce configuration authority: {capabilities:?}"
13328            );
13329        };
13330        assert_default(&ControlHandler::new(Arc::new(Registry::default())));
13331        #[cfg(unix)]
13332        {
13333            // A local store does not prove that project-membership validation,
13334            // active-scope usage tracking and approved-program launch selection
13335            // are connected. Defaults must not advertise that authority.
13336            let (_rig, _owner, _supervisor, handler, _spec) =
13337                crate::config_gate::runtime::RuntimeOwner::unactivated_supervisor_fixture_for_test(
13338                );
13339            assert_default(&handler);
13340        }
13341    }
13342
13343    #[test]
13344    fn server_describe_surfaces_required_capability_verdict_fields() {
13345        let registry = Arc::new(Registry::default());
13346        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
13347            [
13348                ("consumer".to_string(), true),
13349                ("provider".to_string(), false),
13350            ],
13351            BTreeMap::new(),
13352        );
13353        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13354        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
13355            provides: Vec::new(),
13356            requires: vec![subc_protocol::manifest::CapabilityRequirement {
13357                capability: "credentials-provider/v1".to_string(),
13358                need: subc_protocol::manifest::CapabilityNeed::Required,
13359            }],
13360            must_never_reach: Vec::new(),
13361        });
13362        let hello = Frame::build(
13363            FrameType::Hello,
13364            control_flags(),
13365            0,
13366            0,
13367            78,
13368            serde_json::to_vec(&ModuleHelloBody {
13369                manifest: consumer,
13370                protocol_ver: PROTOCOL_VERSION,
13371                control_ops: None,
13372                launch_nonce: None,
13373            })
13374            .expect("HELLO serializes"),
13375        )
13376        .expect("HELLO frame builds");
13377        handler
13378            .handle_control(ConnectionId::new(78), hello)
13379            .expect("consumer registers");
13380        let describe = Frame::build(
13381            FrameType::Request,
13382            control_flags(),
13383            0,
13384            0,
13385            79,
13386            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
13387                .expect("request serializes"),
13388        )
13389        .expect("describe frame builds");
13390        let response = handler
13391            .handle_server_describe(describe)
13392            .expect("server.describe succeeds");
13393        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
13394        let requirement = &rendered["capability_requirements"][0];
13395        assert_eq!(requirement["consumer"], "consumer");
13396        assert_eq!(requirement["verdict"], "never_provided");
13397        assert_eq!(requirement["episode_seq"], 1);
13398        assert_eq!(requirement["config_satisfiable"], false);
13399        assert_eq!(requirement["runtime_available"], false);
13400        assert!(requirement["detail"]
13401            .as_str()
13402            .expect("detail string")
13403            .contains("credentials-provider/v1"));
13404    }
13405
13406    /// Register a consumer that requires `project-identity/v1` and the provider
13407    /// of it, optionally mark daemon shutdown as begun, then deregister the
13408    /// provider as its connection closes. Returns the `capability.requirement`
13409    /// events that deregistration logged and the verdict the evaluator then
13410    /// holds for the requirement (the one `route.open` refusals read).
13411    fn deregister_required_provider(
13412        daemon_shutdown_begun: bool,
13413    ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
13414        const CAPABILITY: &str = "project-identity/v1";
13415        let registry = Arc::new(Registry::default());
13416        let daemon_shutdown = DaemonShutdownFlag::default();
13417        let handler = ControlHandler::new(Arc::clone(&registry))
13418            .with_capability_config(
13419                [
13420                    ("consumer".to_string(), true),
13421                    ("provider".to_string(), true),
13422                ],
13423                BTreeMap::new(),
13424            )
13425            .with_daemon_shutdown_flag(daemon_shutdown.clone());
13426        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
13427        consumer.capabilities = Some(CapabilityDeclarations {
13428            provides: Vec::new(),
13429            requires: vec![subc_protocol::manifest::CapabilityRequirement {
13430                capability: CAPABILITY.to_string(),
13431                need: subc_protocol::manifest::CapabilityNeed::Required,
13432            }],
13433            must_never_reach: Vec::new(),
13434        });
13435        let provider = capability_manifest("provider", &[CAPABILITY], &[]);
13436        let provider_conn = ConnectionId::new(182);
13437        for (conn, manifest, corr) in [
13438            (ConnectionId::new(181), consumer, 181),
13439            (provider_conn, provider, 182),
13440        ] {
13441            let hello = handler
13442                .handle_control(conn, hello_frame_with_manifest(manifest, corr))
13443                .expect("module registers");
13444            assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13445        }
13446        assert_eq!(
13447            handler.capability_evaluator.verdict("consumer", CAPABILITY),
13448            Some(CapabilityVerdict::Provided),
13449            "the registered provider must satisfy the requirement first"
13450        );
13451        if daemon_shutdown_begun {
13452            daemon_shutdown.set_for_test();
13453        }
13454
13455        let capture = EventCapture::default();
13456        let _subscriber =
13457            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13458        handler
13459            .cleanup_connection(provider_conn)
13460            .expect("provider deregisters");
13461        let events = capture
13462            .events()
13463            .into_iter()
13464            .filter(|event| {
13465                event.fields.get("message").map(String::as_str) == Some("capability.requirement")
13466            })
13467            .collect();
13468        (
13469            events,
13470            handler.capability_evaluator.verdict("consumer", CAPABILITY),
13471        )
13472    }
13473
13474    #[test]
13475    fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
13476        let (events, verdict) = deregister_required_provider(true);
13477        assert_eq!(
13478            verdict,
13479            Some(CapabilityVerdict::NeverProvided),
13480            "shutdown must not change the verdict route.open refusals read"
13481        );
13482        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13483        let event = &events[0];
13484        assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
13485        assert_eq!(
13486            event.fields.get("cause").map(String::as_str),
13487            Some("\"daemon_shutdown\""),
13488            "{event:?}"
13489        );
13490        assert_eq!(
13491            event.fields.get("verdict").map(String::as_str),
13492            Some("\"never_provided\""),
13493            "{event:?}"
13494        );
13495    }
13496
13497    #[test]
13498    fn required_provider_leaving_while_daemon_runs_still_logs_error() {
13499        let (events, verdict) = deregister_required_provider(false);
13500        assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
13501        assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
13502        let event = &events[0];
13503        assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
13504        assert_eq!(event.fields.get("cause"), None, "{event:?}");
13505        assert_eq!(
13506            event.fields.get("verdict").map(String::as_str),
13507            Some("\"never_provided\""),
13508            "{event:?}"
13509        );
13510    }
13511
13512    #[test]
13513    fn catalog_list_omits_capabilities_for_legacy_manifest() {
13514        let registry = Arc::new(Registry::default());
13515        let handler = ControlHandler::new(Arc::clone(&registry));
13516        let hello = handler
13517            .handle_control(
13518                ConnectionId::new(101),
13519                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
13520            )
13521            .expect("legacy HELLO registers");
13522        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
13523
13524        let request = Frame::build(
13525            FrameType::Request,
13526            control_flags(),
13527            0,
13528            0,
13529            102,
13530            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
13531                .expect("catalog request serializes"),
13532        )
13533        .expect("catalog request frame builds");
13534        let response = handler
13535            .handle_catalog_list(request, None)
13536            .expect("catalog list succeeds");
13537        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
13538        assert!(
13539            body["modules"][0].get("capabilities").is_none(),
13540            "legacy manifest must retain an absent capabilities field on catalog.list"
13541        );
13542    }
13543
13544    #[test]
13545    fn hello_ack_omits_storage_when_no_storage_config() {
13546        let registry = Arc::new(Registry::default());
13547        let handler = ControlHandler::new(Arc::clone(&registry));
13548        let responses = handler
13549            .handle_control(
13550                ConnectionId::new(1),
13551                hello_frame("aft", PROTOCOL_VERSION, 7),
13552            )
13553            .unwrap();
13554        let ack = parse_ack(&responses[0]);
13555        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
13556        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
13557    }
13558
13559    #[tokio::test]
13560    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
13561        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
13562        let registry = Arc::new(Registry::default());
13563        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
13564        let responses = handler
13565            .handle_control(
13566                ConnectionId::new(1),
13567                hello_frame("aft", PROTOCOL_VERSION, 7),
13568            )
13569            .unwrap();
13570        let ack = parse_ack(&responses[0]);
13571        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
13572
13573        let described = handler
13574            .handle_control_frame(
13575                &route_ctx(ConnectionId::new(2)).0,
13576                Frame::build(
13577                    FrameType::Request,
13578                    control_flags(),
13579                    0,
13580                    0,
13581                    9,
13582                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13583                )
13584                .unwrap(),
13585            )
13586            .await
13587            .unwrap();
13588        let ClientControlResponse::ServerDescribe { machine_id, .. } =
13589            serde_json::from_slice(&described[0].body).unwrap()
13590        else {
13591            panic!("server.describe answered with another shape");
13592        };
13593        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
13594    }
13595
13596    #[test]
13597    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
13598        // With a central sqlite storage policy, each registering module gets its
13599        // own resolved descriptor in HELLO_ACK, keyed by its module id.
13600        let registry = Arc::new(Registry::default());
13601        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
13602            crate::daemon_config::StorageConfig::Sqlite {
13603                data_home: std::path::PathBuf::from("/data"),
13604            },
13605        ));
13606
13607        let responses = handler
13608            .handle_control(
13609                ConnectionId::new(1),
13610                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
13611            )
13612            .unwrap();
13613        let ack = parse_ack(&responses[0]);
13614        assert_eq!(
13615            ack.storage,
13616            Some(serde_json::json!({
13617                "module_id": "alfonso-routing",
13618                "storage_namespace": "default",
13619                "isolation": { "kind": "module" },
13620                "backend": {
13621                    "backend": "sqlite",
13622                    "path": "/data/cortexkit/alfonso-routing/store.db"
13623                }
13624            })),
13625            "the delivered descriptor is the module's own sqlite store path"
13626        );
13627    }
13628
13629    #[test]
13630    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
13631        let registry = Arc::new(Registry::default());
13632        let handler = ControlHandler::new(Arc::clone(&registry));
13633        let conn = ConnectionId::new(1);
13634        let responses = handler
13635            .handle_control(
13636                conn,
13637                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13638            )
13639            .unwrap();
13640        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
13641        let registration = registry.get_module("aft").unwrap().unwrap();
13642        assert_eq!(registration.control_ops, module_baseline_control_ops());
13643
13644        let frame =
13645            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
13646        assert!(handler
13647            .guard_module_control_op(&frame, "aft", "route.bind")
13648            .unwrap()
13649            .is_none());
13650        let error = handler
13651            .guard_module_control_op(&frame, "aft", "test.synthetic")
13652            .unwrap()
13653            .expect("synthetic ungranted op should be rejected");
13654        assert_eq!(error.header.ty, FrameType::Error);
13655        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
13656    }
13657
13658    #[test]
13659    fn hello_control_ops_some_adds_optional_grants() {
13660        let registry = Arc::new(Registry::default());
13661        let handler = ControlHandler::new(Arc::clone(&registry));
13662        handler
13663            .handle_control(
13664                ConnectionId::new(1),
13665                hello_frame_with_control_ops(
13666                    "aft",
13667                    PROTOCOL_VERSION,
13668                    7,
13669                    Some(vec![
13670                        "future.synthetic".to_string(),
13671                        "route.bind".to_string(),
13672                    ]),
13673                ),
13674            )
13675            .unwrap();
13676        let registration = registry.get_module("aft").unwrap().unwrap();
13677        assert_eq!(
13678            registration.control_ops,
13679            vec![
13680                "route.bind".to_string(),
13681                "route.status".to_string(),
13682                "future.synthetic".to_string(),
13683            ]
13684        );
13685        let frame =
13686            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
13687        assert!(handler
13688            .guard_module_control_op(&frame, "aft", "future.synthetic")
13689            .unwrap()
13690            .is_none());
13691    }
13692
13693    #[tokio::test]
13694    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
13695        let registry = Arc::new(Registry::default());
13696        let forwarding = Arc::new(ForwardingTable::default());
13697        let handler =
13698            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13699        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
13700        hello_via_sink(
13701            &handler,
13702            &module_ctx,
13703            &mut module_rx,
13704            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
13705        )
13706        .await;
13707
13708        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
13709        let responses = handler
13710            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
13711            .await
13712            .unwrap();
13713        assert_eq!(responses.len(), 1);
13714        assert_eq!(responses[0].header.ty, FrameType::Error);
13715        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
13716        assert!(module_rx.try_recv().is_err());
13717    }
13718
13719    #[tokio::test]
13720    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
13721        let registry = Arc::new(Registry::default());
13722        let forwarding = Arc::new(ForwardingTable::default());
13723        let handler =
13724            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13725        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
13726        hello_via_sink(
13727            &handler,
13728            &module_ctx,
13729            &mut module_rx,
13730            hello_frame_with_control_ops(
13731                "aft",
13732                PROTOCOL_VERSION,
13733                7,
13734                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13735            ),
13736        )
13737        .await;
13738
13739        let project_root = unique_project_root("demux");
13740        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
13741        let route_handler = handler.clone();
13742        let route_task = tokio::spawn(async move {
13743            route_handler
13744                .handle_control_frame(
13745                    &route_client_ctx,
13746                    route_open_frame(100, "aft", project_root),
13747                )
13748                .await
13749                .unwrap()
13750        });
13751        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13752            .await
13753            .unwrap()
13754            .unwrap();
13755        assert!(matches!(
13756            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
13757            ModuleControlRequest::RouteBind { .. }
13758        ));
13759
13760        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
13761        let health_handler = handler.clone();
13762        let health_task = tokio::spawn(async move {
13763            health_handler
13764                .handle_control_frame(
13765                    &health_client_ctx,
13766                    supervisor_health_probe_frame(101, "aft"),
13767                )
13768                .await
13769                .unwrap()
13770        });
13771        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13772            .await
13773            .unwrap()
13774            .unwrap();
13775        assert_eq!(
13776            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13777            ModuleControlRequest::HealthCheck {}
13778        );
13779
13780        handler
13781            .handle_control_frame(
13782                &module_ctx,
13783                health_response(health_frame.header.corr, HealthStatus::Degraded),
13784            )
13785            .await
13786            .unwrap();
13787        let health_response = health_task.await.unwrap();
13788        assert_eq!(health_response.len(), 1);
13789        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13790            ClientControlResponse::SupervisorHealthProbe {
13791                module_id,
13792                status,
13793                detail,
13794                metrics,
13795            } => {
13796                assert_eq!(module_id, "aft");
13797                assert_eq!(status, HealthStatus::Degraded);
13798                assert_eq!(detail.as_deref(), Some("warming"));
13799                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
13800            }
13801            other => panic!("unexpected health response: {other:?}"),
13802        }
13803
13804        handler
13805            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13806            .await
13807            .unwrap();
13808        let route_response = route_task.await.unwrap();
13809        assert!(route_response.is_empty());
13810        let published = route_client_rx.recv().await.unwrap();
13811        assert!(matches!(
13812            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13813            ClientControlResponse::RouteOpen { .. }
13814        ));
13815    }
13816
13817    /// Start one `route.open` on `client_connection` and return its still-running
13818    /// handler task together with the `route.bind` the module received for it.
13819    /// The handler blocks until the module answers, so it has to run as a task
13820    /// while the test drives the module side.
13821    async fn relay_route_open(
13822        handler: &ControlHandler,
13823        client_connection: ConnectionId,
13824        client_egress: &FrameSink,
13825        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13826        corr: u64,
13827        module_id: &str,
13828        project_root_label: &str,
13829    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
13830        let ctx = RouteCtx {
13831            connection_id: client_connection,
13832            egress: client_egress.clone(),
13833        };
13834        let handler = handler.clone();
13835        let project_root = unique_project_root(project_root_label);
13836        let module_id = module_id.to_string();
13837        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
13838        let task = tokio::spawn(async move {
13839            let _guard = tracing::dispatcher::set_default(&dispatch);
13840            handler
13841                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
13842                .await
13843                .unwrap()
13844        });
13845        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13846            .await
13847            .expect("module receives the relayed route.bind")
13848            .expect("module egress is open");
13849        (task, bind.frame)
13850    }
13851
13852    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
13853        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
13854            ModuleControlRequest::RouteBind {
13855                route_channel,
13856                epoch,
13857                ..
13858            } => (route_channel, epoch),
13859            other => panic!("expected a route.bind request, got {other:?}"),
13860        }
13861    }
13862
13863    fn published_route(frame: &Frame) -> (u16, u32) {
13864        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
13865            ClientControlResponse::RouteOpen {
13866                route_channel,
13867                route_epoch,
13868                ..
13869            } => (route_channel, route_epoch),
13870            other => panic!("expected a route.open response, got {other:?}"),
13871        }
13872    }
13873
13874    /// Reproduction of a production outage. A client had `route.open`s in
13875    /// flight to a module and was already marked closing -- its egress had refused a
13876    /// module frame, so the daemon asked its connection to end -- while its sink
13877    /// was still open. When the module acked those binds, the daemon refused to
13878    /// commit a route for a closing client, and that refusal was returned from
13879    /// the MODULE connection's frame handler, where a router error that has no
13880    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
13881    /// the supervisor correctly did not respawn a clean exit, and every seat lost
13882    /// its tools for hours -- one client's teardown took down a connection
13883    /// carrying ~170 other routes.
13884    ///
13885    /// The window is opened here by calling the production path that opens it
13886    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
13887    /// state that matters is "in `closing_connections`, sink still open, relay
13888    /// still pending", and it lasts only from the close request until the
13889    /// connection loop reacts to it; a socket-level test can flood a client into
13890    /// that escalation but cannot pin the module's ack inside the window. Closing
13891    /// the socket instead takes the other path entirely -- connection teardown
13892    /// removes the pending relay under the same lock, so the ack finds nothing.
13893    #[tokio::test]
13894    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
13895        let registry = Arc::new(Registry::default());
13896        let forwarding = Arc::new(ForwardingTable::default());
13897        let handler =
13898            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
13899
13900        let module_connection = ConnectionId::new(30);
13901        let (module_ctx, mut module_rx) = route_ctx(module_connection);
13902        hello_via_sink(
13903            &handler,
13904            &module_ctx,
13905            &mut module_rx,
13906            hello_frame("aft", PROTOCOL_VERSION, 7),
13907        )
13908        .await;
13909
13910        let dying_client = ConnectionId::new(31);
13911        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
13912
13913        // A published route on the dying client. The escalation below only marks
13914        // a connection closing for a route it has already published.
13915        let (first_task, first_bind) = relay_route_open(
13916            &handler,
13917            dying_client,
13918            &dying_ctx.egress,
13919            &mut module_rx,
13920            100,
13921            "aft",
13922            "closing-first",
13923        )
13924        .await;
13925        handler
13926            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
13927            .await
13928            .unwrap();
13929        assert!(first_task.await.unwrap().is_empty());
13930        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
13931
13932        // A second route.open from the same client, relayed and awaiting its ack.
13933        let (second_task, second_bind) = relay_route_open(
13934            &handler,
13935            dying_client,
13936            &dying_ctx.egress,
13937            &mut module_rx,
13938            101,
13939            "aft",
13940            "closing-second",
13941        )
13942        .await;
13943        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
13944
13945        // The window: the client is closing, its sink is still open, and its
13946        // second bind is still pending.
13947        assert!(forwarding
13948            .escalate_client_delivery_failure(
13949                dying_client,
13950                first_channel,
13951                first_epoch,
13952                CloseReason::new(
13953                    "module_to_client_delivery_failed",
13954                    "client egress refused a module frame",
13955                ),
13956                crate::forwarding::UndeliveredFrame {
13957                    module_id: None,
13958                    sink: &dying_ctx.egress,
13959                },
13960            )
13961            .unwrap());
13962        assert!(!dying_ctx.egress.is_closed());
13963
13964        // The frame that used to end the module connection.
13965        let ack = handler
13966            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
13967            .await;
13968        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13969        if module_loop_error.is_some() {
13970            // What the server's connection loop does with a router error that has
13971            // no ERROR-frame translation: end the connection, which releases the
13972            // module's registration and every route on it.
13973            handler.cleanup_connection(module_connection).unwrap();
13974        }
13975        // Read the module's next frame before opening the co-tenant's route, so
13976        // the GOODBYE assertion below is about THIS ack and not about later
13977        // traffic. `None` means the module was told nothing.
13978        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13979            .await
13980            .ok()
13981            .flatten();
13982
13983        // 1. The module connection is still registered.
13984        assert!(
13985            registry
13986                .get_module_by_connection(module_connection)
13987                .unwrap()
13988                .is_some(),
13989            "one client's closing connection ended the shared module connection: \
13990             {module_loop_error:?}"
13991        );
13992        // ...and still serving: another client can open and use a route on it.
13993        let cotenant = ConnectionId::new(32);
13994        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
13995        let (cotenant_task, cotenant_bind) = relay_route_open(
13996            &handler,
13997            cotenant,
13998            &cotenant_ctx.egress,
13999            &mut module_rx,
14000            102,
14001            "aft",
14002            "closing-cotenant",
14003        )
14004        .await;
14005        handler
14006            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
14007            .await
14008            .unwrap();
14009        assert!(cotenant_task.await.unwrap().is_empty());
14010        let (cotenant_channel, cotenant_epoch) =
14011            published_route(&cotenant_rx.recv().await.unwrap());
14012        assert!(matches!(
14013            forwarding
14014                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14015                .unwrap(),
14016            DataRoute::Client(DataRouteState::Bound(_))
14017        ));
14018
14019        // 2. The module was told to drop the binding it created for the route
14020        //    that will never be published.
14021        let goodbye = post_ack_module_frame
14022            .expect("module receives a GOODBYE for the abandoned route channel");
14023        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14024        assert_eq!(goodbye.header.channel, abandoned_channel);
14025        assert_eq!(goodbye.header.epoch, abandoned_epoch);
14026
14027        // 3. The dying client received nothing: no route was ever published to
14028        //    it. Its route.open is answered as unavailable, which the connection
14029        //    loop would write to a socket that is already going away.
14030        assert!(dying_rx.try_recv().is_err());
14031        let second_response = second_task.await.unwrap();
14032        assert_eq!(second_response.len(), 1);
14033        assert_eq!(
14034            parse_error(&second_response[0])["code"],
14035            "target_unavailable"
14036        );
14037    }
14038
14039    /// One `catalog.list` as a client sees it: the generation and the listed
14040    /// module ids with their readiness.
14041    async fn catalog_snapshot(
14042        handler: &ControlHandler,
14043        client_ctx: &RouteCtx,
14044        corr: u64,
14045    ) -> (u64, Vec<(String, bool)>) {
14046        let body =
14047            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
14048        let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
14049        let reply = handler
14050            .handle_control_frame(client_ctx, request)
14051            .await
14052            .unwrap()
14053            .pop()
14054            .unwrap();
14055        let ClientControlResponse::CatalogList {
14056            generation,
14057            modules,
14058            ..
14059        } = serde_json::from_slice(&reply.body).unwrap()
14060        else {
14061            panic!("not a catalog.list reply");
14062        };
14063        (
14064            generation,
14065            modules
14066                .into_iter()
14067                .map(|entry| (entry.module_id, entry.ready))
14068                .collect(),
14069        )
14070    }
14071
14072    /// A handler whose HELLO for module "aft" on connection 40 is parked
14073    /// between reserving its registry entry and installing its forwarding
14074    /// endpoint. Returns the handler, the two-party barrier (wait once to know
14075    /// the HELLO is parked, once more to release it), the HELLO's thread, and
14076    /// the module's context and receiver.
14077    fn hello_held_before_forwarding() -> (
14078        ControlHandler,
14079        Arc<std::sync::Barrier>,
14080        std::thread::JoinHandle<Vec<Frame>>,
14081        RouteCtx,
14082        mpsc::Receiver<crate::router::OutboundFrame>,
14083    ) {
14084        let hold = Arc::new(std::sync::Barrier::new(2));
14085        let handler = ControlHandler::with_forwarding(
14086            Arc::new(Registry::default()),
14087            Arc::new(ForwardingTable::default()),
14088        )
14089        .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14090        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14091        let hello = {
14092            let handler = handler.clone();
14093            let module_ctx = module_ctx.clone();
14094            std::thread::spawn(move || {
14095                handler
14096                    .handle_hello(
14097                        module_ctx.connection_id,
14098                        Some(module_ctx.egress.clone()),
14099                        hello_frame("aft", PROTOCOL_VERSION, 7),
14100                    )
14101                    .unwrap()
14102            })
14103        };
14104        (handler, hold, hello, module_ctx, module_rx)
14105    }
14106
14107    /// If a lookup can acquire the registry while publication is parked, let
14108    /// the lookup finish before releasing HELLO. Otherwise release HELLO only
14109    /// after the lookup has tried the lock. Both schedules avoid timing guesses
14110    /// and return the actual lookup result, including an unpublished snapshot
14111    /// when the publication lock is missing.
14112    fn lookup_during_hello_publication<T: Send + 'static>(
14113        registry: &Registry,
14114        hold: &std::sync::Barrier,
14115        lookup: impl FnOnce() -> T + Send + 'static,
14116    ) -> T {
14117        let attempted = registry.observe_next_lock_attempt();
14118        let reader = std::thread::spawn(lookup);
14119        if attempted.recv().unwrap() {
14120            let result = reader.join().unwrap();
14121            hold.wait();
14122            result
14123        } else {
14124            hold.wait();
14125            reader.join().unwrap()
14126        }
14127    }
14128
14129    #[test]
14130    fn catalog_read_after_hello_ack_waits_for_publication() {
14131        let registry = Arc::new(Registry::default());
14132        let hold = Arc::new(std::sync::Barrier::new(2));
14133        let mut handler = ControlHandler::with_forwarding(
14134            Arc::clone(&registry),
14135            Arc::new(ForwardingTable::default()),
14136        );
14137        handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
14138        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14139        let hello = {
14140            let handler = handler.clone();
14141            std::thread::spawn(move || {
14142                handler
14143                    .handle_hello(
14144                        module_ctx.connection_id,
14145                        Some(module_ctx.egress),
14146                        hello_frame("aft", PROTOCOL_VERSION, 7),
14147                    )
14148                    .unwrap()
14149            })
14150        };
14151        hold.wait();
14152        assert_eq!(
14153            module_rx.try_recv().unwrap().frame.header.ty,
14154            FrameType::HelloAck
14155        );
14156        let replies = lookup_during_hello_publication(&registry, &hold, move || {
14157            // A different connection can issue catalog.list as soon as the
14158            // module has read its ACK, even while HELLO has not returned.
14159            let request = Frame::build(
14160                FrameType::Request,
14161                control_flags(),
14162                0,
14163                0,
14164                300,
14165                serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
14166            )
14167            .unwrap();
14168            handler.handle_catalog_list(request, None).unwrap()
14169        });
14170        assert!(hello.join().unwrap().is_empty());
14171        let ClientControlResponse::CatalogList {
14172            generation,
14173            modules,
14174            ..
14175        } = serde_json::from_slice(&replies[0].body).unwrap()
14176        else {
14177            panic!("not a catalog.list reply");
14178        };
14179        assert_eq!(
14180            modules
14181                .iter()
14182                .map(|entry| entry.module_id.as_str())
14183                .collect::<Vec<_>>(),
14184            vec!["aft"]
14185        );
14186        assert_eq!(generation, 1);
14187    }
14188
14189    /// While a HELLO has reserved its registry entry but its forwarding
14190    /// endpoint is not installed yet, `route.open` to the module can only be
14191    /// refused. `catalog.list` must not list the module in that window: a
14192    /// client that opens a route as soon as it sees a module listed and ready
14193    /// would be refused for a module the daemon had just advertised. The
14194    /// refusal in the window stays the retryable `target_unavailable`, never
14195    /// `unknown_module`, and the id stays claimed against a duplicate HELLO.
14196    /// Once the HELLO completes, the module is listed and a route to it opens.
14197    #[tokio::test]
14198    async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
14199        let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
14200        let client = ConnectionId::new(41);
14201        let (client_ctx, mut client_rx) = route_ctx(client);
14202
14203        hold.wait();
14204        // Parked: the registry holds "aft", forwarding has no endpoint for it.
14205        assert!(handler.registry.get_module("aft").unwrap().is_some());
14206        assert!(!handler
14207            .forwarding
14208            .has_live_module_connection("aft")
14209            .unwrap());
14210        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
14211        assert!(
14212            listed.is_empty(),
14213            "catalog.list listed a module route.open cannot reach yet: {listed:?}"
14214        );
14215        let refused = handler
14216            .handle_control_frame(
14217                &client_ctx,
14218                route_open_frame(301, "aft", unique_project_root("listed-window")),
14219            )
14220            .await
14221            .unwrap();
14222        assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
14223        let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
14224        let duplicate = handler
14225            .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
14226            .await
14227            .unwrap();
14228        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14229
14230        hold.wait();
14231        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14232        let ack = module_rx.try_recv().unwrap().frame;
14233        assert_eq!(ack.header.ty, FrameType::HelloAck);
14234        let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14235        assert_eq!(listed, vec![("aft".to_string(), true)]);
14236
14237        let (open_task, bind) = relay_route_open(
14238            &handler,
14239            client,
14240            &client_ctx.egress,
14241            &mut module_rx,
14242            303,
14243            "aft",
14244            "listed-after",
14245        )
14246        .await;
14247        assert!(handler
14248            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14249            .await
14250            .unwrap()
14251            .is_empty());
14252        assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
14253        published_route(&client_rx.try_recv().unwrap().frame);
14254    }
14255
14256    /// A catalog watcher keys on the generation: when it changes, it re-reads
14257    /// the listing. So the generation must advance exactly when the module
14258    /// appears. If it advanced while the module was still hidden, a watcher
14259    /// would read the new generation without the module, and the module would
14260    /// later appear under that same generation with nothing telling the
14261    /// watcher to look again.
14262    #[tokio::test]
14263    async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
14264        let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
14265        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14266        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14267
14268        hold.wait();
14269        let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
14270        assert!(listed.is_empty());
14271        assert_eq!(
14272            during, before,
14273            "the generation moved while the module was not listed"
14274        );
14275
14276        hold.wait();
14277        assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
14278        let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
14279        assert_eq!(listed, vec![("aft".to_string(), true)]);
14280        assert_eq!(
14281            after,
14282            before + 1,
14283            "the module appeared without exactly one generation bump"
14284        );
14285    }
14286
14287    /// A HELLO whose forwarding endpoint cannot be installed (here the
14288    /// module's sink is already closed, so its HELLO_ACK cannot be queued) is
14289    /// refused and leaves nothing behind: no registry entry, no catalog entry,
14290    /// and the catalog generation where it was.
14291    #[tokio::test]
14292    async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
14293        let registry = Arc::new(Registry::default());
14294        let handler = ControlHandler::with_forwarding(
14295            Arc::clone(&registry),
14296            Arc::new(ForwardingTable::default()),
14297        );
14298        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14299        let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
14300        let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
14301        drop(module_rx);
14302
14303        let replies = handler
14304            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
14305            .await
14306            .unwrap();
14307        assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
14308        assert!(registry.get_module("aft").unwrap().is_none());
14309        assert!(registry
14310            .get_module_by_connection(module_ctx.connection_id)
14311            .unwrap()
14312            .is_none());
14313        assert_eq!(
14314            catalog_snapshot(&handler, &client_ctx, 301).await,
14315            (before, Vec::new())
14316        );
14317    }
14318
14319    /// A handler whose `route.open` parks after queueing the route.bind and
14320    /// before marking it enqueued, with module "aft" registered on connection
14321    /// 40. Returns the two-party barrier that parks and releases it.
14322    async fn route_open_held_after_bind_send() -> (
14323        ControlHandler,
14324        Arc<tokio::sync::Barrier>,
14325        RouteCtx,
14326        mpsc::Receiver<crate::router::OutboundFrame>,
14327    ) {
14328        let hold = Arc::new(tokio::sync::Barrier::new(2));
14329        let handler = ControlHandler::with_forwarding(
14330            Arc::new(Registry::default()),
14331            Arc::new(ForwardingTable::default()),
14332        )
14333        .with_route_bind_relay_sent_hold(Arc::clone(&hold));
14334        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14335        hello_via_sink(
14336            &handler,
14337            &module_ctx,
14338            &mut module_rx,
14339            hello_frame("aft", PROTOCOL_VERSION, 7),
14340        )
14341        .await;
14342        (handler, hold, module_ctx, module_rx)
14343    }
14344
14345    /// A fast module acks the route.bind, and the reader commits the route,
14346    /// before `route.open` has marked the relay enqueued. The daemon used to
14347    /// read the missing relay as an abandoned bind and send the module a
14348    /// GOODBYE for the route it had just committed: the client was told its
14349    /// route was open, the module unbound the channel, and the client's first
14350    /// request was dropped on the module side as unbound.
14351    #[tokio::test]
14352    async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
14353        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14354        let client = ConnectionId::new(41);
14355        let (client_ctx, mut client_rx) = route_ctx(client);
14356
14357        let (open_task, bind) = relay_route_open(
14358            &handler,
14359            client,
14360            &client_ctx.egress,
14361            &mut module_rx,
14362            200,
14363            "aft",
14364            "acked-before-mark",
14365        )
14366        .await;
14367        let (module_channel, module_epoch) = route_bind_channel(&bind);
14368        // route.open is now parked between queueing the bind and marking it.
14369        hold.wait().await;
14370
14371        // The module answers inside that window, and the route commits.
14372        assert!(handler
14373            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14374            .await
14375            .unwrap()
14376            .is_empty());
14377        let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
14378
14379        hold.wait().await;
14380        assert!(
14381            open_task.await.unwrap().is_empty(),
14382            "route.open answers accepted"
14383        );
14384
14385        // Every frame route.open could have sent is already queued, because it
14386        // has returned; the module must have been sent nothing.
14387        match module_rx.try_recv() {
14388            Err(mpsc::error::TryRecvError::Empty) => {}
14389            Ok(outbound) => panic!(
14390                "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
14391                outbound.frame.header.ty,
14392                outbound.frame.header.channel,
14393                outbound.frame.header.epoch
14394            ),
14395            Err(err) => panic!("module egress closed: {err}"),
14396        }
14397
14398        // The client's first request on the route is the next thing the module
14399        // reads, on the binding it just created.
14400        let router = Router::with_control_handler(Arc::new(handler.clone()));
14401        let request = Frame::build(
14402            FrameType::Request,
14403            Flags::new(true, Priority::Interactive, false),
14404            client_channel,
14405            client_epoch,
14406            201,
14407            b"first request".to_vec(),
14408        )
14409        .unwrap();
14410        router
14411            .route_for_connection(&client_ctx, request)
14412            .await
14413            .unwrap();
14414        let delivered = module_rx.try_recv().unwrap().frame;
14415        assert_eq!(delivered.header.ty, FrameType::Request);
14416        assert_eq!(
14417            (delivered.header.channel, delivered.header.epoch),
14418            (module_channel, module_epoch)
14419        );
14420        assert_eq!(delivered.body, b"first request");
14421    }
14422
14423    /// The client goes away while `route.open` is still queueing the bind, so
14424    /// the relay is dropped with no module reply. The teardown cannot send the
14425    /// GOODBYE itself (it could reach the module ahead of the bind), so
14426    /// `route.open` must send it once the bind is queued; otherwise the module
14427    /// keeps a binding the daemon will never commit.
14428    #[tokio::test]
14429    async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
14430        let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14431        let client = ConnectionId::new(42);
14432        let (client_ctx, _client_rx) = route_ctx(client);
14433
14434        let (open_task, bind) = relay_route_open(
14435            &handler,
14436            client,
14437            &client_ctx.egress,
14438            &mut module_rx,
14439            300,
14440            "aft",
14441            "released-before-mark",
14442        )
14443        .await;
14444        let (module_channel, module_epoch) = route_bind_channel(&bind);
14445        hold.wait().await;
14446
14447        handler.cleanup_connection(client).unwrap();
14448        assert!(
14449            module_rx.try_recv().is_err(),
14450            "client teardown must not GOODBYE a bind route.open is still queueing"
14451        );
14452
14453        hold.wait().await;
14454        let response = open_task.await.unwrap();
14455        assert_eq!(response.len(), 1);
14456        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14457
14458        let goodbye = module_rx
14459            .try_recv()
14460            .expect("module is told to drop the binding the daemon abandoned")
14461            .frame;
14462        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14463        assert_eq!(
14464            (goodbye.header.channel, goodbye.header.epoch),
14465            (module_channel, module_epoch)
14466        );
14467        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14468    }
14469
14470    /// The module acks inside the same window, but the client's egress closed
14471    /// first, so the reader refuses to commit. The ack proves the module holds
14472    /// the binding, so the reader sends the GOODBYE, and `route.open` must not
14473    /// send a second one.
14474    #[tokio::test]
14475    async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
14476        let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
14477        let client = ConnectionId::new(43);
14478        let (client_ctx, client_rx) = route_ctx(client);
14479
14480        let (open_task, bind) = relay_route_open(
14481            &handler,
14482            client,
14483            &client_ctx.egress,
14484            &mut module_rx,
14485            400,
14486            "aft",
14487            "gone-client-before-mark",
14488        )
14489        .await;
14490        let (module_channel, module_epoch) = route_bind_channel(&bind);
14491        hold.wait().await;
14492
14493        drop(client_rx);
14494        assert!(client_ctx.egress.is_closed());
14495        handler
14496            .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
14497            .await
14498            .unwrap();
14499
14500        hold.wait().await;
14501        let response = open_task.await.unwrap();
14502        assert_eq!(response.len(), 1);
14503        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14504
14505        let goodbye = module_rx
14506            .try_recv()
14507            .expect("module is told to drop the binding it acked")
14508            .frame;
14509        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14510        assert_eq!(
14511            (goodbye.header.channel, goodbye.header.epoch),
14512            (module_channel, module_epoch)
14513        );
14514        assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
14515        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
14516    }
14517
14518    /// The fence at the module-loop boundary, stated as its own contract: which
14519    /// forwarding failures are allowed to end the module connection that is being
14520    /// served. A `ConnectionClosing` naming some client is about that client, and
14521    /// a module connection is shared; the same error naming the module's own
14522    /// connection is about this connection and must stay fatal, as must failures
14523    /// that are about the forwarding table itself.
14524    #[test]
14525    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
14526        let handler = ControlHandler::default();
14527        let module_connection = ConnectionId::new(30);
14528        let client_connection = ConnectionId::new(31);
14529
14530        handler
14531            .refuse_to_end_module_connection_for_a_client(
14532                module_connection,
14533                77,
14534                ForwardingError::ConnectionClosing {
14535                    connection_id: client_connection,
14536                },
14537            )
14538            .expect("a closing client must never end the module connection");
14539
14540        assert!(matches!(
14541            handler.refuse_to_end_module_connection_for_a_client(
14542                module_connection,
14543                78,
14544                ForwardingError::ConnectionClosing {
14545                    connection_id: module_connection,
14546                },
14547            ),
14548            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
14549                connection_id
14550            })) if connection_id == module_connection
14551        ));
14552        assert!(matches!(
14553            handler.refuse_to_end_module_connection_for_a_client(
14554                module_connection,
14555                79,
14556                ForwardingError::Poisoned,
14557            ),
14558            Err(RouterError::Forwarding(ForwardingError::Poisoned))
14559        ));
14560        assert!(matches!(
14561            handler.refuse_to_end_module_connection_for_a_client(
14562                module_connection,
14563                80,
14564                ForwardingError::StaleModuleEndpoint,
14565            ),
14566            Err(RouterError::Forwarding(
14567                ForwardingError::StaleModuleEndpoint
14568            ))
14569        ));
14570    }
14571
14572    /// The spawn-attestation guard is what stops a connected module from claiming
14573    /// another module's identity and being stamped `Reserved` for it. Every other
14574    /// test that supplies a consumer_identity supplies a CORRECT one, because a
14575    /// correct one is what the rest of the flow needs -- so the guard's rejection
14576    /// branch was never the subject of an assertion, only its acceptance branch.
14577    ///
14578    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
14579    /// whole subc-core library suite green; only the forwarding integration tests
14580    /// notice, and they notice for unrelated reasons. This test exists so the
14581    /// refusal itself is asserted where the guard lives: it fails if the identity
14582    /// check stops refusing, which is the direction that matters, since a guard
14583    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
14584    #[tokio::test]
14585    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
14586        let registry = Arc::new(Registry::default());
14587        let forwarding = Arc::new(ForwardingTable::default());
14588        let supervisor = SupervisorHandle::new();
14589        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14590        let handler =
14591            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14592                .with_supervisor(supervisor);
14593
14594        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
14595        hello_via_sink(
14596            &handler,
14597            &target_ctx,
14598            &mut target_rx,
14599            hello_frame("target", PROTOCOL_VERSION, 1),
14600        )
14601        .await;
14602
14603        // A real supervised module id presenting the wrong nonce. This is the
14604        // impersonation case: the attacker knows a privileged module_id, which is
14605        // public, and guesses at the nonce, which is not.
14606        let wrong_nonce = handler
14607            .handle_control_frame(
14608                &route_ctx(ConnectionId::new(91)).0,
14609                route_open_frame_with_admission_facts(
14610                    20,
14611                    "target",
14612                    unique_project_root("admission-facts"),
14613                    Some(subc_control::ConsumerIdentity {
14614                        module_id: "fed".to_string(),
14615                        launch_nonce: "not-the-real-nonce".to_string(),
14616                    }),
14617                    None,
14618                ),
14619            )
14620            .await
14621            .unwrap();
14622        assert_eq!(
14623            parse_error(&wrong_nonce[0])["code"],
14624            "bad_consumer_identity",
14625            "a mismatched launch nonce must be refused, not stamped Reserved"
14626        );
14627
14628        // A module id the supervisor never spawned at all, so no nonce exists to
14629        // compare against. An implementation that treats "no record" as "nothing
14630        // to check" fails open here while passing the case above.
14631        let never_spawned = handler
14632            .handle_control_frame(
14633                &route_ctx(ConnectionId::new(92)).0,
14634                route_open_frame_with_admission_facts(
14635                    21,
14636                    "target",
14637                    unique_project_root("admission-facts"),
14638                    Some(subc_control::ConsumerIdentity {
14639                        module_id: "never-spawned".to_string(),
14640                        launch_nonce: "any-nonce".to_string(),
14641                    }),
14642                    None,
14643                ),
14644            )
14645            .await
14646            .unwrap();
14647        assert_eq!(
14648            parse_error(&never_spawned[0])["code"],
14649            "bad_consumer_identity",
14650            "an unspawned module_id must be refused rather than accepted for lack of a record"
14651        );
14652    }
14653
14654    /// The refusal test above proves the guard says NO. Nothing proved it can say
14655    /// YES, and the difference is not academic: replacing the whole authorization
14656    /// with `false` -- admitting no consumer identity at all, revoking Reserved
14657    /// standing for every supervised module in the fleet -- leaves 110 of the 111
14658    /// library tests GREEN. The one that notices does so by HANGING, because it
14659    /// waits for a bind that can no longer happen.
14660    ///
14661    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
14662    /// or flaky test, invites a RETRY rather than an investigation, and the retry
14663    /// hangs too and gets blamed on the runner. So a total revocation of the
14664    /// daemon's trust grant would have shipped behind a symptom nobody attributes
14665    /// to code.
14666    ///
14667    /// The bias is structural rather than accidental. A REFUSAL looks like a
14668    /// failure someone writes a test for; a GRANT looks like the happy path. Every
14669    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
14670    /// suite for that reason, and this one is the purest case in the daemon.
14671    ///
14672    /// This test asserts the EFFECT rather than the absence of an error: the module
14673    /// receives a RouteBind and it carries `Reserved` naming the attested module.
14674    /// A guard that admitted nobody would produce no bind at all; one that admitted
14675    /// everybody would stamp the wrong principal, which the refusal test catches.
14676    #[tokio::test]
14677    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
14678        let registry = Arc::new(Registry::default());
14679        let forwarding = Arc::new(ForwardingTable::default());
14680        let supervisor = SupervisorHandle::new();
14681        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14682        let handler =
14683            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14684                .with_supervisor(supervisor);
14685
14686        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
14687        hello_via_sink(
14688            &handler,
14689            &target_ctx,
14690            &mut target_rx,
14691            hello_frame("target", PROTOCOL_VERSION, 1),
14692        )
14693        .await;
14694
14695        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
14696        let route_handler = handler.clone();
14697        let route_task = tokio::spawn(async move {
14698            route_handler
14699                .handle_control_frame(
14700                    &client_ctx,
14701                    route_open_frame_with_admission_facts(
14702                        30,
14703                        "target",
14704                        unique_project_root("admission-facts"),
14705                        Some(subc_control::ConsumerIdentity {
14706                            module_id: "fed".to_string(),
14707                            launch_nonce: "fed-nonce".to_string(),
14708                        }),
14709                        None,
14710                    ),
14711                )
14712                .await
14713                .unwrap()
14714        });
14715
14716        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
14717        // the very mutation it exists to catch -- a guard that admits nobody -- no
14718        // bind is ever sent, so it HUNG rather than failing. That reproduces the
14719        // exact defect being fixed: a total revocation detected only as a stalled
14720        // suite, which reads as flakiness and invites a retry. An acceptance test
14721        // that waits for an effect must bound the wait, or a red becomes a hang.
14722        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14723            .await
14724            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
14725            .expect("module control channel closed before route.bind");
14726        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
14727        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14728            panic!("expected route.bind")
14729        };
14730        assert_eq!(
14731            principal,
14732            Some(Principal::Reserved {
14733                module_id: "fed".to_string()
14734            }),
14735            "a correctly attested consumer must be stamped Reserved for its own id"
14736        );
14737
14738        handler
14739            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
14740            .await
14741            .unwrap();
14742        assert!(route_task.await.unwrap().is_empty());
14743        assert!(
14744            matches!(
14745                serde_json::from_slice::<ClientControlResponse>(
14746                    &client_rx.recv().await.unwrap().body
14747                )
14748                .unwrap(),
14749                ClientControlResponse::RouteOpen { .. }
14750            ),
14751            "the route must actually open, not merely avoid an error"
14752        );
14753    }
14754
14755    #[tokio::test(start_paused = true)]
14756    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
14757        let registry = Arc::new(Registry::default());
14758        let forwarding = Arc::new(ForwardingTable::default());
14759        let supervisor = SupervisorHandle::new();
14760        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
14761        let handler =
14762            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
14763                .with_supervisor(supervisor);
14764
14765        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
14766        hello_via_sink(
14767            &handler,
14768            &target_ctx,
14769            &mut target_rx,
14770            hello_frame("target", PROTOCOL_VERSION, 1),
14771        )
14772        .await;
14773
14774        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
14775        let direct_handler = handler.clone();
14776        let direct_open = tokio::spawn(async move {
14777            direct_handler
14778                .handle_control_frame(
14779                    &direct_ctx,
14780                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
14781                )
14782                .await
14783                .unwrap()
14784        });
14785        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14786            .await
14787            .expect("no direct route.bind within 5s")
14788            .expect("target control channel closed before direct route.bind");
14789        handler
14790            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
14791            .await
14792            .unwrap();
14793        assert!(direct_open.await.unwrap().is_empty());
14794        let _ = direct_rx.recv().await.unwrap();
14795
14796        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
14797        let reserved_handler = handler.clone();
14798        let reserved_open = tokio::spawn(async move {
14799            reserved_handler
14800                .handle_control_frame(
14801                    &reserved_ctx,
14802                    route_open_frame_with_admission_facts(
14803                        3,
14804                        "target",
14805                        unique_project_root("admission-facts"),
14806                        Some(ConsumerIdentity {
14807                            module_id: "fed".to_string(),
14808                            launch_nonce: "fed-nonce".to_string(),
14809                        }),
14810                        None,
14811                    ),
14812                )
14813                .await
14814                .unwrap()
14815        });
14816        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14817            .await
14818            .expect("no reserved route.bind within 5s")
14819            .expect("target control channel closed before reserved route.bind");
14820        handler
14821            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
14822            .await
14823            .unwrap();
14824        assert!(reserved_open.await.unwrap().is_empty());
14825        let _ = reserved_rx.recv().await.unwrap();
14826
14827        forwarding
14828            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
14829            .unwrap();
14830        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
14831        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
14832            module_id: Some("target".to_string()),
14833        })
14834        .unwrap();
14835        let census_frame =
14836            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
14837        let response = handler
14838            .handle_control_frame(&census_ctx, census_frame)
14839            .await
14840            .unwrap()
14841            .pop()
14842            .unwrap();
14843        let actual: Value = serde_json::from_slice(&response.body).unwrap();
14844        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
14845        assert!(matches!(
14846            decoded,
14847            ClientControlResponse::SupervisorRoutes { .. }
14848        ));
14849        let routes = actual["modules"][0]["routes"].as_array().unwrap();
14850        assert_eq!(routes.len(), 2);
14851        assert!(routes.iter().all(|route| route["draining"] == true));
14852        // The census carries WHY: the reason the drain was begun with, in the
14853        // route.closing vocabulary, on every draining route this drain marked.
14854        assert!(
14855            routes.iter().all(|route| route["drain_reason"] == "reload"),
14856            "draining routes must name the drain's reason: {routes:?}"
14857        );
14858        assert!(routes.iter().any(|route| {
14859            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
14860        }));
14861        assert!(routes.iter().any(|route| {
14862            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
14863        }));
14864
14865        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
14866            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
14867        if std::env::var_os("UPDATE_GOLDEN").is_some() {
14868            std::fs::write(
14869                &golden_path,
14870                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
14871            )
14872            .unwrap();
14873        }
14874        let expected: Value =
14875            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
14876        assert_eq!(actual, expected);
14877    }
14878
14879    async fn query_live_roots(
14880        handler: &ControlHandler,
14881        module_ctx: &RouteCtx,
14882    ) -> ModuleControlResponseToModule {
14883        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
14884        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
14885        let response = handler
14886            .handle_control_frame(module_ctx, frame)
14887            .await
14888            .unwrap()
14889            .pop()
14890            .unwrap();
14891        serde_json::from_slice(&response.body).unwrap()
14892    }
14893
14894    #[tokio::test(start_paused = true)]
14895    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
14896        let registry = Arc::new(Registry::default());
14897        let forwarding = Arc::new(ForwardingTable::default());
14898        let handler = ControlHandler::with_forwarding(registry, forwarding);
14899        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
14900        hello_via_sink(
14901            &handler,
14902            &target_ctx,
14903            &mut target_rx,
14904            hello_frame("target", PROTOCOL_VERSION, 1),
14905        )
14906        .await;
14907        let root = unique_project_root("live-roots-known");
14908        let path = ProjectRootId::from_path_allowing_missing(root.path())
14909            .unwrap()
14910            .as_path()
14911            .to_path_buf();
14912        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
14913        let open_handler = handler.clone();
14914        let opened = tokio::spawn(async move {
14915            open_handler
14916                .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
14917                .await
14918                .unwrap()
14919        });
14920        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14921            .await
14922            .unwrap()
14923            .unwrap();
14924        handler
14925            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
14926            .await
14927            .unwrap();
14928        assert!(opened.await.unwrap().is_empty());
14929        let _ = client_rx.recv().await.unwrap();
14930
14931        let root = unique_project_root("live-roots-pending");
14932        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
14933            .unwrap()
14934            .as_path()
14935            .to_path_buf();
14936        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
14937        let open_handler = handler.clone();
14938        let pending = tokio::spawn(async move {
14939            open_handler
14940                .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
14941                .await
14942                .unwrap()
14943        });
14944        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
14945            .await
14946            .unwrap()
14947            .unwrap();
14948        let actual = query_live_roots(&handler, &target_ctx).await;
14949        let ModuleControlResponseToModule::LiveRoots {
14950            roots,
14951            unknown_root_bindings,
14952            total_bindings,
14953        } = actual
14954        else {
14955            panic!("expected live roots")
14956        };
14957        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
14958        assert_eq!(unknown_root_bindings, 0);
14959        assert_eq!(
14960            roots.len(),
14961            2,
14962            "root-known arm must retain each canonical root"
14963        );
14964        assert_eq!(
14965            total_bindings,
14966            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
14967        );
14968        let counts = roots
14969            .iter()
14970            .map(|root| (root.project_root.clone(), root.bound, root.pending))
14971            .collect::<Vec<_>>();
14972        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
14973        expected.sort_by(|a, b| a.0.cmp(&b.0));
14974        assert_eq!(
14975            counts, expected,
14976            "roots must sort by path and count pending separately"
14977        );
14978        handler
14979            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
14980            .await
14981            .unwrap();
14982        assert!(pending.await.unwrap().is_empty());
14983    }
14984
14985    #[tokio::test(start_paused = true)]
14986    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
14987        let forwarding = Arc::new(ForwardingTable::default());
14988        let handler =
14989            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
14990        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
14991        hello_via_sink(
14992            &handler,
14993            &target_ctx,
14994            &mut target_rx,
14995            hello_frame("target", PROTOCOL_VERSION, 1),
14996        )
14997        .await;
14998        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
14999        let pending = forwarding
15000            .begin_route_bind_relay_for_test(
15001                client_ctx.connection_id,
15002                client_ctx.egress.clone(),
15003                2,
15004                "target",
15005            )
15006            .unwrap();
15007        forwarding
15008            .complete_pending_relay(
15009                target_ctx.connection_id,
15010                pending.corr,
15011                RouteBindRelayOutcome::Accepted,
15012            )
15013            .unwrap();
15014        let actual = query_live_roots(&handler, &target_ctx).await;
15015        let ModuleControlResponseToModule::LiveRoots {
15016            roots,
15017            unknown_root_bindings,
15018            total_bindings,
15019        } = actual
15020        else {
15021            panic!("expected live roots")
15022        };
15023        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
15024        assert_eq!(
15025            unknown_root_bindings, 1,
15026            "unknown-root arm must not read as no bindings"
15027        );
15028        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
15029        assert_eq!(
15030            total_bindings,
15031            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15032        );
15033    }
15034
15035    /// A module reads HELLO_ACK as its first frame and exits on anything else,
15036    /// so the ack has to be on its outbound queue before the module is
15037    /// routable. The connection loop writes a handler's replies only after the
15038    /// handler returns; this test stops in exactly that gap, runs a real
15039    /// route.open from another connection, and only then writes whatever the
15040    /// HELLO handler returned, the way the loop would. If the ack were still a
15041    /// reply, the route.bind request would reach the module first.
15042    #[tokio::test(start_paused = true)]
15043    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
15044        let forwarding = Arc::new(ForwardingTable::default());
15045        let handler =
15046            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
15047        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
15048        let replies = handler
15049            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
15050            .await
15051            .unwrap();
15052        let queued_by_hello = module_rx.len();
15053
15054        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
15055        let open_handler = handler.clone();
15056        let open = tokio::spawn(async move {
15057            open_handler
15058                .handle_control_frame(
15059                    &client_ctx,
15060                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
15061                )
15062                .await
15063                .unwrap()
15064        });
15065        // Let the route.open run until its route.bind is on the module's queue.
15066        let mut spins = 0;
15067        while module_rx.len() == queued_by_hello {
15068            spins += 1;
15069            assert!(spins < 10_000, "route.open never queued a route.bind");
15070            tokio::task::yield_now().await;
15071        }
15072
15073        // Now the connection loop's half: write the HELLO handler's replies.
15074        for reply in replies {
15075            module_ctx.egress.send(reply).await.unwrap();
15076        }
15077
15078        let first = module_rx.recv().await.unwrap().frame;
15079        assert_eq!(
15080            first.header.ty,
15081            FrameType::HelloAck,
15082            "the first frame a registering module reads must be its HELLO_ACK"
15083        );
15084        assert_eq!(first.header.corr, 7);
15085        let second = module_rx.recv().await.unwrap().frame;
15086        assert_eq!(second.header.ty, FrameType::Request);
15087        assert!(
15088            matches!(
15089                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
15090                ModuleControlRequest::RouteBind { .. }
15091            ),
15092            "the route.bind follows the ack"
15093        );
15094        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
15095
15096        handler
15097            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
15098            .await
15099            .unwrap();
15100        assert!(open.await.unwrap().is_empty());
15101        let _ = client_rx.recv().await.unwrap();
15102    }
15103
15104    #[tokio::test(start_paused = true)]
15105    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
15106        let handler = ControlHandler::with_forwarding(
15107            Arc::new(Registry::default()),
15108            Arc::new(ForwardingTable::default()),
15109        );
15110        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
15111        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
15112        hello_via_sink(
15113            &handler,
15114            &first_ctx,
15115            &mut first_rx,
15116            hello_frame("first", PROTOCOL_VERSION, 1),
15117        )
15118        .await;
15119        hello_via_sink(
15120            &handler,
15121            &second_ctx,
15122            &mut second_rx,
15123            hello_frame("second", PROTOCOL_VERSION, 2),
15124        )
15125        .await;
15126        let root = unique_project_root("second-only");
15127        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
15128        let cloned = handler.clone();
15129        let open = tokio::spawn(async move {
15130            cloned
15131                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
15132                .await
15133                .unwrap()
15134        });
15135        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
15136            .await
15137            .unwrap()
15138            .unwrap();
15139        let first = query_live_roots(&handler, &first_ctx).await;
15140        let second = query_live_roots(&handler, &second_ctx).await;
15141        assert!(
15142            matches!(
15143                first,
15144                ModuleControlResponseToModule::LiveRoots {
15145                    total_bindings: 0,
15146                    ..
15147                }
15148            ),
15149            "cross-module scope must not expose another module's roots"
15150        );
15151        assert!(
15152            matches!(
15153                second,
15154                ModuleControlResponseToModule::LiveRoots {
15155                    total_bindings: 1,
15156                    ..
15157                }
15158            ),
15159            "second module must see its pending route"
15160        );
15161        handler
15162            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
15163            .await
15164            .unwrap();
15165        assert!(open.await.unwrap().is_empty());
15166    }
15167
15168    #[tokio::test(start_paused = true)]
15169    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
15170        let handler = ControlHandler::with_forwarding(
15171            Arc::new(Registry::default()),
15172            Arc::new(ForwardingTable::default()),
15173        );
15174        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
15175        hello_via_sink(
15176            &handler,
15177            &target_ctx,
15178            &mut target_rx,
15179            hello_frame("target", PROTOCOL_VERSION, 1),
15180        )
15181        .await;
15182        let actual = query_live_roots(&handler, &target_ctx).await;
15183        let ModuleControlResponseToModule::LiveRoots {
15184            roots,
15185            unknown_root_bindings,
15186            total_bindings,
15187        } = actual
15188        else {
15189            panic!("expected live roots")
15190        };
15191        assert!(roots.is_empty());
15192        assert_eq!(unknown_root_bindings, 0);
15193        assert_eq!(total_bindings, 0);
15194        assert_eq!(
15195            total_bindings,
15196            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
15197        );
15198    }
15199
15200    /// Read the vendored fed corpus rather than hand-building a package.
15201    ///
15202    /// A hand-built object encodes what the test author believed the carrier
15203    /// emits. These vectors are what it actually emits, and one of them exists
15204    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
15205    /// ignores additive unknown fields at the traversal emit terminus."
15206    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
15207        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
15208            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
15209        let text = std::fs::read_to_string(&path)
15210            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
15211        let vectors: Vec<(String, Value)> = text
15212            .lines()
15213            .filter(|line| !line.trim().is_empty())
15214            .map(|line| {
15215                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
15216                let id = entry["corpus_id"]
15217                    .as_str()
15218                    .expect("every vector carries a corpus_id")
15219                    .to_string();
15220                (id, entry["package"].clone())
15221            })
15222            .collect();
15223        // Pin the count: a corpus that silently shrinks would take its coverage
15224        // with it, and a suite reading N-1 vectors reports the same clean pass
15225        // as one reading N.
15226        assert_eq!(
15227            vectors.len(),
15228            3,
15229            "vendored fed corpus changed size; re-sync from subc-federation"
15230        );
15231
15232        // Pin what makes the corpus DISCRIMINATING, not just present.
15233        //
15234        // The relay test below takes its expected value from the corpus, so the
15235        // corpus supplies the test's power to detect a lossy relay rather than
15236        // its correctness. A relay that dropped unrecognised fields would still
15237        // be caught -- but only by a package carrying fields it does not know.
15238        // Shrink every package to the handful of keys any implementation would
15239        // recognise and the test keeps passing over an input that can no longer
15240        // fail, which is the same clean green as a corpus that shrank away.
15241        //
15242        // So assert the precondition rather than duplicating the packages here:
15243        // at least one vector must carry a field beyond the small common set.
15244        // That is one claim to maintain instead of nine, and it fails loudly if
15245        // a re-sync ever flattens the corpus.
15246        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
15247        let richest = vectors
15248            .iter()
15249            .filter_map(|(_, package)| package.as_object())
15250            .map(|object| {
15251                object
15252                    .keys()
15253                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
15254                    .count()
15255            })
15256            .max()
15257            .unwrap_or(0);
15258        assert!(
15259            richest >= 2,
15260            "vendored corpus no longer carries a package with unmodelled fields, \
15261             so the relay test can no longer distinguish a verbatim relay from a lossy one"
15262        );
15263
15264        vectors
15265    }
15266
15267    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
15268    ///
15269    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
15270    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
15271    /// three-key object, so a relay that quietly dropped fields it did not
15272    /// recognise would satisfy it. These vectors carry nine keys including ones
15273    /// this crate has no type for, so a typed relay fails here and only here.
15274    #[tokio::test]
15275    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
15276        for (corpus_id, package) in fed_admission_facts_vectors() {
15277            let registry = Arc::new(Registry::default());
15278            let forwarding = Arc::new(ForwardingTable::default());
15279            let supervisor = SupervisorHandle::new();
15280            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15281            let handler =
15282                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15283                    .with_supervisor(supervisor)
15284                    .with_admission_facts_config(
15285                        Some("fed".to_string()),
15286                        Some(vec!["target".to_string()]),
15287                    );
15288
15289            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
15290            hello_via_sink(
15291                &handler,
15292                &target_ctx,
15293                &mut target_rx,
15294                hello_frame("target", PROTOCOL_VERSION, 1),
15295            )
15296            .await;
15297
15298            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
15299            let route_handler = handler.clone();
15300            let expected = package.clone();
15301            let route_task = tokio::spawn(async move {
15302                route_handler
15303                    .handle_control_frame(
15304                        &client_ctx,
15305                        route_open_frame_with_admission_facts(
15306                            20,
15307                            "target",
15308                            unique_project_root("admission-facts"),
15309                            Some(subc_control::ConsumerIdentity {
15310                                module_id: "fed".to_string(),
15311                                launch_nonce: "fed-nonce".to_string(),
15312                            }),
15313                            Some(package),
15314                        ),
15315                    )
15316                    .await
15317                    .unwrap()
15318            });
15319
15320            let bind_frame = target_rx.recv().await.unwrap();
15321            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15322            let ModuleControlRequest::RouteBind {
15323                admission_facts, ..
15324            } = bind
15325            else {
15326                panic!("{corpus_id}: expected route.bind")
15327            };
15328            assert_eq!(
15329                admission_facts,
15330                Some(expected),
15331                "{corpus_id}: relay must not add, drop or reshape any field"
15332            );
15333
15334            handler
15335                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15336                .await
15337                .unwrap();
15338            route_task.await.unwrap();
15339        }
15340    }
15341
15342    #[tokio::test]
15343    async fn route_bind_never_relays_the_openers_build_class() {
15344        let registry = Arc::new(Registry::default());
15345        let forwarding = Arc::new(ForwardingTable::default());
15346        let handler =
15347            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15348        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
15349        hello_via_sink(
15350            &handler,
15351            &target_ctx,
15352            &mut target_rx,
15353            hello_frame("target", PROTOCOL_VERSION, 1),
15354        )
15355        .await;
15356
15357        let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
15358        identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
15359        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15360            busy_refusals: None,
15361            target: RouteTarget::ToolProvider {
15362                module_id: "target".to_string(),
15363            },
15364            identity,
15365            consumer_identity: None,
15366            consumer_capabilities: None,
15367            role_versions: None,
15368            admission_facts: None,
15369            scope: None,
15370        })
15371        .unwrap();
15372        let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
15373
15374        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
15375        let route_handler = handler.clone();
15376        let route_task = tokio::spawn(async move {
15377            route_handler
15378                .handle_control_frame(&client_ctx, open)
15379                .await
15380                .unwrap()
15381        });
15382        let bind_frame = target_rx.recv().await.unwrap();
15383        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15384        let ModuleControlRequest::RouteBind { identity, .. } = bind else {
15385            panic!("expected route.bind")
15386        };
15387        assert_eq!(
15388            identity.build_class, None,
15389            "route.bind carried the build class"
15390        );
15391        assert!(
15392            !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
15393            "the route.bind bytes still name build_class"
15394        );
15395        handler
15396            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15397            .await
15398            .unwrap();
15399        route_task.await.unwrap();
15400    }
15401
15402    #[tokio::test]
15403    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
15404        let registry = Arc::new(Registry::default());
15405        let forwarding = Arc::new(ForwardingTable::default());
15406        let supervisor = SupervisorHandle::new();
15407        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
15408        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
15409        let handler =
15410            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
15411                .with_supervisor(supervisor)
15412                .with_admission_facts_config(
15413                    Some("fed".to_string()),
15414                    Some(vec!["target".to_string()]),
15415                );
15416
15417        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
15418        hello_via_sink(
15419            &handler,
15420            &target_ctx,
15421            &mut target_rx,
15422            hello_frame("target", PROTOCOL_VERSION, 1),
15423        )
15424        .await;
15425        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
15426        hello_via_sink(
15427            &handler,
15428            &other_ctx,
15429            &mut other_rx,
15430            hello_frame("other", PROTOCOL_VERSION, 2),
15431        )
15432        .await;
15433
15434        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
15435        let expected_facts = facts.clone();
15436        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
15437        let route_handler = handler.clone();
15438        let route_task = tokio::spawn(async move {
15439            route_handler
15440                .handle_control_frame(
15441                    &client_ctx,
15442                    route_open_frame_with_admission_facts(
15443                        10,
15444                        "target",
15445                        unique_project_root("admission-facts"),
15446                        Some(subc_control::ConsumerIdentity {
15447                            module_id: "fed".to_string(),
15448                            launch_nonce: "fed-nonce".to_string(),
15449                        }),
15450                        Some(facts.clone()),
15451                    ),
15452                )
15453                .await
15454                .unwrap()
15455        });
15456        let bind_frame = target_rx.recv().await.unwrap();
15457        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15458        let ModuleControlRequest::RouteBind {
15459            admission_facts, ..
15460        } = bind
15461        else {
15462            panic!("expected route.bind")
15463        };
15464        assert_eq!(admission_facts, Some(expected_facts));
15465        handler
15466            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
15467            .await
15468            .unwrap();
15469        assert!(route_task.await.unwrap().is_empty());
15470        assert!(matches!(
15471            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
15472                .unwrap(),
15473            ClientControlResponse::RouteOpen { .. }
15474        ));
15475
15476        let direct = handler
15477            .handle_control_frame(
15478                &route_ctx(ConnectionId::new(73)).0,
15479                route_open_frame_with_admission_facts(
15480                    11,
15481                    "target",
15482                    unique_project_root("admission-facts"),
15483                    None,
15484                    Some(json!({"x": 1})),
15485                ),
15486            )
15487            .await
15488            .unwrap();
15489        assert_eq!(
15490            parse_error(&direct[0])["code"],
15491            "admission_facts_not_permitted"
15492        );
15493
15494        let different_reserved = handler
15495            .handle_control_frame(
15496                &route_ctx(ConnectionId::new(77)).0,
15497                route_open_frame_with_admission_facts(
15498                    15,
15499                    "target",
15500                    unique_project_root("admission-facts"),
15501                    Some(subc_control::ConsumerIdentity {
15502                        module_id: "other".to_string(),
15503                        launch_nonce: "other-nonce".to_string(),
15504                    }),
15505                    Some(json!({"x": 1})),
15506                ),
15507            )
15508            .await
15509            .unwrap();
15510        assert_eq!(
15511            parse_error(&different_reserved[0])["code"],
15512            "admission_facts_not_permitted"
15513        );
15514
15515        let other_target = handler
15516            .handle_control_frame(
15517                &route_ctx(ConnectionId::new(74)).0,
15518                route_open_frame_with_admission_facts(
15519                    12,
15520                    "other",
15521                    unique_project_root("admission-facts"),
15522                    Some(subc_control::ConsumerIdentity {
15523                        module_id: "fed".to_string(),
15524                        launch_nonce: "fed-nonce".to_string(),
15525                    }),
15526                    Some(json!({"x": 1})),
15527                ),
15528            )
15529            .await
15530            .unwrap();
15531        assert_eq!(
15532            parse_error(&other_target[0])["code"],
15533            "admission_facts_target_not_allowed"
15534        );
15535
15536        let nonexistent = handler
15537            .handle_control_frame(
15538                &route_ctx(ConnectionId::new(75)).0,
15539                route_open_frame_with_admission_facts(
15540                    13,
15541                    "missing",
15542                    unique_project_root("admission-facts"),
15543                    None,
15544                    Some(json!({"x": 1})),
15545                ),
15546            )
15547            .await
15548            .unwrap();
15549        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
15550
15551        let described = handler
15552            .handle_control_frame(
15553                &route_ctx(ConnectionId::new(76)).0,
15554                Frame::build(
15555                    FrameType::Request,
15556                    control_flags(),
15557                    0,
15558                    0,
15559                    14,
15560                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
15561                )
15562                .unwrap(),
15563            )
15564            .await
15565            .unwrap();
15566        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15567            serde_json::from_slice(&described[0].body).unwrap()
15568        else {
15569            panic!("expected server.describe response")
15570        };
15571        assert!(capabilities
15572            .iter()
15573            .any(|cap| cap == "admission_facts_relay_v1"));
15574    }
15575
15576    #[tokio::test]
15577    async fn admission_facts_without_configured_carrier_are_rejected() {
15578        let registry = Arc::new(Registry::default());
15579        let forwarding = Arc::new(ForwardingTable::default());
15580        let handler = ControlHandler::with_forwarding(registry, forwarding);
15581        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
15582        hello_via_sink(
15583            &handler,
15584            &target_ctx,
15585            &mut target_rx,
15586            hello_frame("target", PROTOCOL_VERSION, 1),
15587        )
15588        .await;
15589
15590        let responses = handler
15591            .handle_control_frame(
15592                &route_ctx(ConnectionId::new(79)).0,
15593                route_open_frame_with_admission_facts(
15594                    16,
15595                    "target",
15596                    unique_project_root("admission-facts"),
15597                    None,
15598                    Some(json!({"x": 1})),
15599                ),
15600            )
15601            .await
15602            .unwrap();
15603        assert_eq!(
15604            parse_error(&responses[0])["code"],
15605            "admission_facts_not_permitted"
15606        );
15607    }
15608
15609    #[tokio::test]
15610    async fn route_open_relays_consumer_capabilities_verbatim() {
15611        let registry = Arc::new(Registry::default());
15612        let forwarding = Arc::new(ForwardingTable::default());
15613        let handler =
15614            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15615        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
15616        hello_via_sink(
15617            &handler,
15618            &module_ctx,
15619            &mut module_rx,
15620            hello_frame("aft", PROTOCOL_VERSION, 7),
15621        )
15622        .await;
15623
15624        let expected = vec!["elicitation".to_string(), "roots".to_string()];
15625        let expected_for_request = expected.clone();
15626        let project_root = unique_project_root("consumer-capabilities-present");
15627        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
15628        let route_handler = handler.clone();
15629        let route_task = tokio::spawn(async move {
15630            route_handler
15631                .handle_control_frame(
15632                    &client_ctx,
15633                    route_open_frame_with_consumer_capabilities(
15634                        401,
15635                        "aft",
15636                        project_root,
15637                        Some(expected_for_request),
15638                    ),
15639                )
15640                .await
15641                .unwrap()
15642        });
15643        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15644            .await
15645            .unwrap()
15646            .unwrap();
15647        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15648        let ModuleControlRequest::RouteBind {
15649            consumer_capabilities,
15650            ..
15651        } = bind
15652        else {
15653            panic!("expected route.bind request, got {bind:?}");
15654        };
15655        assert_eq!(consumer_capabilities, Some(expected.clone()));
15656
15657        handler
15658            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15659            .await
15660            .unwrap();
15661        let route_response = route_task.await.unwrap();
15662        assert!(route_response.is_empty());
15663        let published = client_rx.recv().await.unwrap();
15664        assert!(matches!(
15665            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15666            ClientControlResponse::RouteOpen { .. }
15667        ));
15668    }
15669
15670    #[tokio::test]
15671    async fn route_open_without_consumer_capabilities_relays_none() {
15672        let registry = Arc::new(Registry::default());
15673        let forwarding = Arc::new(ForwardingTable::default());
15674        let handler =
15675            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15676        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
15677        hello_via_sink(
15678            &handler,
15679            &module_ctx,
15680            &mut module_rx,
15681            hello_frame("aft", PROTOCOL_VERSION, 7),
15682        )
15683        .await;
15684
15685        let project_root = unique_project_root("consumer-capabilities-absent");
15686        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
15687        let route_handler = handler.clone();
15688        let route_task = tokio::spawn(async move {
15689            route_handler
15690                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
15691                .await
15692                .unwrap()
15693        });
15694        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15695            .await
15696            .unwrap()
15697            .unwrap();
15698        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15699        let ModuleControlRequest::RouteBind {
15700            consumer_capabilities,
15701            ..
15702        } = bind
15703        else {
15704            panic!("expected route.bind request, got {bind:?}");
15705        };
15706        assert_eq!(consumer_capabilities, None);
15707
15708        handler
15709            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15710            .await
15711            .unwrap();
15712        let route_response = route_task.await.unwrap();
15713        assert!(route_response.is_empty());
15714        let published = client_rx.recv().await.unwrap();
15715        assert!(matches!(
15716            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15717            ClientControlResponse::RouteOpen { .. }
15718        ));
15719    }
15720
15721    /// Opens a route to a freshly registered `aft` with `sent` as the
15722    /// route.open's role_versions, acks the bind, and returns the role_versions
15723    /// the module's bind carried.
15724    async fn bind_role_versions_for(
15725        sent: Option<BTreeMap<String, String>>,
15726        connection: u64,
15727    ) -> Option<BTreeMap<String, String>> {
15728        let registry = Arc::new(Registry::default());
15729        let forwarding = Arc::new(ForwardingTable::default());
15730        let handler =
15731            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15732        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
15733        hello_via_sink(
15734            &handler,
15735            &module_ctx,
15736            &mut module_rx,
15737            hello_frame("aft", PROTOCOL_VERSION, 7),
15738        )
15739        .await;
15740        let project_root = unique_project_root("role-versions");
15741        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
15742        let route_handler = handler.clone();
15743        let route_task = tokio::spawn(async move {
15744            route_handler
15745                .handle_control_frame(
15746                    &client_ctx,
15747                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
15748                )
15749                .await
15750                .unwrap()
15751        });
15752        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15753            .await
15754            .expect("a well-formed route.open reaches the module as a bind")
15755            .unwrap();
15756        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
15757        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
15758            panic!("expected route.bind request, got {bind:?}");
15759        };
15760        handler
15761            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
15762            .await
15763            .unwrap();
15764        assert!(route_task.await.unwrap().is_empty());
15765        let published = client_rx.recv().await.unwrap();
15766        assert!(matches!(
15767            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
15768            ClientControlResponse::RouteOpen { .. }
15769        ));
15770        role_versions
15771    }
15772
15773    #[tokio::test]
15774    async fn route_open_relays_role_versions_verbatim() {
15775        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
15776        assert_eq!(
15777            bind_role_versions_for(Some(sent.clone()), 141).await,
15778            Some(sent)
15779        );
15780    }
15781
15782    /// An empty map declares nothing, so the provider sees no field rather
15783    /// than an empty object it would have to treat as a second "none".
15784    #[tokio::test]
15785    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
15786        assert_eq!(bind_role_versions_for(None, 143).await, None);
15787        assert_eq!(
15788            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
15789            None
15790        );
15791    }
15792
15793    #[tokio::test]
15794    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
15795        let registry = Arc::new(Registry::default());
15796        let forwarding = Arc::new(ForwardingTable::default());
15797        let handler =
15798            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
15799        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
15800        hello_via_sink(
15801            &handler,
15802            &module_ctx,
15803            &mut module_rx,
15804            hello_frame("aft", PROTOCOL_VERSION, 7),
15805        )
15806        .await;
15807
15808        let nine: BTreeMap<String, String> = (0..9)
15809            .map(|index| (format!("role-{index}"), "v1".to_string()))
15810            .collect();
15811        for (label, malformed) in [
15812            (
15813                "invalid role name",
15814                role_versions(&[("Tool_Provider", "v1")]),
15815            ),
15816            ("invalid version", role_versions(&[("tool-provider", "v0")])),
15817            ("nine entries", nine),
15818        ] {
15819            // A refused open answers at once; one that reached the module
15820            // would wait for its bind ack and trip this timeout.
15821            let responses = tokio::time::timeout(
15822                Duration::from_secs(1),
15823                handler.handle_control_frame(
15824                    &route_ctx(ConnectionId::new(148)).0,
15825                    route_open_frame_with_role_versions(
15826                        404,
15827                        "aft",
15828                        unique_project_root("role-versions-malformed"),
15829                        Some(malformed),
15830                    ),
15831                ),
15832            )
15833            .await
15834            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
15835            .unwrap();
15836            assert_eq!(responses.len(), 1, "{label}");
15837            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
15838            let error = parse_error(&responses[0]);
15839            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
15840            assert_eq!(
15841                error["detail"]["field"], "role_versions",
15842                "{label}: {error}"
15843            );
15844            assert!(
15845                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
15846                "{label}: a malformed declaration is terminal"
15847            );
15848            assert!(
15849                module_rx.try_recv().is_err(),
15850                "{label}: the module must never see a bind"
15851            );
15852        }
15853    }
15854
15855    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
15856    /// consumer can tell this daemon forwards the field from one that would
15857    /// drop it.
15858    #[tokio::test]
15859    async fn route_role_versions_capability_is_advertised() {
15860        let handler = ControlHandler::new(Arc::new(Registry::default()));
15861        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
15862        let ack = hello_via_sink(
15863            &handler,
15864            &ctx,
15865            &mut rx,
15866            hello_frame("m", PROTOCOL_VERSION, 1),
15867        )
15868        .await;
15869        let ack = parse_ack(&ack);
15870        assert!(
15871            ack.subc_capabilities
15872                .iter()
15873                .any(|c| c == "route-role-versions/v1"),
15874            "{:?}",
15875            ack.subc_capabilities
15876        );
15877
15878        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15879        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15880        let reply = handler
15881            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
15882            .await
15883            .unwrap()
15884            .pop()
15885            .unwrap();
15886        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15887            serde_json::from_slice(&reply.body).unwrap()
15888        else {
15889            panic!("not a server.describe reply");
15890        };
15891        assert!(
15892            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
15893            "{capabilities:?}"
15894        );
15895    }
15896
15897    /// `catalog-events/v1` is in `server.describe`, so a consumer can tell a
15898    /// catalog entry without `events` (the module declares none) from a daemon
15899    /// that never reports declared events.
15900    #[tokio::test]
15901    async fn catalog_events_capability_is_advertised() {
15902        let handler = ControlHandler::new(Arc::new(Registry::default()));
15903        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15904        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15905        let reply = handler
15906            .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
15907            .await
15908            .unwrap()
15909            .pop()
15910            .unwrap();
15911        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15912            serde_json::from_slice(&reply.body).unwrap()
15913        else {
15914            panic!("not a server.describe reply");
15915        };
15916        assert!(
15917            capabilities
15918                .iter()
15919                .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
15920            "{capabilities:?}"
15921        );
15922    }
15923
15924    #[tokio::test]
15925    async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
15926        let handler = ControlHandler::new(Arc::new(Registry::default()));
15927        let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
15928        let ack = hello_via_sink(
15929            &handler,
15930            &module,
15931            &mut module_rx,
15932            hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
15933        )
15934        .await;
15935        let ack = parse_ack(&ack);
15936        assert!(
15937            ack.subc_capabilities
15938                .iter()
15939                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15940            "{:?}",
15941            ack.subc_capabilities
15942        );
15943        assert!(ack
15944            .subc_capabilities
15945            .iter()
15946            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15947
15948        let (client, _client_rx) = route_ctx(ConnectionId::new(153));
15949        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15950        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15951        let reply = handler
15952            .handle_control_frame(&client, frame)
15953            .await
15954            .unwrap()
15955            .pop()
15956            .unwrap();
15957        let ClientControlResponse::ServerDescribe { capabilities, .. } =
15958            serde_json::from_slice(&reply.body).unwrap()
15959        else {
15960            panic!("not a server.describe reply");
15961        };
15962        assert!(
15963            capabilities
15964                .iter()
15965                .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
15966            "{capabilities:?}"
15967        );
15968        assert!(capabilities
15969            .iter()
15970            .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
15971    }
15972
15973    #[tokio::test]
15974    async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
15975        let handler = ControlHandler::new(Arc::new(Registry::default()));
15976        let (module, mut rx) = route_ctx(ConnectionId::new(156));
15977        let ack = hello_via_sink(
15978            &handler,
15979            &module,
15980            &mut rx,
15981            hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
15982        )
15983        .await;
15984        assert!(parse_ack(&ack)
15985            .subc_capabilities
15986            .iter()
15987            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
15988        assert!(parse_ack(&ack)
15989            .subc_capabilities
15990            .iter()
15991            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
15992        let (client, _rx) = route_ctx(ConnectionId::new(157));
15993        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15994        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15995        let reply = handler
15996            .handle_control_frame(&client, frame)
15997            .await
15998            .unwrap()
15999            .pop()
16000            .unwrap();
16001        let ClientControlResponse::ServerDescribe { capabilities, .. } =
16002            serde_json::from_slice(&reply.body).unwrap()
16003        else {
16004            panic!("server.describe reply")
16005        };
16006        assert!(capabilities
16007            .iter()
16008            .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
16009        assert!(capabilities
16010            .iter()
16011            .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
16012    }
16013
16014    #[tokio::test]
16015    async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
16016        let handler = ControlHandler::new(Arc::new(Registry::default()));
16017        let (module, mut rx) = route_ctx(ConnectionId::new(158));
16018        let ack = hello_via_sink(
16019            &handler,
16020            &module,
16021            &mut rx,
16022            hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
16023        )
16024        .await;
16025        assert!(parse_ack(&ack)
16026            .subc_capabilities
16027            .iter()
16028            .any(|cap| cap == "scope-workdir/v1"));
16029        let (client, _rx) = route_ctx(ConnectionId::new(159));
16030        let frame = Frame::build(
16031            FrameType::Request,
16032            control_flags(),
16033            0,
16034            0,
16035            5,
16036            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
16037        )
16038        .unwrap();
16039        let reply = handler
16040            .handle_control_frame(&client, frame)
16041            .await
16042            .unwrap()
16043            .remove(0);
16044        let ClientControlResponse::ServerDescribe { capabilities, .. } =
16045            serde_json::from_slice(&reply.body).unwrap()
16046        else {
16047            panic!("server.describe reply")
16048        };
16049        assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
16050    }
16051
16052    #[tokio::test]
16053    async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
16054        let handler = ControlHandler::new(Arc::new(Registry::default()));
16055        let (module, mut rx) = route_ctx(ConnectionId::new(160));
16056        let ack = hello_via_sink(
16057            &handler,
16058            &module,
16059            &mut rx,
16060            hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
16061        )
16062        .await;
16063        assert!(parse_ack(&ack)
16064            .subc_capabilities
16065            .iter()
16066            .any(|cap| cap == "worker-runs/v1"));
16067        let (client, _rx) = route_ctx(ConnectionId::new(161));
16068        let frame = Frame::build(
16069            FrameType::Request,
16070            control_flags(),
16071            0,
16072            0,
16073            5,
16074            serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
16075        )
16076        .unwrap();
16077        let reply = handler
16078            .handle_control_frame(&client, frame)
16079            .await
16080            .unwrap()
16081            .remove(0);
16082        let ClientControlResponse::ServerDescribe { capabilities, .. } =
16083            serde_json::from_slice(&reply.body).unwrap()
16084        else {
16085            panic!("server.describe reply")
16086        };
16087        assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
16088    }
16089
16090    #[tokio::test]
16091    async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
16092        let handler = ControlHandler::new(Arc::new(Registry::default()));
16093        let (module, mut rx) = route_ctx(ConnectionId::new(154));
16094        let ack = hello_via_sink(
16095            &handler,
16096            &module,
16097            &mut rx,
16098            hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
16099        )
16100        .await;
16101        assert!(parse_ack(&ack)
16102            .subc_capabilities
16103            .iter()
16104            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16105        let (client, _rx) = route_ctx(ConnectionId::new(155));
16106        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16107        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16108        let reply = handler
16109            .handle_control_frame(&client, frame)
16110            .await
16111            .unwrap()
16112            .pop()
16113            .unwrap();
16114        let ClientControlResponse::ServerDescribe { capabilities, .. } =
16115            serde_json::from_slice(&reply.body).unwrap()
16116        else {
16117            panic!("not a server.describe reply")
16118        };
16119        assert!(capabilities
16120            .iter()
16121            .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
16122    }
16123
16124    #[tokio::test]
16125    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
16126        let registry = Arc::new(Registry::default());
16127        let forwarding = Arc::new(ForwardingTable::default());
16128        let handler =
16129            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
16130                .with_health_probe_timeout(Duration::from_secs(5));
16131        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
16132        hello_via_sink(
16133            &handler,
16134            &module_ctx,
16135            &mut module_rx,
16136            non_routable_hello_frame_with_control_ops(
16137                "mcp",
16138                300,
16139                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
16140            ),
16141        )
16142        .await;
16143        assert!(registry
16144            .get_module("mcp")
16145            .unwrap()
16146            .unwrap()
16147            .manifest
16148            .provides
16149            .is_empty());
16150
16151        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
16152        let route_response = handler
16153            .handle_control_frame(
16154                &route_client_ctx,
16155                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
16156            )
16157            .await
16158            .unwrap();
16159        assert_eq!(route_response[0].header.ty, FrameType::Error);
16160        assert_eq!(
16161            parse_error(&route_response[0])["code"],
16162            "target_unavailable"
16163        );
16164        assert!(parse_error(&route_response[0])["message"]
16165            .as_str()
16166            .unwrap()
16167            .contains("does not provide the requested target"));
16168        assert!(module_rx.try_recv().is_err());
16169
16170        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
16171        let health_handler = handler.clone();
16172        let health_task = tokio::spawn(async move {
16173            health_handler
16174                .handle_control_frame(
16175                    &health_client_ctx,
16176                    supervisor_health_probe_frame(302, "mcp"),
16177                )
16178                .await
16179                .unwrap()
16180        });
16181        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
16182            .await
16183            .unwrap()
16184            .unwrap();
16185        assert_eq!(
16186            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
16187            ModuleControlRequest::HealthCheck {}
16188        );
16189        handler
16190            .handle_control_frame(
16191                &module_ctx,
16192                health_response(health_frame.header.corr, HealthStatus::Ok),
16193            )
16194            .await
16195            .unwrap();
16196        let health_response = health_task.await.unwrap();
16197        assert_eq!(health_response[0].header.ty, FrameType::Response);
16198        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
16199            ClientControlResponse::SupervisorHealthProbe {
16200                module_id, status, ..
16201            } => {
16202                assert_eq!(module_id, "mcp");
16203                assert_eq!(status, HealthStatus::Ok);
16204            }
16205            other => panic!("unexpected health response: {other:?}"),
16206        }
16207
16208        // Exercise the forwarding cleanup path directly while leaving the registry
16209        // advertisement in place. If cleanup leaves a stale control sink behind,
16210        // the next probe will enqueue onto it and wait for the long probe timeout
16211        // instead of returning an immediate no-connection error.
16212        forwarding
16213            .cleanup_connection(module_ctx.connection_id)
16214            .unwrap();
16215        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
16216        let cleanup_response = tokio::time::timeout(
16217            Duration::from_millis(200),
16218            handler.handle_control_frame(
16219                &cleanup_probe_ctx,
16220                supervisor_health_probe_frame(303, "mcp"),
16221            ),
16222        )
16223        .await
16224        .expect("probe should fail immediately when the control lane is gone")
16225        .unwrap();
16226        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
16227        assert_eq!(
16228            parse_error(&cleanup_response[0])["code"],
16229            "target_unavailable"
16230        );
16231        assert!(parse_error(&cleanup_response[0])["message"]
16232            .as_str()
16233            .unwrap()
16234            .contains("no module connection"));
16235
16236        handler
16237            .cleanup_connection(module_ctx.connection_id)
16238            .unwrap();
16239    }
16240
16241    #[tokio::test]
16242    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
16243        let registry = Arc::new(Registry::default());
16244        let supervisor_handle = SupervisorHandle::new();
16245        let supervisor =
16246            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
16247                .with_handle(supervisor_handle.clone())
16248                .with_connection_file_path(
16249                    std::env::temp_dir()
16250                        .join(format!("subc-route-open-warming-{}", std::process::id())),
16251                );
16252        let module = supervisor
16253            .supervise_configured(
16254                ModuleSpec {
16255                    script: None,
16256                    carry_containment: false,
16257                    module_id: "warming".to_string(),
16258                    program: fake_aft_stub_path(),
16259                    args: Vec::new(),
16260                    env: Vec::new(),
16261                    reserved: false,
16262                    reserved_prefixes: Vec::new(),
16263                    protocol: ModuleProtocol::Subc,
16264                    overlap: Default::default(),
16265                },
16266                true,
16267            )
16268            .unwrap();
16269        assert_eq!(module.state().unwrap(), ModuleState::Running);
16270
16271        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
16272        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
16273        let response = handler
16274            .handle_control_frame(
16275                &ctx,
16276                route_open_frame(304, "warming", unique_project_root("warming")),
16277            )
16278            .await
16279            .unwrap();
16280        module.stop().await.unwrap();
16281
16282        assert_eq!(response[0].header.ty, FrameType::Error);
16283        let error = parse_error(&response[0]);
16284        assert_eq!(error["code"], "module_warming");
16285        assert!(error["message"]
16286            .as_str()
16287            .unwrap()
16288            .contains("state=running, enabled=true, live=false"));
16289    }
16290
16291    #[test]
16292    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
16293        let handler = ControlHandler::new(Arc::new(Registry::default()));
16294        let capture = EventCapture::default();
16295        let _subscriber =
16296            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16297        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
16298        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
16299        let pending = (0..limit).collect::<Vec<_>>();
16300        let response = handler
16301            .route_open_capacity_refusal(
16302                &ctx,
16303                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
16304                "busy",
16305                pending.len(),
16306                limit,
16307            )
16308            .unwrap();
16309        assert_eq!(parse_error(&response)["code"], "target_unavailable");
16310        let event = capture
16311            .events()
16312            .into_iter()
16313            .find(|event| {
16314                event.target == "control"
16315                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
16316            })
16317            .expect("connection admission refusal event");
16318        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16319        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16320        // Refused before `handle_route_open` parses the request, so the caller
16321        // fields are read from the frame itself.
16322        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
16323        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
16324    }
16325
16326    #[test]
16327    fn route_open_target_cap_logs_admission_reason_and_capacity() {
16328        let handler = ControlHandler::new(Arc::new(Registry::default()));
16329        let capture = EventCapture::default();
16330        let _subscriber =
16331            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16332        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
16333        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
16334        let guards = (0..limit)
16335            .map(|_| {
16336                handler
16337                    .route_bind_concurrency
16338                    .try_admit("busy", limit)
16339                    .unwrap()
16340            })
16341            .collect::<Vec<_>>();
16342        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
16343            Err(in_flight) => in_flight,
16344            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
16345        };
16346        let response = handler
16347            .route_open_target_capacity_refusal(
16348                &ctx,
16349                &RouteOpenCaller::unknown(),
16350                &route_open_frame(397, "busy", unique_project_root("target-cap")),
16351                "busy",
16352                in_flight,
16353            )
16354            .unwrap();
16355        assert_eq!(parse_error(&response)["code"], "target_unavailable");
16356        let event = capture
16357            .events()
16358            .into_iter()
16359            .find(|event| {
16360                event.target == "control"
16361                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
16362            })
16363            .expect("target admission refusal event");
16364        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
16365        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
16366        drop(guards);
16367    }
16368
16369    #[cfg(unix)]
16370    #[tokio::test(start_paused = true)]
16371    async fn supervisor_list_shows_a_failed_modules_next_retry() {
16372        let registry = Arc::new(Registry::default());
16373        let handle = SupervisorHandle::new();
16374        let supervisor =
16375            Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
16376                .with_handle(handle.clone());
16377        let module = supervisor
16378            .spawn(crate::ModuleSpec {
16379                script: None,
16380                carry_containment: false,
16381                module_id: "retry-list".into(),
16382                program: PathBuf::from("/bin/sh"),
16383                args: vec!["-c".into(), "exit 17".into()],
16384                env: vec![],
16385                reserved: false,
16386                reserved_prefixes: vec![],
16387                protocol: ModuleProtocol::None,
16388                overlap: Default::default(),
16389            })
16390            .unwrap();
16391        let deadline = std::time::Instant::now() + Duration::from_secs(10);
16392        while module.state().unwrap() != crate::ModuleState::Failed {
16393            assert!(
16394                std::time::Instant::now() < deadline,
16395                "waiting for budget failure"
16396            );
16397            tokio::task::yield_now().await;
16398        }
16399        let handler = ControlHandler::new(registry).with_supervisor(handle);
16400        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16401        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16402        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16403        assert_eq!(body["modules"][0]["state"], "failed");
16404        let retry = body["modules"][0]["next_retry_at_ms"]
16405            .as_u64()
16406            .expect("scheduled retry on the wire");
16407        let now = std::time::SystemTime::now()
16408            .duration_since(std::time::UNIX_EPOCH)
16409            .unwrap()
16410            .as_millis() as u64;
16411        assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
16412        module.set_enabled(false).await.unwrap();
16413        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16414        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16415        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16416        assert!(body["modules"][0].get("next_retry_at_ms").is_none());
16417    }
16418
16419    #[tokio::test]
16420    async fn slow_step_before_pid_read_does_not_spend_its_budget() {
16421        let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
16422        let supervisor =
16423            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
16424        let module = supervisor
16425            .spawn(crate::ModuleSpec {
16426                script: None,
16427                carry_containment: false,
16428                module_id: "pid-read-budget".into(),
16429                program: std::env::current_exe().unwrap(),
16430                args: vec![
16431                    "--exact".into(),
16432                    "control::tests::pid_resource_read_fixture".into(),
16433                ],
16434                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16435                    .into_iter()
16436                    .map(|key| (key.into(), dir.display().to_string()))
16437                    .chain(std::iter::once((
16438                        "SUBC_SUBTREE_PID_FIXTURE".into(),
16439                        "1".into(),
16440                    )))
16441                    .collect(),
16442                reserved: false,
16443                reserved_prefixes: Vec::new(),
16444                protocol: ModuleProtocol::None,
16445                overlap: Default::default(),
16446            })
16447            .unwrap();
16448        let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16449        while !matches!(
16450            module.child_resource_usage(),
16451            subc_control::ChildResourceUsage::Measured(_)
16452        ) {
16453            assert!(
16454                tokio::time::Instant::now() < ready_deadline,
16455                "fixture process was not measurable"
16456            );
16457            tokio::time::sleep(Duration::from_millis(10)).await;
16458        }
16459        tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
16460        let reading = {
16461            let _guard = crate::subtree_resources::fault_read_guard().await;
16462            crate::subtree_resources::read_process(module.clone()).await
16463        };
16464        stop_resource_fixture(&module, &dir).await;
16465        assert!(
16466            matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
16467            "{reading:?}"
16468        );
16469    }
16470
16471    #[test]
16472    fn pid_resource_read_fixture() {
16473        if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
16474            let dir = std::path::PathBuf::from(std::env::var_os("XDG_DATA_HOME").unwrap());
16475            let deadline = std::time::Instant::now() + Duration::from_secs(60);
16476            while !dir.join("stop-module").exists() && std::time::Instant::now() < deadline {
16477                std::thread::sleep(Duration::from_millis(5));
16478            }
16479            std::fs::write(dir.join("module-stopped"), "").unwrap();
16480        }
16481    }
16482
16483    async fn stop_resource_fixture(
16484        module: &crate::supervise::SupervisedModule,
16485        dir: &std::path::Path,
16486    ) {
16487        let stopping = module.stop();
16488        tokio::pin!(stopping);
16489        tokio::select! {
16490            result = &mut stopping => result.unwrap(),
16491            () = async {
16492                while module.state().unwrap() != crate::supervise::ModuleState::Draining {
16493                    tokio::time::sleep(Duration::from_millis(5)).await;
16494                }
16495                std::fs::write(dir.join("stop-module"), "").unwrap();
16496            } => stopping.await.unwrap(),
16497        }
16498        #[cfg(windows)]
16499        assert!(
16500            dir.join("module-stopped").exists(),
16501            "the resource fixture must exit cooperatively, not wait for the drain kill"
16502        );
16503        assert_eq!(
16504            module.state().unwrap(),
16505            crate::supervise::ModuleState::Stopped
16506        );
16507    }
16508
16509    #[tokio::test]
16510    async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
16511        let registry = Arc::new(Registry::default());
16512        let handle = SupervisorHandle::new();
16513        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16514            .with_handle(handle.clone());
16515        supervisor
16516            .supervise_configured(
16517                crate::ModuleSpec {
16518                    script: None,
16519                    module_id: "disabled-resource-reading".into(),
16520                    program: std::env::current_exe().unwrap(),
16521                    args: Vec::new(),
16522                    env: Vec::new(),
16523                    reserved: false,
16524                    reserved_prefixes: Vec::new(),
16525                    protocol: ModuleProtocol::None,
16526                    overlap: Default::default(),
16527                    carry_containment: false,
16528                },
16529                false,
16530            )
16531            .unwrap();
16532        let handler = ControlHandler::new(registry).with_supervisor(handle);
16533        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16534        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16535        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16536        assert!(body["modules"][0].get("tree").is_none());
16537        assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
16538    }
16539
16540    #[tokio::test]
16541    async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
16542        let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
16543        let registry = Arc::new(Registry::default());
16544        let handle = SupervisorHandle::new();
16545        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16546            .with_handle(handle.clone());
16547        let module = supervisor
16548            .spawn(crate::ModuleSpec {
16549                script: None,
16550                carry_containment: false,
16551                module_id: "list-process-priority".into(),
16552                program: std::env::current_exe().unwrap(),
16553                args: vec![
16554                    "--exact".into(),
16555                    "control::tests::pid_resource_read_fixture".into(),
16556                ],
16557                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16558                    .into_iter()
16559                    .map(|key| (key.into(), dir.display().to_string()))
16560                    .chain(std::iter::once((
16561                        "SUBC_SUBTREE_PID_FIXTURE".into(),
16562                        "1".into(),
16563                    )))
16564                    .collect(),
16565                reserved: false,
16566                reserved_prefixes: Vec::new(),
16567                protocol: ModuleProtocol::None,
16568                overlap: Default::default(),
16569            })
16570            .unwrap();
16571        let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16572        loop {
16573            let image = module.running_image_agreement().await;
16574            if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
16575                break;
16576            }
16577            assert!(
16578                tokio::time::Instant::now() < ready_deadline,
16579                "the fixture process never became observable: {image:?}"
16580            );
16581            tokio::time::sleep(Duration::from_millis(10)).await;
16582        }
16583        let guard = crate::subtree_resources::fault_read_guard().await;
16584        let (target, release, _) = crate::subtree_resources::held();
16585        module.set_subtree_target_for_test(target);
16586        let handler = ControlHandler::new(registry).with_supervisor(handle);
16587        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16588        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16589        drop(release);
16590        drop(guard);
16591        stop_resource_fixture(&module, &dir).await;
16592        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16593        assert_eq!(
16594            body["modules"][0]["resources"]["status"], "measured",
16595            "the optional tree timeout must not discard the process reading: {body}"
16596        );
16597        assert!(body["modules"][0].get("tree").is_none());
16598        assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
16599        assert_eq!(
16600            body["modules"][0]["pending_reload"]["image"]["status"], "match",
16601            "the tree timeout must not hide a completed image observation"
16602        );
16603    }
16604
16605    #[cfg(any(target_os = "linux", target_os = "macos"))]
16606    #[tokio::test]
16607    async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
16608        let mut handler = ControlHandler::new(Arc::new(Registry::default()))
16609            .with_daemon_provenance(
16610                std::process::id(),
16611                0,
16612                Some(std::env::current_exe().unwrap()),
16613                None,
16614                None,
16615            );
16616        let (probe, release, began) = ExecutableIdentityProbe::held();
16617        handler.daemon_provenance.probe = probe;
16618        let observe = |handler: ControlHandler, corr| {
16619            tokio::spawn(async move {
16620                let (ctx, _) = route_ctx(ConnectionId::new(corr));
16621                let frame =
16622                    Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
16623                        .unwrap();
16624                handler
16625                    .handle_supervisor_provenance(&ctx, frame, None)
16626                    .await
16627                    .unwrap()
16628            })
16629        };
16630        let explicit = observe(handler.clone(), 114);
16631        began.await.unwrap();
16632        let concurrent = observe(handler.clone(), 115);
16633        // Keep explicit provenance waiting on its first image observation beyond
16634        // the list deadline; release it only after the concurrent list replies.
16635        tokio::time::sleep(Duration::from_millis(300)).await;
16636        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16637        let list = tokio::time::timeout(
16638            Duration::from_secs(1),
16639            handler.handle_supervisor_list(frame),
16640        )
16641        .await;
16642        let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
16643        drop(release);
16644        let replies = explicit.await.unwrap();
16645        let concurrent_replies = concurrent.await.unwrap();
16646        assert!(
16647            explicit_was_pending,
16648            "explicit provenance must await the full observation"
16649        );
16650        let list = list
16651            .expect("a concurrent explicit probe must not stall list")
16652            .unwrap();
16653        let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
16654        assert_eq!(list_body["op"], "supervisor.list");
16655        for replies in [replies, concurrent_replies] {
16656            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16657            assert_eq!(
16658                body["daemon"]["daemon_observed"]["running_image"]["status"],
16659                "match"
16660            );
16661        }
16662    }
16663
16664    #[tokio::test]
16665    async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
16666        let registry = Arc::new(Registry::default());
16667        let handle = SupervisorHandle::new();
16668        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16669            .with_handle(handle.clone());
16670        let module = supervisor
16671            .supervise_configured(
16672                crate::ModuleSpec {
16673                    script: None,
16674                    carry_containment: false,
16675                    module_id: "memory-read".into(),
16676                    program: PathBuf::from("disabled-module"),
16677                    args: Vec::new(),
16678                    env: Vec::new(),
16679                    reserved: false,
16680                    reserved_prefixes: Vec::new(),
16681                    protocol: ModuleProtocol::Subc,
16682                    overlap: Default::default(),
16683                },
16684                true,
16685            )
16686            .unwrap();
16687        let _guard = crate::subtree_resources::fault_read_guard().await;
16688        let (target, release, began) = crate::subtree_resources::held();
16689        // Optional list work may be skipped when its deadline is already spent.
16690        // Establish the stalled fixture before asking the list to observe it.
16691        let stalled_target = target.clone();
16692        let stalled = tokio::task::spawn_blocking(move || {
16693            stalled_target.tree(&subc_os::ProcessTable::default())
16694        });
16695        began.await.unwrap();
16696        module.set_subtree_target_for_test(target);
16697        let handler = ControlHandler::new(registry).with_supervisor(handle);
16698        let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
16699        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16700        let replies = handler
16701            .handle_client_control_request(
16702                &ctx,
16703                frame,
16704                ClientControlRequest::SupervisorList {},
16705                &mut None,
16706            )
16707            .await
16708            .unwrap();
16709        assert!(
16710            replies.is_empty(),
16711            "list reply must be detached from the connection reader"
16712        );
16713        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16714        let replies = handler
16715            .handle_client_control_request(
16716                &ctx,
16717                frame,
16718                ClientControlRequest::SupervisorSpawnSnapshot {},
16719                &mut None,
16720            )
16721            .await
16722            .unwrap();
16723        assert_eq!(
16724            replies.len(),
16725            1,
16726            "another request must complete while the subtree read stalls"
16727        );
16728        let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16729            .await
16730            .unwrap()
16731            .unwrap();
16732        release.send(()).unwrap();
16733        drop(release);
16734        stalled.await.unwrap();
16735        let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
16736        assert!(
16737            body["modules"][0].get("tree").is_none(),
16738            "an unknown tree reading must be omitted after the bounded read expires"
16739        );
16740        // Lists export completed work rather than measuring synchronously.
16741        // Complete a refresh after the fixture is released before asserting
16742        // that a later list contains its measured tree.
16743        handler
16744            .resource_tracker
16745            .read(handler.supervisor.list(), false, "released_tree_fixture")
16746            .await;
16747        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
16748        let replies = handler.handle_supervisor_list(frame).await.unwrap();
16749        let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16750        assert_eq!(
16751            body["modules"][0]["tree"]["source"], "process_tree",
16752            "the next request must export the completed refresh"
16753        );
16754        assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
16755    }
16756
16757    #[tokio::test]
16758    async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
16759        use subc_control::{
16760            AgentCommandResources, ChildMemoryKind, ChildResourceReading,
16761            ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
16762            TreeReading, TreeSource,
16763        };
16764        let registry = Arc::new(Registry::default());
16765        let handle = SupervisorHandle::new();
16766        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16767            .with_handle(handle.clone());
16768        supervisor
16769            .supervise_configured(
16770                crate::ModuleSpec {
16771                    script: None,
16772                    carry_containment: false,
16773                    module_id: "budget-worker".into(),
16774                    program: PathBuf::from("disabled-module"),
16775                    args: Vec::new(),
16776                    env: Vec::new(),
16777                    reserved: false,
16778                    reserved_prefixes: Vec::new(),
16779                    protocol: ModuleProtocol::Subc,
16780                    overlap: Default::default(),
16781                },
16782                true,
16783            )
16784            .unwrap();
16785        let handler = ControlHandler::new(registry).with_supervisor(handle);
16786        let now = u64::try_from(
16787            std::time::SystemTime::now()
16788                .duration_since(std::time::UNIX_EPOCH)
16789                .unwrap()
16790                .as_millis(),
16791        )
16792        .unwrap();
16793        let measured = now - 1_000;
16794        let hour = CpuLastHour::new(100, 60_000);
16795        let daemon = DaemonResources::new(7, 1, "test")
16796            .with_measured_at_ms(Some(measured))
16797            .with_resources(Some(ChildResourceUsage::Measured(
16798                ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
16799            )))
16800            .with_cpu_last_hour(Some(hour.clone()));
16801        let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
16802            .with_measured_at_ms(Some(measured))
16803            .with_cpu_last_hour(Some(hour.clone()));
16804        let tree = TreeReading::new(
16805            150,
16806            50,
16807            ChildMemoryKind::ResidentSet,
16808            1,
16809            TreeSource::ProcessTree,
16810        )
16811        .with_measured_at_ms(Some(measured));
16812        handler.resource_tracker.seed_committed_for_test(
16813            &handler.supervisor.list(),
16814            crate::resource_totals::Readings {
16815                modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
16816                daemon: Some(daemon),
16817                agent_commands: Some(vec![agents]),
16818                ..Default::default()
16819            },
16820            now,
16821        );
16822        list_diagnostics::scope(async {
16823            tokio::time::sleep(Duration::from_millis(110)).await;
16824            assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
16825            let request =
16826                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16827            let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
16828            let ClientControlResponse::SupervisorList {
16829                modules,
16830                daemon,
16831                agent_commands,
16832                ..
16833            } = serde_json::from_slice(&replies[0].body).unwrap()
16834            else {
16835                panic!("expected a list response")
16836            };
16837            assert!(matches!(
16838                modules[0].resources,
16839                Some(ChildResourceUsage::Unavailable {
16840                    reason: ChildResourceUnavailableReason::Unreadable
16841                })
16842            ));
16843            assert_eq!(
16844                (
16845                    modules[0].tree.as_ref().unwrap().cpu_ms,
16846                    modules[0].tree.as_ref().unwrap().measured_at_ms
16847                ),
16848                (150, Some(measured))
16849            );
16850            let daemon = daemon.unwrap();
16851            let agents = agent_commands.unwrap();
16852            assert_eq!(daemon.measured_at_ms, Some(measured));
16853            assert_eq!(
16854                (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
16855                (1, 50, Some(measured))
16856            );
16857            for hour in [
16858                modules[0].cpu_last_hour.as_ref(),
16859                daemon.cpu_last_hour.as_ref(),
16860                agents[0].cpu_last_hour.as_ref(),
16861            ] {
16862                let hour = hour.unwrap();
16863                assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
16864            }
16865        })
16866        .await;
16867    }
16868
16869    #[tokio::test]
16870    async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
16871        let registry = Arc::new(Registry::default());
16872        let handle = SupervisorHandle::new();
16873        let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
16874            .with_handle(handle.clone());
16875        supervisor
16876            .supervise_configured(
16877                crate::ModuleSpec {
16878                    script: None,
16879                    carry_containment: false,
16880                    module_id: "drift".into(),
16881                    program: PathBuf::from("disabled-module"),
16882                    args: Vec::new(),
16883                    env: Vec::new(),
16884                    reserved: false,
16885                    reserved_prefixes: Vec::new(),
16886                    protocol: ModuleProtocol::Subc,
16887                    overlap: Default::default(),
16888                },
16889                false,
16890            )
16891            .unwrap();
16892        let handler = ControlHandler::new(registry).with_supervisor(handle);
16893        for configured in [None, Some(false), Some(true)] {
16894            handler.capability_evaluator.configure(
16895                configured.map(|enabled| ("drift".to_string(), enabled)),
16896                BTreeMap::new(),
16897            );
16898            let frame =
16899                Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16900            let replies = handler
16901                .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
16902                .await
16903                .unwrap();
16904            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16905            assert!(body.get("configured_enabled").is_none());
16906            assert_eq!(body["op"], "supervisor.ack");
16907            let frame =
16908                Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
16909            let replies = handler.handle_supervisor_list(frame).await.unwrap();
16910            let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
16911            assert_eq!(
16912                body["modules"][0].get("configured_enabled"),
16913                configured.map(Value::Bool).as_ref()
16914            );
16915            assert!(body["modules"][0].get("tree").is_none());
16916            assert!(body["modules"][0].get("subtree").is_none());
16917        }
16918    }
16919
16920    #[tokio::test(flavor = "current_thread")]
16921    async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
16922        let handler = ControlHandler::new(Arc::new(Registry::default()));
16923        let capture = EventCapture::default();
16924        let _subscriber =
16925            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16926        let (ctx, _rx) = route_ctx(ConnectionId::new(101));
16927        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16928
16929        let _ = handler
16930            .handle_client_control_request(
16931                &ctx,
16932                frame,
16933                ClientControlRequest::SupervisorSetEnabled {
16934                    module_id: "broca".to_string(),
16935                    enabled: false,
16936                },
16937                &mut None,
16938            )
16939            .await
16940            .unwrap();
16941
16942        let events = capture
16943            .events()
16944            .into_iter()
16945            .filter(|event| {
16946                event.target == "control"
16947                    && event.fields.get("message").map(String::as_str)
16948                        == Some("supervisor request received")
16949            })
16950            .collect::<Vec<_>>();
16951        assert_eq!(events.len(), 1, "one supervisor request log line");
16952        let event = &events[0];
16953        assert_eq!(event.level, tracing::Level::INFO);
16954        assert_eq!(
16955            event.fields.get("op"),
16956            Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
16957        );
16958        assert_eq!(
16959            event.fields.get("module_id"),
16960            Some(&"\"broca\"".to_string())
16961        );
16962        assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
16963        assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
16964        assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
16965    }
16966
16967    #[tokio::test(flavor = "current_thread")]
16968    async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
16969        let registry = Arc::new(Registry::default());
16970        let module_connection = ConnectionId::new(303);
16971        registry
16972            .register_with_control_ops(
16973                manifest("aft", PROTOCOL_VERSION),
16974                PROTOCOL_VERSION,
16975                module_connection,
16976                module_baseline_control_ops(),
16977            )
16978            .unwrap();
16979        let handler = ControlHandler::new(registry);
16980        let capture = EventCapture::default();
16981        let _subscriber =
16982            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
16983        let (ctx, _rx) = route_ctx(module_connection);
16984        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
16985
16986        let _ = handler
16987            .handle_client_control_request(
16988                &ctx,
16989                frame,
16990                ClientControlRequest::SupervisorSetEnabled {
16991                    module_id: "broca".to_string(),
16992                    enabled: false,
16993                },
16994                &mut None,
16995            )
16996            .await
16997            .unwrap();
16998
16999        let callers = capture
17000            .events()
17001            .into_iter()
17002            .filter(|event| {
17003                event.fields.get("message").map(String::as_str)
17004                    == Some("supervisor request received")
17005            })
17006            .map(|event| event.fields.get("caller").cloned())
17007            .collect::<Vec<_>>();
17008        assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
17009    }
17010
17011    #[tokio::test(flavor = "current_thread")]
17012    async fn supervisor_list_does_not_log_request_received() {
17013        let handler = ControlHandler::new(Arc::new(Registry::default()));
17014        let capture = EventCapture::default();
17015        let _subscriber =
17016            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17017        let (ctx, _rx) = route_ctx(ConnectionId::new(102));
17018        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
17019
17020        handler
17021            .handle_client_control_request(
17022                &ctx,
17023                frame,
17024                ClientControlRequest::SupervisorList {},
17025                &mut None,
17026            )
17027            .await
17028            .unwrap();
17029
17030        assert!(capture.events().into_iter().all(|event| {
17031            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
17032        }));
17033    }
17034
17035    #[tokio::test(flavor = "current_thread")]
17036    async fn supervisor_rescan_preview_does_not_log_request_received() {
17037        let handler = ControlHandler::new(Arc::new(Registry::default()));
17038        let capture = EventCapture::default();
17039        let _subscriber =
17040            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17041        let (ctx, _rx) = route_ctx(ConnectionId::new(103));
17042        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
17043
17044        handler
17045            .handle_client_control_request(
17046                &ctx,
17047                frame,
17048                ClientControlRequest::SupervisorRescan { preview: true },
17049                &mut None,
17050            )
17051            .await
17052            .unwrap();
17053
17054        assert!(capture.events().into_iter().all(|event| {
17055            event.fields.get("message").map(String::as_str) != Some("supervisor request received")
17056        }));
17057    }
17058
17059    struct ApproveOperator;
17060    impl crate::operator_confirm::OperatorProvider for ApproveOperator {
17061        fn prompt(
17062            &self,
17063            _: &str,
17064            _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
17065        ) -> crate::operator_confirm::ProviderResult {
17066            crate::operator_confirm::ProviderResult::Approved
17067        }
17068    }
17069
17070    #[tokio::test]
17071    async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
17072        assert_eq!(
17073            ControlHandler::default().privileged_operator_prompt,
17074            PrivilegedOperatorPrompt::Log
17075        );
17076        for (initial, changed, running) in [
17077            ("log", "enforce", PrivilegedOperatorPrompt::Log),
17078            ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
17079        ] {
17080            let dir = ScratchDir::new("rescan-privileged-operator-prompt");
17081            let path = dir.join("subc.jsonc");
17082            std::fs::write(
17083                &path,
17084                json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
17085            )
17086            .unwrap();
17087            let config = crate::daemon_config::load(&path).unwrap().unwrap();
17088            let registry = Arc::new(Registry::default());
17089            let handle = SupervisorHandle::new();
17090            let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
17091                .with_handle(handle.clone());
17092            let handler = ControlHandler::new(registry)
17093                .with_supervisor(handle)
17094                .with_privileged_operator_prompt(config.privileged_operator_prompt)
17095                .with_supervisor_rescan(supervisor, &path, config.port);
17096            handler.forwarding.operator_confirms().configure(
17097                Arc::new(ApproveOperator),
17098                crate::operator_confirm::OperatorLimits::default(),
17099            );
17100            let (ctx, _rx) = route_ctx(ConnectionId::new(104));
17101
17102            for (disk_mode, needs_restart) in [
17103                (Some(initial), false),
17104                (Some(changed), true),
17105                (Some(changed), true),
17106                (None, running == PrivilegedOperatorPrompt::Enforce),
17107                (Some(initial), false),
17108            ] {
17109                let mut doc = json!({"version":1});
17110                if let Some(mode) = disk_mode {
17111                    doc["privileged_operator_prompt"] = json!(mode);
17112                }
17113                std::fs::write(&path, doc.to_string()).unwrap();
17114                for preview in [true, false] {
17115                    let request = ClientControlRequest::SupervisorRescan { preview };
17116                    let frame = Frame::build(
17117                        FrameType::Request,
17118                        control_flags(),
17119                        0,
17120                        0,
17121                        1,
17122                        serde_json::to_vec(&request).unwrap(),
17123                    )
17124                    .unwrap();
17125                    let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17126                    assert_eq!(frames.len(), 1);
17127                    let ClientControlResponse::SupervisorRescan { result } =
17128                        serde_json::from_slice(&frames[0].body).unwrap()
17129                    else {
17130                        panic!("expected a rescan response");
17131                    };
17132                    let expected = if needs_restart {
17133                        vec!["privileged_operator_prompt"]
17134                    } else {
17135                        Vec::new()
17136                    };
17137                    assert_eq!(result.restart_required, expected);
17138                    assert_eq!(handler.privileged_operator_prompt, running);
17139                    assert_eq!(
17140                        handler.rescan.as_ref().unwrap().privileged_operator_prompt,
17141                        running
17142                    );
17143                }
17144            }
17145        }
17146    }
17147
17148    #[tokio::test]
17149    async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
17150        let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
17151        let path = dir.join("subc.jsonc");
17152        let registry = Arc::new(Registry::default());
17153        let handle = SupervisorHandle::new();
17154        let supervisor =
17155            Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
17156        let handler = ControlHandler::new(registry)
17157            .with_supervisor(handle)
17158            .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
17159            .with_supervisor_rescan(supervisor, &path, None);
17160        handler.forwarding.operator_confirms().configure(
17161            Arc::new(ApproveOperator),
17162            crate::operator_confirm::OperatorLimits::default(),
17163        );
17164        let (ctx, _rx) = route_ctx(ConnectionId::new(105));
17165        for preview in [true, false] {
17166            std::fs::write(
17167                &path,
17168                r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
17169            )
17170            .unwrap();
17171            let request = ClientControlRequest::SupervisorRescan { preview };
17172            let frame = Frame::build(
17173                FrameType::Request,
17174                control_flags(),
17175                0,
17176                0,
17177                1,
17178                serde_json::to_vec(&request).unwrap(),
17179            )
17180            .unwrap();
17181            let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
17182            assert_eq!(frames.len(), 1);
17183            assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
17184            assert_eq!(
17185                handler.privileged_operator_prompt,
17186                PrivilegedOperatorPrompt::Enforce
17187            );
17188        }
17189    }
17190
17191    /// One wire code has several senders, so the refusal line names the check
17192    /// that refused. This drives the shared refusal path for ordinary refusals
17193    /// with an unregistered
17194    /// target and requires the branch label on the event.
17195    #[tokio::test]
17196    async fn route_open_refusal_names_the_check_that_refused() {
17197        let handler = ControlHandler::new(Arc::new(Registry::default()));
17198        let capture = EventCapture::default();
17199        let _subscriber =
17200            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17201        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17202        let response = handler
17203            .handle_control_frame(
17204                &ctx,
17205                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
17206            )
17207            .await
17208            .unwrap();
17209
17210        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17211        let event = capture
17212            .events()
17213            .into_iter()
17214            .find(|event| {
17215                event.target == "control"
17216                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17217            })
17218            .expect("route.open refusal event");
17219        assert_eq!(
17220            event.fields.get("reason"),
17221            Some(&"\"not_registered\"".to_string())
17222        );
17223    }
17224
17225    #[tokio::test]
17226    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
17227        let registry = Arc::new(Registry::default());
17228        let supervisor_handle = SupervisorHandle::new();
17229        let supervisor =
17230            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
17231                .with_handle(supervisor_handle.clone())
17232                .with_connection_file_path(std::env::temp_dir().join(format!(
17233                    "subc-route-open-refusal-info-{}",
17234                    std::process::id()
17235                )));
17236        let module = supervisor
17237            .supervise_configured(
17238                ModuleSpec {
17239                    script: None,
17240                    carry_containment: false,
17241                    module_id: "warming".to_string(),
17242                    program: fake_aft_stub_path(),
17243                    args: Vec::new(),
17244                    env: Vec::new(),
17245                    reserved: false,
17246                    reserved_prefixes: Vec::new(),
17247                    protocol: ModuleProtocol::Subc,
17248                    overlap: Default::default(),
17249                },
17250                true,
17251            )
17252            .unwrap();
17253        assert_eq!(module.state().unwrap(), ModuleState::Running);
17254
17255        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17256        assert!(handler
17257            .counters()
17258            .snapshot()
17259            .get("route_open_refused_by_code")
17260            .is_none());
17261        let capture = EventCapture::default();
17262        let _subscriber =
17263            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17264        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
17265        let response = handler
17266            .handle_control_frame(
17267                &ctx,
17268                route_open_frame(394, "warming", unique_project_root("refusal-info")),
17269            )
17270            .await
17271            .unwrap();
17272        module.stop().await.unwrap();
17273
17274        assert_eq!(parse_error(&response[0])["code"], "module_warming");
17275        let event = capture
17276            .events()
17277            .into_iter()
17278            .find(|event| {
17279                event.target == "control"
17280                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
17281            })
17282            .expect("route.open refusal event");
17283        assert_eq!(
17284            event.fields.get("module_id"),
17285            Some(&"\"warming\"".to_string())
17286        );
17287        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
17288        assert_eq!(
17289            event.fields.get("reason"),
17290            Some(&"\"supervised_not_registered\"".to_string())
17291        );
17292        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
17293        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
17294        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
17295        assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
17296        assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
17297        assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
17298        assert_eq!(
17299            handler.counters().snapshot()["route_open_refused_by_code"],
17300            json!({ "module_warming": 1 })
17301        );
17302    }
17303
17304    const OUTAGE_START: &str = "route.open refusing module: not serving";
17305    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
17306
17307    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
17308        capture
17309            .events()
17310            .into_iter()
17311            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
17312            .collect()
17313    }
17314
17315    fn supervise_stub(
17316        registry: &Arc<Registry>,
17317        module_id: &str,
17318        enabled: bool,
17319    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
17320        let supervisor_handle = SupervisorHandle::new();
17321        let supervisor =
17322            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
17323                .with_handle(supervisor_handle.clone())
17324                .with_connection_file_path(std::env::temp_dir().join(format!(
17325                    "subc-route-outage-{module_id}-{}",
17326                    std::process::id()
17327                )));
17328        let module = supervisor
17329            .supervise_configured(
17330                ModuleSpec {
17331                    script: None,
17332                    carry_containment: false,
17333                    module_id: module_id.to_string(),
17334                    program: fake_aft_stub_path(),
17335                    args: Vec::new(),
17336                    env: Vec::new(),
17337                    reserved: false,
17338                    reserved_prefixes: Vec::new(),
17339                    protocol: ModuleProtocol::Subc,
17340                    overlap: Default::default(),
17341                },
17342                enabled,
17343            )
17344            .unwrap();
17345        (supervisor_handle, module)
17346    }
17347
17348    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
17349        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
17350            module_id: module_id.to_string(),
17351            drain_timeout_ms: Some(50),
17352        })
17353        .unwrap();
17354        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
17355    }
17356
17357    /// Two handlers built over one forwarding table must share one outage
17358    /// tracker; separate trackers would each log their own opening line for
17359    /// the same outage.
17360    #[test]
17361    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
17362        let registry = Arc::new(Registry::default());
17363        let forwarding = Arc::new(ForwardingTable::default());
17364        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17365        let second = ControlHandler::with_forwarding(registry, forwarding);
17366        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
17367    }
17368
17369    /// A client can name any module id it likes. Refusing an unknown one,
17370    /// however often, must not create outage state or outage lines, or the
17371    /// tracker would be a memory sink any client could fill.
17372    #[tokio::test(flavor = "current_thread")]
17373    async fn route_open_unknown_module_refusals_add_no_outage_state() {
17374        let handler = ControlHandler::new(Arc::new(Registry::default()));
17375        let capture = EventCapture::default();
17376        let _subscriber =
17377            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17378        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
17379        for corr in 0..8 {
17380            let response = handler
17381                .handle_control_frame(
17382                    &ctx,
17383                    route_open_frame(
17384                        380 + corr,
17385                        &format!("nobody-{corr}"),
17386                        unique_project_root("outage-unknown"),
17387                    ),
17388                )
17389                .await
17390                .unwrap();
17391            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17392        }
17393
17394        assert_eq!(handler.route_outages.tracked_module_count(), 0);
17395        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
17396        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
17397    }
17398
17399    /// Drives the refusal path end to end: a supervised module that served
17400    /// before and stopped being registered with no instruction to stop is a
17401    /// WARN, and the same module refused after an operator `supervisor.restart`
17402    /// is an INFO.
17403    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17404    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
17405        let registry = Arc::new(Registry::default());
17406        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
17407        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17408        let capture = EventCapture::default();
17409        let _subscriber =
17410            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17411        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
17412        // The stub never registers, so pretend it served once: otherwise every
17413        // refusal would fall in its startup window.
17414        handler.route_outages.record_accepted("outage-restart");
17415
17416        let response = handler
17417            .handle_control_frame(
17418                &ctx,
17419                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
17420            )
17421            .await
17422            .unwrap();
17423        assert_eq!(response[0].header.ty, FrameType::Error);
17424        let starts = outage_lines(&capture, OUTAGE_START);
17425        assert_eq!(starts.len(), 1, "{starts:?}");
17426        assert_eq!(starts[0].level, tracing::Level::WARN);
17427        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
17428        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
17429        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
17430        handler.route_outages.record_accepted("outage-restart");
17431        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
17432
17433        let restart = handler
17434            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
17435            .await
17436            .unwrap();
17437        assert_eq!(
17438            restart[0].header.ty,
17439            FrameType::Response,
17440            "{:?}",
17441            parse_error(&restart[0])
17442        );
17443        handler
17444            .handle_control_frame(
17445                &ctx,
17446                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
17447            )
17448            .await
17449            .unwrap();
17450        module.stop().await.unwrap();
17451
17452        let starts = outage_lines(&capture, OUTAGE_START);
17453        assert_eq!(starts.len(), 2, "{starts:?}");
17454        assert_eq!(starts[1].level, tracing::Level::INFO);
17455        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
17456    }
17457
17458    /// A restart refused before it touched the module (here: the module is
17459    /// disabled) must clear its operator mark, so the next real outage is
17460    /// still reported as a warning.
17461    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17462    async fn failed_operator_restart_leaves_no_operator_mark() {
17463        let registry = Arc::new(Registry::default());
17464        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
17465        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17466        let capture = EventCapture::default();
17467        let _subscriber =
17468            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17469        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
17470        handler.route_outages.record_accepted("outage-disabled");
17471
17472        let restart = handler
17473            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
17474            .await
17475            .unwrap();
17476        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
17477        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
17478
17479        handler
17480            .handle_control_frame(
17481                &ctx,
17482                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
17483            )
17484            .await
17485            .unwrap();
17486        let starts = outage_lines(&capture, OUTAGE_START);
17487        assert_eq!(starts.len(), 1, "{starts:?}");
17488        assert_eq!(starts[0].level, tracing::Level::WARN);
17489    }
17490
17491    #[tokio::test(flavor = "current_thread")]
17492    async fn route_open_unknown_module_escapes_target_module_id() {
17493        let handler = ControlHandler::new(Arc::new(Registry::default()));
17494        let capture = EventCapture::default();
17495        let _subscriber =
17496            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17497        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
17498        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17499        let response = handler
17500            .handle_control_frame(
17501                &ctx,
17502                route_open_frame(
17503                    395,
17504                    hostile_module_id,
17505                    unique_project_root("hostile-target-module-id"),
17506                ),
17507            )
17508            .await
17509            .unwrap();
17510
17511        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17512        let event = capture
17513            .events()
17514            .into_iter()
17515            .find(|event| {
17516                event.target == "control"
17517                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
17518            })
17519            .expect("route.open unknown-module refusal event");
17520        let logged = event.fields.get("module_id").expect("module_id field");
17521        assert!(!logged.bytes().any(|byte| byte < 0x20));
17522        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17523    }
17524
17525    /// A route.open refusal is the daemon's own answer, so its ERROR carries
17526    /// the daemon-origin bit on the frame the client receives.
17527    #[tokio::test(flavor = "current_thread")]
17528    async fn route_open_refusal_carries_daemon_origin() {
17529        let handler = ControlHandler::new(Arc::new(Registry::default()));
17530        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
17531        let response = handler
17532            .handle_control_frame(
17533                &ctx,
17534                route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
17535            )
17536            .await
17537            .unwrap();
17538
17539        assert_eq!(response.len(), 1);
17540        assert_eq!(response[0].header.ty, FrameType::Error);
17541        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
17542        assert!(
17543            response[0].header.flags.is_daemon_origin(),
17544            "route.open refusal must carry daemon origin: {:?}",
17545            response[0].header.flags
17546        );
17547    }
17548
17549    /// The drain `module_reloading` refusal is daemon-synthesized for an
17550    /// endpoint that is reloading, so it carries the daemon-origin bit on the
17551    /// frame the client receives.
17552    #[tokio::test(flavor = "current_thread")]
17553    async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
17554        let registry = Arc::new(Registry::default());
17555        let forwarding = Arc::new(ForwardingTable::default());
17556        let handler =
17557            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17558        let module_connection = ConnectionId::new(95);
17559        let (module_ctx, mut module_rx) = route_ctx(module_connection);
17560        hello_via_sink(
17561            &handler,
17562            &module_ctx,
17563            &mut module_rx,
17564            hello_frame("aft", PROTOCOL_VERSION, 395),
17565        )
17566        .await;
17567        forwarding
17568            .begin_module_drain("aft", RouteCloseReason::Restart)
17569            .unwrap()
17570            .expect("the module endpoint drains");
17571
17572        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
17573        let response = handler
17574            .handle_control_frame(
17575                &client_ctx,
17576                route_open_frame(396, "aft", unique_project_root("drain-origin")),
17577            )
17578            .await
17579            .unwrap();
17580
17581        assert_eq!(response.len(), 1);
17582        assert_eq!(response[0].header.ty, FrameType::Error);
17583        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17584        assert!(
17585            response[0].header.flags.is_daemon_origin(),
17586            "drain module_reloading must carry daemon origin: {:?}",
17587            response[0].header.flags
17588        );
17589    }
17590
17591    /// A module that refuses a bind with a daemon code still cannot mint a
17592    /// daemon refusal: the relayed `module_reloading` body reaches the client
17593    /// with the daemon-origin bit clear.
17594    #[tokio::test(flavor = "current_thread")]
17595    async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
17596        let registry = Arc::new(Registry::default());
17597        let forwarding = Arc::new(ForwardingTable::default());
17598        let handler =
17599            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17600        let module_connection = ConnectionId::new(95);
17601        let (module_ctx, mut module_rx) = route_ctx(module_connection);
17602        hello_via_sink(
17603            &handler,
17604            &module_ctx,
17605            &mut module_rx,
17606            hello_frame("aft", PROTOCOL_VERSION, 395),
17607        )
17608        .await;
17609
17610        let client_connection = ConnectionId::new(96);
17611        let (client_ctx, _client_rx) = route_ctx(client_connection);
17612        let (route_task, bind) = relay_route_open(
17613            &handler,
17614            client_connection,
17615            &client_ctx.egress,
17616            &mut module_rx,
17617            396,
17618            "aft",
17619            "spoofed-reloading",
17620        )
17621        .await;
17622        // The module sets the origin bit on its own rejection too: neither
17623        // the bit nor the daemon code may survive the relay as daemon-owned.
17624        let rejection = Frame::build(
17625            FrameType::Error,
17626            crate::router::daemon_error_flags(),
17627            0,
17628            0,
17629            bind.header.corr,
17630            serde_json::to_vec(&ErrorBody::new(
17631                "module_reloading",
17632                "module refused route.bind",
17633            ))
17634            .unwrap(),
17635        )
17636        .unwrap();
17637        handler
17638            .handle_control_frame(&module_ctx, rejection)
17639            .await
17640            .unwrap();
17641
17642        let response = route_task.await.unwrap();
17643        assert_eq!(response.len(), 1);
17644        assert_eq!(response[0].header.ty, FrameType::Error);
17645        assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17646        assert!(
17647            !response[0].header.flags.is_daemon_origin(),
17648            "a module's own rejection must not carry daemon origin: {:?}",
17649            response[0].header.flags
17650        );
17651    }
17652
17653    #[tokio::test(flavor = "current_thread")]
17654    async fn route_open_module_rejection_uses_daemon_counter_key() {
17655        let registry = Arc::new(Registry::default());
17656        let forwarding = Arc::new(ForwardingTable::default());
17657        let handler =
17658            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17659        let module_connection = ConnectionId::new(95);
17660        let (module_ctx, mut module_rx) = route_ctx(module_connection);
17661        hello_via_sink(
17662            &handler,
17663            &module_ctx,
17664            &mut module_rx,
17665            hello_frame("aft", PROTOCOL_VERSION, 395),
17666        )
17667        .await;
17668
17669        let client_connection = ConnectionId::new(96);
17670        let (client_ctx, _client_rx) = route_ctx(client_connection);
17671        let capture = EventCapture::default();
17672        // tracing-core decides once per callsite whether its events are enabled,
17673        // and caches the answer. If a parallel test with no subscriber reaches
17674        // this callsite first, the cached answer is "disabled" and this test's
17675        // capture sees nothing. Keeping a second dispatch alive makes tracing-core
17676        // ask every live subscriber instead, while only this test's default
17677        // receives the events.
17678        static INTEREST_ANCHOR: std::sync::OnceLock<tracing::Dispatch> = std::sync::OnceLock::new();
17679        INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
17680        let _subscriber =
17681            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
17682        let (route_task, bind) = relay_route_open(
17683            &handler,
17684            client_connection,
17685            &client_ctx.egress,
17686            &mut module_rx,
17687            396,
17688            "aft",
17689            "hostile-module-code",
17690        )
17691        .await;
17692        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
17693        let rejection = Frame::build(
17694            FrameType::Error,
17695            control_flags(),
17696            0,
17697            0,
17698            bind.header.corr,
17699            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
17700        )
17701        .unwrap();
17702        handler
17703            .handle_control_frame(&module_ctx, rejection)
17704            .await
17705            .unwrap();
17706
17707        let response = route_task.await.unwrap();
17708        assert_eq!(parse_error(&response[0])["code"], hostile_code);
17709        let counters = handler.counters().snapshot();
17710        assert_eq!(
17711            counters["route_open_refused_by_code"],
17712            json!({ "module_rejected": 1 })
17713        );
17714        assert!(counters["route_open_refused_by_code"]
17715            .get(hostile_code)
17716            .is_none());
17717
17718        let event = capture
17719            .events()
17720            .into_iter()
17721            .find(|event| {
17722                event.target == "control"
17723                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
17724            })
17725            .expect("route.open module-rejection refusal event");
17726        let logged = event.fields.get("module_code").expect("module_code field");
17727        assert!(!logged.bytes().any(|byte| byte < 0x20));
17728        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
17729    }
17730
17731    #[tokio::test]
17732    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
17733        let registry = Arc::new(Registry::default());
17734        let supervisor_handle = SupervisorHandle::new();
17735        let missing_program = std::env::temp_dir().join(format!(
17736            "subc-route-open-missing-program-{}",
17737            std::process::id()
17738        ));
17739        let supervisor =
17740            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
17741                .with_handle(supervisor_handle.clone());
17742        let module = supervisor
17743            .supervise_configured(
17744                ModuleSpec {
17745                    script: None,
17746                    carry_containment: false,
17747                    module_id: "failed".to_string(),
17748                    program: missing_program,
17749                    args: Vec::new(),
17750                    env: Vec::new(),
17751                    reserved: false,
17752                    reserved_prefixes: Vec::new(),
17753                    protocol: ModuleProtocol::Subc,
17754                    overlap: Default::default(),
17755                },
17756                true,
17757            )
17758            .unwrap();
17759        assert_eq!(module.state().unwrap(), ModuleState::Failed);
17760
17761        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
17762        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
17763        let response = handler
17764            .handle_control_frame(
17765                &ctx,
17766                route_open_frame(305, "failed", unique_project_root("failed")),
17767            )
17768            .await
17769            .unwrap();
17770
17771        assert_eq!(response[0].header.ty, FrameType::Error);
17772        let error = parse_error(&response[0]);
17773        assert_eq!(error["code"], "target_unavailable");
17774        assert!(error["message"]
17775            .as_str()
17776            .unwrap()
17777            .contains("state=failed, enabled=true, live=false"));
17778    }
17779
17780    #[tokio::test]
17781    async fn route_open_role_mismatch_remains_target_unavailable() {
17782        let registry = Arc::new(Registry::default());
17783        let handler = ControlHandler::new(Arc::clone(&registry));
17784        handler
17785            .handle_control(
17786                ConnectionId::new(41),
17787                non_routable_hello_frame_with_control_ops("health-only", 306, None),
17788            )
17789            .unwrap();
17790
17791        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
17792        let response = handler
17793            .handle_control_frame(
17794                &ctx,
17795                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
17796            )
17797            .await
17798            .unwrap();
17799
17800        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17801        assert!(parse_error(&response[0])["message"]
17802            .as_str()
17803            .unwrap()
17804            .contains("does not provide the requested target"));
17805    }
17806
17807    #[tokio::test]
17808    async fn route_open_inactive_registration_remains_target_unavailable() {
17809        let registry = Arc::new(Registry::default());
17810        let handler = ControlHandler::new(Arc::clone(&registry));
17811        handler
17812            .handle_control(
17813                ConnectionId::new(43),
17814                hello_frame("inactive", PROTOCOL_VERSION, 308),
17815            )
17816            .unwrap();
17817        assert!(registry
17818            .set_module_state_for_test("inactive", ChannelState::Closed)
17819            .unwrap());
17820
17821        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
17822        let response = handler
17823            .handle_control_frame(
17824                &ctx,
17825                route_open_frame(309, "inactive", unique_project_root("inactive")),
17826            )
17827            .await
17828            .unwrap();
17829
17830        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
17831        assert!(parse_error(&response[0])["message"]
17832            .as_str()
17833            .unwrap()
17834            .contains("is not active"));
17835    }
17836
17837    #[tokio::test]
17838    async fn late_health_reply_is_recorded_through_the_module_response_path() {
17839        let registry = Arc::new(Registry::default());
17840        let forwarding = Arc::new(ForwardingTable::default());
17841        let supervisor_handle = SupervisorHandle::new();
17842        let supervisor =
17843            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
17844                .with_forwarding(Arc::clone(&forwarding))
17845                .with_handle(supervisor_handle.clone());
17846        let module = supervisor
17847            .supervise_configured(
17848                crate::ModuleSpec {
17849                    script: None,
17850                    carry_containment: false,
17851                    module_id: "late-health-response".to_string(),
17852                    program: PathBuf::from("disabled-module"),
17853                    args: Vec::new(),
17854                    env: Vec::new(),
17855                    reserved: false,
17856                    reserved_prefixes: Vec::new(),
17857                    protocol: ModuleProtocol::Subc,
17858                    overlap: Default::default(),
17859                },
17860                false,
17861            )
17862            .unwrap();
17863        let handler =
17864            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17865                .with_supervisor(supervisor_handle);
17866        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
17867        handler
17868            .handle_control_frame(
17869                &module_ctx,
17870                hello_frame_with_control_ops(
17871                    "late-health-response",
17872                    PROTOCOL_VERSION,
17873                    7,
17874                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17875                ),
17876            )
17877            .await
17878            .unwrap();
17879        let probe_started_at = Instant::now() - Duration::from_millis(80);
17880        let pending = forwarding
17881            .begin_health_probe_rpc_for(
17882                "late-health-response",
17883                MODULE_CONTROL_OP_HEALTH_CHECK,
17884                probe_started_at,
17885                Instant::now() - Duration::from_millis(1),
17886            )
17887            .unwrap();
17888        assert!(forwarding
17889            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
17890            .unwrap());
17891
17892        let responses = handler
17893            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
17894            .await
17895            .unwrap();
17896
17897        assert!(responses.is_empty());
17898        let health = module.status().unwrap().health;
17899        assert_eq!(health.late_answer_count, 1);
17900        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
17901    }
17902
17903    #[tokio::test]
17904    async fn health_probe_timeout_and_module_death_are_typed() {
17905        let registry = Arc::new(Registry::default());
17906        let forwarding = Arc::new(ForwardingTable::default());
17907        let handler =
17908            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
17909                .with_health_probe_timeout(Duration::from_millis(50));
17910        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
17911        hello_via_sink(
17912            &handler,
17913            &module_ctx,
17914            &mut module_rx,
17915            hello_frame_with_control_ops(
17916                "aft",
17917                PROTOCOL_VERSION,
17918                7,
17919                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
17920            ),
17921        )
17922        .await;
17923
17924        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
17925        let responses = handler
17926            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
17927            .await
17928            .unwrap();
17929        assert_eq!(responses[0].header.ty, FrameType::Error);
17930        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
17931        let _ = module_rx.try_recv();
17932
17933        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
17934        let health_handler = handler.clone();
17935        let death_task = tokio::spawn(async move {
17936            health_handler
17937                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
17938                .await
17939                .unwrap()
17940        });
17941        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
17942            .await
17943            .unwrap()
17944            .unwrap();
17945        handler
17946            .cleanup_connection(module_ctx.connection_id)
17947            .unwrap();
17948        let responses = death_task.await.unwrap();
17949        assert_eq!(responses[0].header.ty, FrameType::Error);
17950        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
17951    }
17952
17953    #[test]
17954    fn hello_requires_exact_protocol_version() {
17955        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
17956            let registry = Arc::new(Registry::default());
17957            let handler = ControlHandler::new(Arc::clone(&registry));
17958            let responses = handler
17959                .handle_control(
17960                    ConnectionId::new(connection),
17961                    hello_frame("aft", offered, 9),
17962                )
17963                .unwrap();
17964
17965            assert_eq!(responses.len(), 1);
17966            assert_eq!(responses[0].header.ty, FrameType::Error);
17967            let error = parse_error(&responses[0]);
17968            assert_eq!(error["code"], "version_unsupported");
17969            assert!(registry.get_module("aft").unwrap().is_none());
17970            assert_eq!(registry.active_registration_count().unwrap(), 0);
17971        }
17972    }
17973
17974    #[test]
17975    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
17976        let registry = Arc::new(Registry::default());
17977        let forwarding = Arc::new(ForwardingTable::default());
17978        let handler =
17979            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
17980        let module_connection = ConnectionId::new(301);
17981        let registration = registry
17982            .register_with_control_ops(
17983                manifest("aft-push", PROTOCOL_VERSION),
17984                PROTOCOL_VERSION,
17985                module_connection,
17986                module_baseline_control_ops(),
17987            )
17988            .unwrap();
17989        let (module_tx, _module_rx) = mpsc::channel(8);
17990        let endpoint = forwarding
17991            .register_module_connection(
17992                module_connection,
17993                "aft-push".to_string(),
17994                PROTOCOL_VERSION,
17995                manifest_concurrency(&registration.manifest),
17996                FrameSink::new(module_tx),
17997            )
17998            .unwrap();
17999
18000        // A push op this version does not know is ignored (forward-compat), not errored.
18001        let unknown = Frame::build(
18002            FrameType::Push,
18003            control_flags(),
18004            0,
18005            0,
18006            5,
18007            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
18008        )
18009        .unwrap();
18010        let out = handler.handle_status_update(endpoint, unknown).unwrap();
18011        assert!(
18012            out.is_empty(),
18013            "unknown push op must be ignored, got {out:?}"
18014        );
18015
18016        // A malformed body for a KNOWN op is a real error worth surfacing.
18017        let malformed = Frame::build(
18018            FrameType::Push,
18019            control_flags(),
18020            0,
18021            0,
18022            6,
18023            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
18024        )
18025        .unwrap();
18026        let out = handler.handle_status_update(endpoint, malformed).unwrap();
18027        assert_eq!(out.len(), 1);
18028        assert_eq!(out[0].header.ty, FrameType::Error);
18029        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
18030    }
18031
18032    #[test]
18033    fn hello_rejected_when_connection_already_owns_client_routes() {
18034        let registry = Arc::new(Registry::default());
18035        let forwarding = Arc::new(ForwardingTable::default());
18036        let handler =
18037            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
18038        // Commits a client route on connection 202 (bound to a module on conn 101).
18039        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
18040        let client_connection = ConnectionId::new(202);
18041
18042        // That same connection now tries to register as a module: rejected, so one
18043        // connection never holds both client-route and module-endpoint state.
18044        let responses = handler
18045            .handle_control(
18046                client_connection,
18047                hello_frame("aft-second", PROTOCOL_VERSION, 9),
18048            )
18049            .unwrap();
18050        assert_eq!(responses[0].header.ty, FrameType::Error);
18051        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
18052        assert!(registry.get_module("aft-second").unwrap().is_none());
18053    }
18054
18055    #[tokio::test]
18056    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
18057        let registry = Arc::new(Registry::default());
18058        let forwarding = Arc::new(ForwardingTable::default());
18059        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
18060        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
18061        hello_via_sink(
18062            &handler,
18063            &module_ctx,
18064            &mut module_rx,
18065            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
18066        )
18067        .await;
18068        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
18069        let pending = forwarding
18070            .begin_route_bind_relay_for_test(
18071                client_ctx.connection_id,
18072                client_ctx.egress.clone(),
18073                2,
18074                "alpha",
18075            )
18076            .unwrap();
18077        forwarding
18078            .complete_pending_relay(
18079                module_ctx.connection_id,
18080                pending.corr,
18081                RouteBindRelayOutcome::Accepted,
18082            )
18083            .unwrap();
18084        client_rx.try_recv().unwrap();
18085        for module_id in ["beta", "alpha"] {
18086            let replies = handler
18087                .handle_control_frame(
18088                    &module_ctx,
18089                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
18090                )
18091                .await
18092                .unwrap();
18093            assert_eq!(replies.len(), 1, "second HELLO must be refused");
18094            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
18095        }
18096        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
18097        assert!(registry.get_module("beta").unwrap().is_none());
18098        assert!(matches!(
18099            forwarding
18100                .lookup_data_route(
18101                    client_ctx.connection_id,
18102                    pending.client_channel,
18103                    pending.client_epoch,
18104                )
18105                .unwrap(),
18106            DataRoute::Client(DataRouteState::Bound(_))
18107        ));
18108        assert!(handler
18109            .hello_launch_nonces
18110            .lock()
18111            .unwrap()
18112            .presented(module_ctx.connection_id, Some("alpha-nonce")));
18113        assert!(module_rx.try_recv().is_err());
18114    }
18115
18116    #[test]
18117    fn reserved_module_hello_requires_matching_launch_nonce() {
18118        let registry = Arc::new(Registry::default());
18119        let supervisor = SupervisorHandle::new();
18120        // The supervisor recorded the nonce it injected when it spawned the reserved
18121        // module; the HELLO verifier checks against the same shared handle.
18122        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
18123        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
18124
18125        // A HELLO with NO nonce is rejected.
18126        let no_nonce = handler
18127            .handle_control(
18128                ConnectionId::new(1),
18129                hello_frame("vault", PROTOCOL_VERSION, 1),
18130            )
18131            .unwrap();
18132        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
18133        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
18134        assert!(registry.get_module("vault").unwrap().is_none());
18135
18136        // A HELLO with the WRONG nonce is rejected.
18137        let wrong = handler
18138            .handle_control(
18139                ConnectionId::new(2),
18140                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
18141            )
18142            .unwrap();
18143        assert_eq!(wrong[0].header.ty, FrameType::Error);
18144        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
18145        assert!(registry.get_module("vault").unwrap().is_none());
18146
18147        // A HELLO with the CORRECT nonce registers.
18148        let ok = handler
18149            .handle_control(
18150                ConnectionId::new(3),
18151                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
18152            )
18153            .unwrap();
18154        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
18155        assert!(registry.get_module("vault").unwrap().is_some());
18156    }
18157
18158    #[test]
18159    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
18160        let registry = Arc::new(Registry::default());
18161        let supervisor = SupervisorHandle::new();
18162        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18163        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18164        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
18165
18166        let squat = handler
18167            .handle_control(
18168                ConnectionId::new(1),
18169                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
18170            )
18171            .unwrap();
18172        assert_eq!(squat[0].header.ty, FrameType::Error);
18173        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
18174        assert!(parse_error(&squat[0])["message"]
18175            .as_str()
18176            .unwrap()
18177            .contains("fed:"));
18178
18179        let accepted_peer = handler
18180            .handle_control(
18181                ConnectionId::new(2),
18182                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
18183            )
18184            .unwrap();
18185        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
18186
18187        let accepted_short = handler
18188            .handle_control(
18189                ConnectionId::new(3),
18190                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
18191            )
18192            .unwrap();
18193        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
18194
18195        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
18196            let response = handler
18197                .handle_control(
18198                    ConnectionId::new(conn),
18199                    hello_frame(module_id, PROTOCOL_VERSION, conn),
18200                )
18201                .unwrap();
18202            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
18203        }
18204    }
18205
18206    #[test]
18207    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
18208        let registry = Arc::new(Registry::default());
18209        let supervisor = SupervisorHandle::new();
18210        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
18211        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
18212        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
18213        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
18214
18215        let owner_nonce = handler
18216            .handle_control(
18217                ConnectionId::new(1),
18218                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
18219            )
18220            .unwrap();
18221        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
18222        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
18223        assert!(registry.get_module("fed:special").unwrap().is_none());
18224
18225        let exact_nonce = handler
18226            .handle_control(
18227                ConnectionId::new(2),
18228                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
18229            )
18230            .unwrap();
18231        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
18232        assert!(registry.get_module("fed:special").unwrap().is_some());
18233    }
18234
18235    #[test]
18236    fn non_reserved_module_ignores_launch_nonce() {
18237        let registry = Arc::new(Registry::default());
18238        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
18239        // registration succeeds whether a spawned process echoes a nonce or not.
18240        let handler = ControlHandler::new(Arc::clone(&registry));
18241        let no_nonce = handler
18242            .handle_control(
18243                ConnectionId::new(1),
18244                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
18245            )
18246            .unwrap();
18247        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
18248        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
18249
18250        let echoed_nonce = handler
18251            .handle_control(
18252                ConnectionId::new(2),
18253                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
18254            )
18255            .unwrap();
18256        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
18257        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
18258    }
18259
18260    #[test]
18261    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
18262        let handler = ControlHandler::default();
18263        let conn = ConnectionId::new(1);
18264        let malformed = Frame::build(
18265            FrameType::Hello,
18266            control_flags(),
18267            0,
18268            0,
18269            3,
18270            b"{not json".to_vec(),
18271        )
18272        .unwrap();
18273
18274        let error = handler.handle_control(conn, malformed).unwrap();
18275        assert_eq!(error[0].header.ty, FrameType::Error);
18276        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
18277
18278        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18279        let pong = handler.handle_control(conn, ping).unwrap();
18280        assert_eq!(pong[0].header.ty, FrameType::Pong);
18281        assert_eq!(pong[0].header.corr, 4);
18282    }
18283
18284    #[test]
18285    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
18286        let registry = Arc::new(Registry::default());
18287        let handler = ControlHandler::new(Arc::clone(&registry));
18288
18289        handler
18290            .handle_control(
18291                ConnectionId::new(1),
18292                hello_frame("aft", PROTOCOL_VERSION, 1),
18293            )
18294            .unwrap();
18295        let duplicate = handler
18296            .handle_control(
18297                ConnectionId::new(2),
18298                hello_frame("aft", PROTOCOL_VERSION, 2),
18299            )
18300            .unwrap();
18301
18302        assert_eq!(duplicate[0].header.ty, FrameType::Error);
18303        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
18304        let registration = registry.get_module("aft").unwrap().unwrap();
18305        assert_eq!(registration.connection_id, ConnectionId::new(1));
18306    }
18307
18308    #[test]
18309    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
18310        let registry = Arc::new(Registry::default());
18311        let forwarding = Arc::new(ForwardingTable::default());
18312        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
18313        let handler =
18314            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
18315                .with_process_liveness(process_liveness);
18316        let (ctx, route_channel, route_epoch) =
18317            bind_liveness_route(&registry, &forwarding, "aft-dead");
18318        let responses = handler
18319            .handle_route_poll(
18320                &ctx,
18321                route_poll_frame(41, PollKind::Liveness, route_channel),
18322                route_channel,
18323                route_epoch,
18324                PollKind::Liveness,
18325            )
18326            .unwrap();
18327
18328        assert_eq!(responses.len(), 1);
18329        assert_eq!(responses[0].header.ty, FrameType::Response);
18330        assert_route_poll_liveness(&responses[0], false);
18331    }
18332
18333    #[test]
18334    fn liveness_poll_without_process_source_uses_bound_route() {
18335        let registry = Arc::new(Registry::default());
18336        let forwarding = Arc::new(ForwardingTable::default());
18337        let handler =
18338            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
18339        let (ctx, route_channel, route_epoch) =
18340            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
18341        let responses = handler
18342            .handle_route_poll(
18343                &ctx,
18344                route_poll_frame(42, PollKind::Liveness, route_channel),
18345                route_channel,
18346                route_epoch,
18347                PollKind::Liveness,
18348            )
18349            .unwrap();
18350
18351        assert_route_poll_liveness(&responses[0], true);
18352    }
18353
18354    #[test]
18355    fn liveness_poll_untracked_process_source_uses_bound_route() {
18356        let registry = Arc::new(Registry::default());
18357        let forwarding = Arc::new(ForwardingTable::default());
18358        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
18359        let handler =
18360            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
18361                .with_process_liveness(process_liveness);
18362        let (ctx, route_channel, route_epoch) =
18363            bind_liveness_route(&registry, &forwarding, "aft-untracked");
18364        let responses = handler
18365            .handle_route_poll(
18366                &ctx,
18367                route_poll_frame(43, PollKind::Liveness, route_channel),
18368                route_channel,
18369                route_epoch,
18370                PollKind::Liveness,
18371            )
18372            .unwrap();
18373
18374        assert_route_poll_liveness(&responses[0], true);
18375    }
18376
18377    #[tokio::test]
18378    async fn unknown_op_returns_unknown_control_op() {
18379        let handler = ControlHandler::default();
18380        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
18381        let request = Frame::build(
18382            FrameType::Request,
18383            control_flags(),
18384            0,
18385            0,
18386            55,
18387            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
18388        )
18389        .unwrap();
18390
18391        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18392
18393        assert_eq!(response.len(), 1);
18394        assert_eq!(response[0].header.ty, FrameType::Error);
18395        assert_eq!(response[0].header.corr, 55);
18396        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
18397    }
18398
18399    #[tokio::test]
18400    async fn supervisor_provenance_rejects_unknown_exact_module() {
18401        let handler = ControlHandler::default();
18402        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
18403        let request = Frame::build(
18404            FrameType::Request,
18405            control_flags(),
18406            0,
18407            0,
18408            57,
18409            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
18410        )
18411        .unwrap();
18412
18413        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18414
18415        assert_eq!(response.len(), 1);
18416        assert_eq!(response[0].header.ty, FrameType::Error);
18417        assert_eq!(response[0].header.corr, 57);
18418        let error = parse_error(&response[0]);
18419        assert_eq!(error["code"], "unknown_module");
18420        assert_eq!(error["message"], "module_id 'missing' is not supervised");
18421    }
18422
18423    #[test]
18424    fn provenance_probe_override_keeps_handler_tests_deterministic() {
18425        let expected = subc_control::RunningImageAgreement::Unavailable {
18426            reason: subc_control::RunningImageUnavailableReason::HashFailed,
18427        };
18428        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
18429        assert_eq!(handler.provenance_probe_override, Some(expected));
18430    }
18431
18432    #[test]
18433    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
18434        let verdict = reload_verdict(
18435            std::path::Path::new("/bin/new"),
18436            Some(std::path::Path::new("/bin/old")),
18437            subc_control::RunningImageAgreement::Unavailable {
18438                reason: subc_control::RunningImageUnavailableReason::HashFailed,
18439            },
18440        );
18441        assert!(matches!(
18442            verdict.path,
18443            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
18444                if configured == std::path::Path::new("/bin/new")
18445                    && spawned_from == std::path::Path::new("/bin/old")
18446        ));
18447    }
18448
18449    #[test]
18450    fn launch_spec_without_running_digest_is_unavailable_not_match() {
18451        use subc_control::{
18452            LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
18453        };
18454        let configured = crate::supervise::ModuleSpec {
18455            script: None,
18456            carry_containment: false,
18457            module_id: "legacy".into(),
18458            program: "/module".into(),
18459            args: Vec::new(),
18460            env: Vec::new(),
18461            reserved: false,
18462            reserved_prefixes: Vec::new(),
18463            protocol: subc_control::ModuleProtocol::Subc,
18464            overlap: Default::default(),
18465        };
18466        let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
18467        assert_eq!(
18468            launch_spec_agreement(&configured, false, Some(&digest)),
18469            Agreement::Unavailable {
18470                reason: Reason::NotRunning
18471            }
18472        );
18473        assert_eq!(
18474            launch_spec_agreement(&configured, true, None),
18475            Agreement::Unavailable {
18476                reason: Reason::SpawnedSpecUnavailable
18477            }
18478        );
18479        assert_eq!(
18480            launch_spec_agreement(&configured, true, Some(&digest)),
18481            Agreement::Match
18482        );
18483    }
18484
18485    #[cfg(windows)]
18486    #[test]
18487    fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
18488        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
18489        let configured = dir.join("module.exe");
18490        std::fs::write(&configured, b"executable path fixture").unwrap();
18491        let spawned = std::fs::canonicalize(&configured).unwrap();
18492        assert_ne!(
18493            configured, spawned,
18494            "fixture must exercise different path spellings"
18495        );
18496        let forward_slashes = configured.to_string_lossy().replace('\\', "/");
18497        let lower_case = configured.to_string_lossy().to_lowercase();
18498        for spelling in [
18499            &configured,
18500            Path::new(&forward_slashes),
18501            Path::new(&lower_case),
18502        ] {
18503            let verdict = reload_verdict(
18504                spelling,
18505                Some(&spawned),
18506                subc_control::RunningImageAgreement::Unavailable {
18507                    reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18508                },
18509            );
18510            assert_eq!(
18511                verdict.path,
18512                ReloadPathAgreement::Match,
18513                "configured {spelling:?}, spawned {spawned:?}"
18514            );
18515        }
18516    }
18517
18518    #[cfg(windows)]
18519    #[test]
18520    fn windows_reload_path_keeps_different_existing_program_pending() {
18521        let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
18522        let old = dir.join("old.exe");
18523        let new = dir.join("new.exe");
18524        std::fs::write(&old, b"same executable content").unwrap();
18525        std::fs::copy(&old, &new).unwrap();
18526        let spawned = std::fs::canonicalize(&old).unwrap();
18527        let image = subc_control::RunningImageAgreement::Unavailable {
18528            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18529        };
18530        let verdict = reload_verdict(&new, Some(&spawned), image.clone());
18531        assert!(
18532            matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
18533            if configured == new && spawned_from == spawned)
18534        );
18535        let missing = dir.join("missing.exe");
18536        assert!(matches!(
18537            reload_verdict(&missing, Some(&spawned), image).path,
18538            ReloadPathAgreement::Mismatch { .. }
18539        ));
18540    }
18541
18542    #[test]
18543    fn reload_verdict_detects_replaced_image_at_same_path() {
18544        let image = subc_control::RunningImageAgreement::Mismatch {
18545            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
18546                digest: "old".into(),
18547            },
18548            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
18549                digest: "new".into(),
18550            },
18551        };
18552        let verdict = reload_verdict(
18553            std::path::Path::new("/bin/same"),
18554            Some(std::path::Path::new("/bin/same")),
18555            image.clone(),
18556        );
18557        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18558        assert_eq!(verdict.image, image);
18559    }
18560
18561    #[test]
18562    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
18563        let image = subc_control::RunningImageAgreement::Unavailable {
18564            reason: subc_control::RunningImageUnavailableReason::NotRunning,
18565        };
18566        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
18567        assert_eq!(
18568            verdict.path,
18569            subc_control::ReloadPathAgreement::Unavailable {
18570                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
18571            }
18572        );
18573        assert_eq!(verdict.image, image);
18574
18575        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
18576            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
18577        };
18578        let verdict = reload_verdict(
18579            std::path::Path::new("/bin/same"),
18580            Some(std::path::Path::new("/bin/same")),
18581            unconfirmed.clone(),
18582        );
18583        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18584        assert_eq!(verdict.image, unconfirmed);
18585    }
18586
18587    #[test]
18588    fn reload_verdict_preserves_each_image_unavailability_reason() {
18589        use subc_control::RunningImageUnavailableReason as Reason;
18590
18591        for reason in [
18592            Reason::NotRunning,
18593            Reason::UnsupportedPlatform,
18594            Reason::RunningExecutableUnreadable,
18595            Reason::SpawnedPathUnreadable,
18596            Reason::HashFailed,
18597            Reason::ProcessIdentityUnconfirmed,
18598            Reason::Unknown("future_probe_reason".to_string()),
18599        ] {
18600            let image = subc_control::RunningImageAgreement::Unavailable {
18601                reason: reason.clone(),
18602            };
18603            let verdict = reload_verdict(
18604                std::path::Path::new("/bin/same"),
18605                Some(std::path::Path::new("/bin/same")),
18606                image.clone(),
18607            );
18608            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
18609            assert_eq!(verdict.image, image, "{reason:?}");
18610        }
18611    }
18612
18613    #[tokio::test]
18614    async fn malformed_control_bodies_return_invalid_control_body() {
18615        let handler = ControlHandler::default();
18616        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
18617
18618        for (corr, body) in [
18619            (56, br#"{"route_channel":1}"#.as_slice()),
18620            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
18621            (
18622                58,
18623                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
18624            ),
18625        ] {
18626            let request = Frame::build(
18627                FrameType::Request,
18628                control_flags(),
18629                0,
18630                0,
18631                corr,
18632                body.to_vec(),
18633            )
18634            .unwrap();
18635            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
18636
18637            assert_eq!(response.len(), 1);
18638            assert_eq!(response[0].header.ty, FrameType::Error);
18639            assert_eq!(response[0].header.corr, corr);
18640            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
18641        }
18642    }
18643
18644    #[tokio::test]
18645    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
18646        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18647        let registry = Arc::new(Registry::default());
18648        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
18649        let router = Router::with_control_handler(Arc::clone(&control));
18650        let connection = router.begin_connection();
18651        let (ctx, mut rx) = route_ctx(connection.id());
18652
18653        router
18654            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
18655            .await
18656            .unwrap();
18657        let response = rx.recv().await.unwrap();
18658        let ack = parse_ack(&response);
18659        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18660        let channel = 1;
18661
18662        let goodbye =
18663            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
18664        router.route_for_connection(&ctx, goodbye).await.unwrap();
18665        assert!(rx.try_recv().is_err());
18666        assert!(registry.get_module("aft").unwrap().is_none());
18667
18668        router
18669            .route_for_connection(&ctx, channel_request(channel, 13))
18670            .await
18671            .unwrap();
18672        let error_frame = rx.recv().await.unwrap();
18673        assert_eq!(error_frame.header.ty, FrameType::Error);
18674        assert_eq!(error_frame.header.channel, channel);
18675        let captured = crate::router::test_log::captured_logs(&logs);
18676        assert_eq!(
18677            captured
18678                .lines()
18679                .filter(|line| {
18680                    line.contains("module_id=aft")
18681                        && line.contains("reason=explicit_goodbye")
18682                        && line.contains("module registration ended")
18683                })
18684                .count(),
18685            1,
18686            "unexpected GOODBYE registry log: {captured}"
18687        );
18688    }
18689
18690    #[tokio::test]
18691    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
18692        let registry = Arc::new(Registry::default());
18693        let handler = ControlHandler::new(registry).with_capability_config(
18694            [("prov".to_string(), true), ("cons".to_string(), true)],
18695            BTreeMap::new(),
18696        );
18697        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
18698        register_capability_manifest(
18699            &handler,
18700            &provider_ctx,
18701            &mut provider_rx,
18702            capability_manifest("prov", &["thing/v1"], &[]),
18703            1,
18704        )
18705        .await;
18706        let mut consumer = capability_manifest("cons", &[], &[]);
18707        consumer.capabilities.as_mut().unwrap().requires.push(
18708            subc_protocol::manifest::CapabilityRequirement {
18709                capability: "thing/v1".to_string(),
18710                need: subc_protocol::manifest::CapabilityNeed::Required,
18711            },
18712        );
18713        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
18714        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
18715        assert_eq!(
18716            handler.capability_evaluator.verdict("cons", "thing/v1"),
18717            Some(CapabilityVerdict::Provided)
18718        );
18719        let (mut client_rx, _) = open_route_for_capability_test(
18720            &handler,
18721            &provider_ctx,
18722            &mut provider_rx,
18723            703,
18724            3,
18725            "prov",
18726            None,
18727        )
18728        .await;
18729        let goodbye =
18730            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
18731        handler
18732            .handle_control_frame(&provider_ctx, goodbye)
18733            .await
18734            .unwrap();
18735        assert_eq!(
18736            handler.capability_evaluator.verdict("cons", "thing/v1"),
18737            Some(CapabilityVerdict::NeverProvided)
18738        );
18739        let closed = client_rx
18740            .try_recv()
18741            .expect("GOODBYE pushes route.closed before route GOODBYE");
18742        assert!(
18743            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
18744            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
18745        );
18746        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
18747        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
18748    }
18749
18750    struct RouteClosedFixture {
18751        handler: ControlHandler,
18752        #[cfg(unix)]
18753        supervisor: Supervisor,
18754        module_ctx: RouteCtx,
18755        #[cfg(unix)]
18756        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18757        client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18758        route: Arc<crate::forwarding::RouteBinding>,
18759    }
18760
18761    async fn route_closed_fixture() -> RouteClosedFixture {
18762        let registry = Arc::new(Registry::default());
18763        let handler = ControlHandler::new(Arc::clone(&registry));
18764        #[cfg(unix)]
18765        let supervisor = Supervisor::new(registry, RestartPolicy::default())
18766            .with_forwarding(Arc::clone(&handler.forwarding));
18767        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
18768        register_capability_manifest(
18769            &handler,
18770            &module_ctx,
18771            &mut module_rx,
18772            capability_manifest("prov", &[], &[]),
18773            1,
18774        )
18775        .await;
18776        let (client_rx, bind) = open_route_for_capability_test(
18777            &handler,
18778            &module_ctx,
18779            &mut module_rx,
18780            702,
18781            2,
18782            "prov",
18783            None,
18784        )
18785        .await;
18786        let ModuleControlRequest::RouteBind {
18787            route_channel: channel,
18788            epoch: route_epoch,
18789            ..
18790        } = bind
18791        else {
18792            panic!("expected route.bind");
18793        };
18794        let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
18795            handler
18796                .forwarding
18797                .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
18798                .unwrap()
18799        else {
18800            panic!("the acknowledged route must be bound");
18801        };
18802        RouteClosedFixture {
18803            handler,
18804            #[cfg(unix)]
18805            supervisor,
18806            module_ctx,
18807            #[cfg(unix)]
18808            module_rx,
18809            client_rx,
18810            route,
18811        }
18812    }
18813
18814    impl RouteClosedFixture {
18815        #[cfg(unix)]
18816        async fn cut(&mut self) {
18817            let notices = async {
18818                let mut draining = self.module_rx.recv().await.unwrap();
18819                assert!(matches!(
18820                    serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
18821                        &draining.body
18822                    )
18823                    .unwrap(),
18824                    subc_protocol::session::ModuleControlCommand::Draining {
18825                        reason: RouteCloseReason::Disable,
18826                        ..
18827                    }
18828                ));
18829                draining.flushed.take().unwrap().send(()).unwrap();
18830                let mut closing = self.client_rx.recv().await.unwrap();
18831                assert!(matches!(
18832                    serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
18833                    ClientControlPush::RouteClosing {
18834                        reason: RouteCloseReason::Restart,
18835                        ..
18836                    }
18837                ));
18838                closing.flushed.take().unwrap().send(()).unwrap();
18839            };
18840            let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
18841            result.unwrap();
18842        }
18843
18844        fn disconnect(&mut self) -> ClientControlPush {
18845            self.handler
18846                .cleanup_connection(self.module_ctx.connection_id)
18847                .unwrap();
18848            let closed = self.client_rx.try_recv().unwrap();
18849            assert_eq!(closed.header.ty, FrameType::Push);
18850            let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
18851            assert!(matches!(
18852                &push,
18853                ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
18854                    if channels == &[self.route.client_channel]
18855            ));
18856            assert_eq!(
18857                self.client_rx.try_recv().unwrap().header.ty,
18858                FrameType::Goodbye
18859            );
18860            assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
18861            push
18862        }
18863    }
18864
18865    #[cfg(unix)]
18866    #[tokio::test(start_paused = true)]
18867    async fn daemon_cut_quiet_routes_report_restart_and_drained() {
18868        let mut fixture = route_closed_fixture().await;
18869        fixture.cut().await;
18870        assert!(matches!(
18871            fixture.disconnect(),
18872            ClientControlPush::RouteClosed {
18873                reason: RouteCloseReason::Restart,
18874                drained: true,
18875                ..
18876            }
18877        ));
18878    }
18879
18880    #[cfg(unix)]
18881    #[tokio::test(start_paused = true)]
18882    async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
18883        let mut fixture = route_closed_fixture().await;
18884        fixture.route.flow.acquire_tagged(99, false).await.unwrap();
18885        let started = tokio::time::Instant::now();
18886        fixture.cut().await;
18887        assert!(started.elapsed() >= Duration::from_secs(2));
18888        assert_eq!(fixture.route.flow.in_flight(), 1);
18889        assert!(matches!(
18890            fixture.disconnect(),
18891            ClientControlPush::RouteClosed {
18892                reason: RouteCloseReason::Restart,
18893                drained: false,
18894                ..
18895            }
18896        ));
18897    }
18898
18899    #[tokio::test]
18900    async fn module_connection_crash_reports_crash_and_not_drained() {
18901        let mut fixture = route_closed_fixture().await;
18902        // The server calls cleanup_connection on module EOF. Without a planned
18903        // daemon shutdown, that disconnect must remain a crash with no prior
18904        // route.closing notice, even when the route has no in-flight calls.
18905        assert!(matches!(
18906            fixture.disconnect(),
18907            ClientControlPush::RouteClosed {
18908                reason: RouteCloseReason::Crash,
18909                drained: false,
18910                ..
18911            }
18912        ));
18913    }
18914
18915    #[tokio::test]
18916    async fn dropping_router_connection_releases_registration() {
18917        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18918        let registry = Arc::new(Registry::default());
18919        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
18920        let router = Router::with_control_handler(Arc::clone(&control));
18921        let connection = router.begin_connection();
18922        let connection_id = connection.id();
18923        let (ctx, mut rx) = route_ctx(connection_id);
18924
18925        router
18926            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
18927            .await
18928            .unwrap();
18929        let response = rx.recv().await.unwrap();
18930        let ack = parse_ack(&response);
18931        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
18932        assert!(registry.get_module("aft").unwrap().is_some());
18933
18934        drop(connection);
18935
18936        assert!(registry.get_module("aft").unwrap().is_none());
18937        assert_eq!(registry.active_registration_count().unwrap(), 0);
18938
18939        control
18940            .cleanup_connection(ConnectionId::new(u64::MAX))
18941            .unwrap();
18942        let captured = crate::router::test_log::captured_logs(&logs);
18943        println!("captured registry lifecycle logs:\n{captured}");
18944        let events: Vec<_> = captured
18945            .lines()
18946            .filter(|line| {
18947                line.contains("module registered module_id=aft ")
18948                    || line.contains("module registration ended")
18949            })
18950            .collect();
18951        assert_eq!(
18952            events.len(),
18953            2,
18954            "unexpected registry lifecycle logs: {captured}"
18955        );
18956        assert!(events[0].contains("module registered module_id=aft "));
18957        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
18958        assert!(events[1].contains(&format!(
18959            "module_id=aft connection_id={}",
18960            connection_id.get()
18961        )));
18962        assert!(events[1].contains("reason=connection_closed"));
18963        assert!(events[1].contains("module registration ended"));
18964        assert_eq!(
18965            captured
18966                .lines()
18967                .filter(|line| line.contains("module registered module_id=aft "))
18968                .count(),
18969            1,
18970            "legacy registration admission line must appear once: {captured}"
18971        );
18972    }
18973
18974    #[tokio::test]
18975    async fn hello_registration_keeps_legacy_module_registered_line_once() {
18976        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
18977        let registry = Arc::new(Registry::default());
18978        let control = ControlHandler::new(Arc::clone(&registry));
18979        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
18980        hello_via_sink(
18981            &control,
18982            &ctx,
18983            &mut rx,
18984            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
18985        )
18986        .await;
18987
18988        let captured = crate::router::test_log::captured_logs(&logs);
18989        let admissions: Vec<_> = captured
18990            .lines()
18991            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
18992            .collect();
18993        assert_eq!(
18994            admissions.len(),
18995            1,
18996            "legacy admission line must remain exactly once: {captured}"
18997        );
18998        assert!(admissions[0].contains("routable_provider=true"));
18999        assert!(admissions[0].contains("connection_id=777"));
19000    }
19001
19002    fn capability_manifest(
19003        module_id: &str,
19004        provides: &[&str],
19005        must_never_reach: &[&str],
19006    ) -> ModuleManifest {
19007        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
19008        manifest.capabilities = Some(CapabilityDeclarations {
19009            provides: provides
19010                .iter()
19011                .map(|capability| (*capability).to_string())
19012                .collect(),
19013            requires: Vec::new(),
19014            must_never_reach: must_never_reach
19015                .iter()
19016                .map(|capability| (*capability).to_string())
19017                .collect(),
19018        });
19019        manifest
19020    }
19021
19022    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
19023        Frame::build(
19024            FrameType::Hello,
19025            control_flags(),
19026            0,
19027            0,
19028            corr,
19029            serde_json::to_vec(&ModuleHelloBody {
19030                protocol_ver: manifest.protocol_ver,
19031                manifest,
19032                control_ops: None,
19033                launch_nonce: None,
19034            })
19035            .expect("capability test HELLO serializes"),
19036        )
19037        .expect("capability test HELLO frame builds")
19038    }
19039
19040    fn catalog_update_with_capabilities_frame(
19041        corr: u64,
19042        capabilities: CapabilityDeclarations,
19043    ) -> Frame {
19044        Frame::build(
19045            FrameType::Request,
19046            control_flags(),
19047            0,
19048            0,
19049            corr,
19050            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
19051                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
19052                capabilities: Some(capabilities),
19053                ready: None,
19054            })
19055            .expect("capability catalog.update serializes"),
19056        )
19057        .expect("capability catalog.update frame builds")
19058    }
19059
19060    async fn register_capability_manifest(
19061        handler: &ControlHandler,
19062        ctx: &RouteCtx,
19063        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
19064        manifest: ModuleManifest,
19065        corr: u64,
19066    ) {
19067        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
19068    }
19069
19070    async fn open_route_for_capability_test(
19071        handler: &ControlHandler,
19072        target_ctx: &RouteCtx,
19073        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
19074        client_connection_id: u64,
19075        corr: u64,
19076        target_module_id: &str,
19077        consumer_identity: Option<ConsumerIdentity>,
19078    ) -> (
19079        mpsc::Receiver<crate::router::OutboundFrame>,
19080        ModuleControlRequest,
19081    ) {
19082        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
19083        let route_handler = handler.clone();
19084        let target_module_id = target_module_id.to_string();
19085        let route_task = tokio::spawn(async move {
19086            route_handler
19087                .handle_control_frame(
19088                    &client_ctx,
19089                    route_open_frame_with_admission_facts(
19090                        corr,
19091                        &target_module_id,
19092                        unique_project_root("admission-facts"),
19093                        consumer_identity,
19094                        None,
19095                    ),
19096                )
19097                .await
19098                .expect("capability test route.open succeeds")
19099        });
19100        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
19101            .await
19102            .expect("capability test route.open must reach route.bind")
19103            .expect("target control receiver stays open");
19104        let bind_request: ModuleControlRequest =
19105            serde_json::from_slice(&bind.body).expect("route.bind decodes");
19106        handler
19107            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
19108            .await
19109            .expect("capability test route.bind ACK succeeds");
19110        assert!(route_task.await.expect("route.open task joins").is_empty());
19111        let opened = client_rx
19112            .recv()
19113            .await
19114            .expect("successful route.open publishes a response");
19115        assert!(matches!(
19116            serde_json::from_slice::<ClientControlResponse>(&opened.body),
19117            Ok(ClientControlResponse::RouteOpen { .. })
19118        ));
19119        (client_rx, bind_request)
19120    }
19121
19122    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
19123        assert_eq!(frame.header.ty, FrameType::Push);
19124        assert_eq!(frame.header.channel, 0);
19125        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
19126            .expect("route.closed control push decodes");
19127        let ClientControlPush::RouteClosed { channels, .. } = &push else {
19128            panic!("expected route.closed");
19129        };
19130        assert_eq!(channels.len(), 1, "exactly one violating route closed");
19131        let closed_channel = channels[0];
19132        let channels = channels.clone();
19133        assert_eq!(
19134            push,
19135            ClientControlPush::RouteClosed {
19136                module_id: target_module_id.to_string(),
19137                channels,
19138                reason: RouteCloseReason::CapabilityDenied,
19139                drained: false,
19140                abandoned: 0,
19141                excluded_subscriptions: 0,
19142                terminal: Some(false),
19143            }
19144        );
19145        closed_channel
19146    }
19147
19148    #[tokio::test]
19149    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
19150        let registry = Arc::new(Registry::default());
19151        let forwarding = Arc::new(ForwardingTable::default());
19152        let supervisor = SupervisorHandle::new();
19153        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19154        let handler =
19155            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19156                .with_supervisor(supervisor);
19157        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
19158        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
19159        register_capability_manifest(
19160            &handler,
19161            &target_ctx,
19162            &mut target_rx,
19163            capability_manifest("target", &["credentials-provider/v1"], &[]),
19164            1,
19165        )
19166        .await;
19167        register_capability_manifest(
19168            &handler,
19169            &opener_ctx,
19170            &mut opener_rx,
19171            capability_manifest("opener", &[], &["credentials-provider/v1"]),
19172            2,
19173        )
19174        .await;
19175
19176        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
19177        let replies = handler
19178            .handle_control_frame(
19179                &client_ctx,
19180                route_open_frame_with_admission_facts(
19181                    3,
19182                    "target",
19183                    unique_project_root("admission-facts"),
19184                    Some(ConsumerIdentity {
19185                        module_id: "opener".to_string(),
19186                        launch_nonce: "opener-nonce".to_string(),
19187                    }),
19188                    None,
19189                ),
19190            )
19191            .await
19192            .expect("denied route.open returns a typed frame");
19193        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19194        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19195        assert!(
19196            target_rx.try_recv().is_err(),
19197            "forbidden route.open must not relay route.bind"
19198        );
19199    }
19200
19201    #[tokio::test]
19202    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
19203        let registry = Arc::new(Registry::default());
19204        let forwarding = Arc::new(ForwardingTable::default());
19205        let supervisor = SupervisorHandle::new();
19206        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19207        let handler =
19208            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19209                .with_supervisor(supervisor);
19210        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
19211        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
19212        register_capability_manifest(
19213            &handler,
19214            &target_ctx,
19215            &mut target_rx,
19216            capability_manifest("target", &["credentials-provider/v1"], &[]),
19217            1,
19218        )
19219        .await;
19220        register_capability_manifest(
19221            &handler,
19222            &old_opener_ctx,
19223            &mut old_opener_rx,
19224            capability_manifest("opener", &[], &[]),
19225            2,
19226        )
19227        .await;
19228        let (mut client_rx, _) = open_route_for_capability_test(
19229            &handler,
19230            &target_ctx,
19231            &mut target_rx,
19232            712,
19233            3,
19234            "target",
19235            Some(ConsumerIdentity {
19236                module_id: "opener".to_string(),
19237                launch_nonce: "opener-nonce".to_string(),
19238            }),
19239        )
19240        .await;
19241        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19242
19243        handler
19244            .cleanup_connection(old_opener_ctx.connection_id)
19245            .expect("old opener registration cleans up");
19246        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
19247        register_capability_manifest(
19248            &handler,
19249            &new_opener_ctx,
19250            &mut new_opener_rx,
19251            capability_manifest("opener", &[], &["credentials-provider/v1"]),
19252            4,
19253        )
19254        .await;
19255
19256        let closed_channel = assert_capability_denied_push(
19257            client_rx
19258                .try_recv()
19259                .expect("HELLO deny addition must emit route.closed")
19260                .frame,
19261            "target",
19262        );
19263        let goodbye = client_rx
19264            .try_recv()
19265            .expect("HELLO deny addition must send the client a route GOODBYE")
19266            .frame;
19267        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19268        assert_eq!(goodbye.header.channel, closed_channel);
19269        assert_ne!(goodbye.header.channel, 0);
19270        assert_ne!(goodbye.header.epoch, 0);
19271        assert!(client_rx.try_recv().is_err());
19272        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19273        assert!(matches!(
19274            target_rx.try_recv(),
19275            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19276        ));
19277    }
19278
19279    #[tokio::test]
19280    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
19281        let registry = Arc::new(Registry::default());
19282        let forwarding = Arc::new(ForwardingTable::default());
19283        let supervisor = SupervisorHandle::new();
19284        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19285        let handler =
19286            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19287                .with_supervisor(supervisor);
19288        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
19289        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
19290        register_capability_manifest(
19291            &handler,
19292            &target_ctx,
19293            &mut target_rx,
19294            capability_manifest("target", &[], &[]),
19295            1,
19296        )
19297        .await;
19298        register_capability_manifest(
19299            &handler,
19300            &opener_ctx,
19301            &mut opener_rx,
19302            capability_manifest("opener", &[], &["credentials-provider/v1"]),
19303            2,
19304        )
19305        .await;
19306        let (mut client_rx, _) = open_route_for_capability_test(
19307            &handler,
19308            &target_ctx,
19309            &mut target_rx,
19310            722,
19311            3,
19312            "target",
19313            Some(ConsumerIdentity {
19314                module_id: "opener".to_string(),
19315                launch_nonce: "opener-nonce".to_string(),
19316            }),
19317        )
19318        .await;
19319        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19320
19321        let replies = handler
19322            .handle_control_frame(
19323                &target_ctx,
19324                catalog_update_with_capabilities_frame(
19325                    4,
19326                    CapabilityDeclarations {
19327                        provides: vec!["credentials-provider/v1".to_string()],
19328                        requires: Vec::new(),
19329                        must_never_reach: Vec::new(),
19330                    },
19331                ),
19332            )
19333            .await
19334            .expect("claim catalog.update succeeds");
19335        assert!(matches!(
19336            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
19337            Ok(ModuleControlResponseToModule::CatalogUpdate {})
19338        ));
19339        let closed_channel = assert_capability_denied_push(
19340            client_rx
19341                .try_recv()
19342                .expect("claim addition must emit route.closed")
19343                .frame,
19344            "target",
19345        );
19346        let goodbye = client_rx
19347            .try_recv()
19348            .expect("claim addition must send the client a route GOODBYE")
19349            .frame;
19350        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19351        assert_eq!(goodbye.header.channel, closed_channel);
19352        assert_ne!(goodbye.header.channel, 0);
19353        assert_ne!(goodbye.header.epoch, 0);
19354        assert!(client_rx.try_recv().is_err());
19355        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19356        assert!(matches!(
19357            target_rx.try_recv(),
19358            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
19359        ));
19360    }
19361
19362    #[tokio::test]
19363    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
19364        let registry = Arc::new(Registry::default());
19365        let forwarding = Arc::new(ForwardingTable::default());
19366        let supervisor = SupervisorHandle::new();
19367        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19368        let handler =
19369            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19370                .with_supervisor(supervisor);
19371        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
19372        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
19373        register_capability_manifest(
19374            &handler,
19375            &target_ctx,
19376            &mut target_rx,
19377            capability_manifest("target", &["credentials-provider/v1"], &[]),
19378            1,
19379        )
19380        .await;
19381        register_capability_manifest(
19382            &handler,
19383            &opener_ctx,
19384            &mut opener_rx,
19385            capability_manifest("opener", &[], &[]),
19386            2,
19387        )
19388        .await;
19389        let (mut client_rx, _) = open_route_for_capability_test(
19390            &handler,
19391            &target_ctx,
19392            &mut target_rx,
19393            732,
19394            3,
19395            "target",
19396            Some(ConsumerIdentity {
19397                module_id: "opener".to_string(),
19398                launch_nonce: "opener-nonce".to_string(),
19399            }),
19400        )
19401        .await;
19402        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19403
19404        handler
19405            .handle_control_frame(
19406                &target_ctx,
19407                catalog_update_with_capabilities_frame(
19408                    4,
19409                    CapabilityDeclarations {
19410                        provides: Vec::new(),
19411                        requires: Vec::new(),
19412                        must_never_reach: Vec::new(),
19413                    },
19414                ),
19415            )
19416            .await
19417            .expect("claim removal catalog.update succeeds");
19418        assert_eq!(
19419            forwarding.active_binding_count().unwrap(),
19420            1,
19421            "removing an attested target claim must leave the route census unchanged"
19422        );
19423        assert!(
19424            client_rx.try_recv().is_err(),
19425            "claim removal must not emit route.closed or a client route GOODBYE"
19426        );
19427        assert!(
19428            target_rx.try_recv().is_err(),
19429            "claim removal must not send the target a route GOODBYE"
19430        );
19431    }
19432
19433    /// A direct client may open a route to a denied capability provider; this
19434    /// policy applies only to attested supervised module origins, not to direct clients.
19435    #[tokio::test]
19436    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
19437        let registry = Arc::new(Registry::default());
19438        let forwarding = Arc::new(ForwardingTable::default());
19439        let supervisor = SupervisorHandle::new();
19440        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
19441        let handler =
19442            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19443                .with_supervisor(supervisor);
19444        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
19445        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
19446        register_capability_manifest(
19447            &handler,
19448            &target_ctx,
19449            &mut target_rx,
19450            capability_manifest("target", &["credentials-provider/v1"], &[]),
19451            1,
19452        )
19453        .await;
19454        register_capability_manifest(
19455            &handler,
19456            &opener_ctx,
19457            &mut opener_rx,
19458            capability_manifest("opener", &[], &["credentials-provider/v1"]),
19459            2,
19460        )
19461        .await;
19462
19463        let (_client_rx, bind) = open_route_for_capability_test(
19464            &handler,
19465            &target_ctx,
19466            &mut target_rx,
19467            742,
19468            3,
19469            "target",
19470            None,
19471        )
19472        .await;
19473        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
19474            panic!("direct scope-honesty route must bind");
19475        };
19476        assert_eq!(principal, Some(Principal::Direct));
19477        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
19478    }
19479
19480    /// A module that denies a capability receives no self-route exemption when it
19481    /// also attestedly provides that capability.
19482    #[tokio::test]
19483    async fn must_never_reach_self_route_is_capability_forbidden() {
19484        let registry = Arc::new(Registry::default());
19485        let forwarding = Arc::new(ForwardingTable::default());
19486        let supervisor = SupervisorHandle::new();
19487        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
19488        let handler =
19489            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
19490                .with_supervisor(supervisor);
19491        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
19492        register_capability_manifest(
19493            &handler,
19494            &self_ctx,
19495            &mut self_rx,
19496            capability_manifest(
19497                "self-provider",
19498                &["credentials-provider/v1"],
19499                &["credentials-provider/v1"],
19500            ),
19501            1,
19502        )
19503        .await;
19504
19505        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
19506        let replies = handler
19507            .handle_control_frame(
19508                &client_ctx,
19509                route_open_frame_with_admission_facts(
19510                    2,
19511                    "self-provider",
19512                    unique_project_root("admission-facts"),
19513                    Some(ConsumerIdentity {
19514                        module_id: "self-provider".to_string(),
19515                        launch_nonce: "self-nonce".to_string(),
19516                    }),
19517                    None,
19518                ),
19519            )
19520            .await
19521            .expect("self-route refusal returns a typed frame");
19522        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
19523        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
19524        assert!(
19525            self_rx.try_recv().is_err(),
19526            "self denial must not relay route.bind"
19527        );
19528    }
19529
19530    #[test]
19531    fn unsupported_channel_zero_frame_returns_error() {
19532        let handler = ControlHandler::default();
19533        let request = Frame::build(
19534            FrameType::Request,
19535            control_flags(),
19536            0,
19537            0,
19538            21,
19539            b"opaque".to_vec(),
19540        )
19541        .unwrap();
19542
19543        let response = handler
19544            .handle_control(ConnectionId::new(1), request)
19545            .unwrap();
19546
19547        assert_eq!(response[0].header.ty, FrameType::Error);
19548        assert_eq!(
19549            parse_error(&response[0])["code"],
19550            "unsupported_control_frame"
19551        );
19552    }
19553
19554    /// Blue/green swap at the control-plane boundary. The supervisor that opens
19555    /// a swap is not wired yet, so the candidate is registered here directly
19556    /// into the registry and forwarding candidate slots, the way the swap's
19557    /// HELLO admission will.
19558    mod swap {
19559        use super::*;
19560
19561        const INCUMBENT: ConnectionId = ConnectionId::new(30);
19562        const CANDIDATE: ConnectionId = ConnectionId::new(40);
19563
19564        struct Swap {
19565            registry: Arc<Registry>,
19566            forwarding: Arc<ForwardingTable>,
19567            handler: ControlHandler,
19568            incumbent_ctx: RouteCtx,
19569            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19570            candidate_ctx: RouteCtx,
19571            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19572        }
19573
19574        async fn swap_with_incumbent() -> Swap {
19575            let registry = Arc::new(Registry::default());
19576            let forwarding = Arc::new(ForwardingTable::default());
19577            let handler =
19578                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
19579            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
19580            hello_via_sink(
19581                &handler,
19582                &incumbent_ctx,
19583                &mut incumbent_rx,
19584                hello_frame("aft", PROTOCOL_VERSION, 7),
19585            )
19586            .await;
19587            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
19588            Swap {
19589                registry,
19590                forwarding,
19591                handler,
19592                incumbent_ctx,
19593                incumbent_rx,
19594                candidate_ctx,
19595                candidate_rx,
19596            }
19597        }
19598
19599        fn register_candidate(swap: &Swap, ready: Option<bool>) {
19600            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
19601            candidate_manifest.ready = ready;
19602            let registration = swap
19603                .registry
19604                .register_candidate_with_control_ops(
19605                    candidate_manifest,
19606                    PROTOCOL_VERSION,
19607                    CANDIDATE,
19608                    module_baseline_control_ops(),
19609                )
19610                .unwrap();
19611            swap.forwarding
19612                .register_candidate_module_connection(
19613                    CANDIDATE,
19614                    "aft".to_string(),
19615                    PROTOCOL_VERSION,
19616                    manifest_concurrency(&registration.manifest),
19617                    swap.candidate_ctx.egress.clone(),
19618                )
19619                .unwrap();
19620        }
19621
19622        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
19623            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
19624            swap.registry.promote_candidate("aft").unwrap().unwrap();
19625            cutover.incumbent.unwrap()
19626        }
19627
19628        /// A swap candidate's HELLO reserves its candidate registry entry
19629        /// before installing its candidate forwarding endpoint. The swap
19630        /// supervisor promotes a ready candidate as soon as it reads it as
19631        /// registered, and promotion cuts forwarding over to the candidate
19632        /// endpoint, so the lookup the supervisor reads
19633        /// (`Registry::published_registration`) must not return the candidate
19634        /// until that endpoint exists. Otherwise the cutover finds no candidate
19635        /// endpoint and the swap fails as if the candidate had disconnected.
19636        #[tokio::test]
19637        async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
19638            let mut swap = swap_with_incumbent().await;
19639            let hold = Arc::new(std::sync::Barrier::new(2));
19640            let handler = swap
19641                .handler
19642                .clone()
19643                .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
19644            handler
19645                .supervisor
19646                .open_swap("aft", "candidate-nonce".to_string());
19647            let hello = {
19648                let handler = handler.clone();
19649                let candidate_ctx = swap.candidate_ctx.clone();
19650                std::thread::spawn(move || {
19651                    handler
19652                        .handle_hello(
19653                            candidate_ctx.connection_id,
19654                            Some(candidate_ctx.egress.clone()),
19655                            hello_frame_with_nonce(
19656                                "aft",
19657                                PROTOCOL_VERSION,
19658                                8,
19659                                Some("candidate-nonce"),
19660                            ),
19661                        )
19662                        .unwrap()
19663                })
19664            };
19665
19666            hold.wait();
19667            // Parked: the candidate slot is claimed but not published, and
19668            // forwarding has no candidate endpoint.
19669            assert!(swap
19670                .registry
19671                .registration(crate::registry::RegistrationSlot::Candidate("aft"))
19672                .unwrap()
19673                .is_some());
19674            assert!(swap
19675                .registry
19676                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19677                .unwrap()
19678                .is_none());
19679            assert!(swap
19680                .forwarding
19681                .module_endpoint_for_connection(CANDIDATE)
19682                .unwrap()
19683                .is_none());
19684
19685            hold.wait();
19686            assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
19687            assert_eq!(
19688                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19689                FrameType::HelloAck
19690            );
19691            let published = swap
19692                .registry
19693                .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19694                .unwrap()
19695                .expect("candidate is published once its HELLO completes");
19696            assert_eq!(published.connection_id, CANDIDATE);
19697            assert!(swap
19698                .forwarding
19699                .module_endpoint_for_connection(CANDIDATE)
19700                .unwrap()
19701                .is_some());
19702        }
19703
19704        #[tokio::test]
19705        async fn candidate_lookup_after_hello_ack_waits_for_publication() {
19706            let mut swap = swap_with_incumbent().await;
19707            let hold = Arc::new(std::sync::Barrier::new(2));
19708            swap.handler.hello_ack_to_publication_hold = Some(Arc::clone(&hold));
19709            swap.handler
19710                .supervisor
19711                .open_swap("aft", "candidate-nonce".to_string());
19712            let hello = {
19713                let handler = swap.handler.clone();
19714                let candidate_ctx = swap.candidate_ctx.clone();
19715                std::thread::spawn(move || {
19716                    handler
19717                        .handle_hello(
19718                            CANDIDATE,
19719                            Some(candidate_ctx.egress),
19720                            hello_frame_with_nonce(
19721                                "aft",
19722                                PROTOCOL_VERSION,
19723                                8,
19724                                Some("candidate-nonce"),
19725                            ),
19726                        )
19727                        .unwrap()
19728                })
19729            };
19730            hold.wait();
19731            assert_eq!(
19732                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19733                FrameType::HelloAck
19734            );
19735            let reader_registry = Arc::clone(&swap.registry);
19736            let published = lookup_during_hello_publication(&swap.registry, &hold, move || {
19737                reader_registry
19738                    .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
19739                    .unwrap()
19740            });
19741            assert!(hello.join().unwrap().is_empty());
19742            assert_eq!(
19743                published
19744                    .expect("ACK makes candidate publication visible")
19745                    .connection_id,
19746                CANDIDATE
19747            );
19748            assert_eq!(
19749                swap.registry.generation().unwrap(),
19750                1,
19751                "candidate is not a second catalog entry"
19752            );
19753        }
19754
19755        #[tokio::test]
19756        async fn audit4_ungated_removed_candidate_publication_rolls_back_admission() {
19757            let mut swap = swap_with_incumbent().await;
19758            let hold = Arc::new(std::sync::Barrier::new(2));
19759            let handler = swap
19760                .handler
19761                .clone()
19762                .with_hello_registry_to_forwarding_hold(hold.clone());
19763            handler
19764                .supervisor
19765                .open_swap("aft", "candidate-nonce".into());
19766            let hello = {
19767                let handler = handler.clone();
19768                let ctx = swap.candidate_ctx.clone();
19769                std::thread::spawn(move || {
19770                    handler
19771                        .handle_hello(
19772                            CANDIDATE,
19773                            Some(ctx.egress),
19774                            hello_frame_with_nonce(
19775                                "aft",
19776                                PROTOCOL_VERSION,
19777                                8,
19778                                Some("candidate-nonce"),
19779                            ),
19780                        )
19781                        .unwrap()
19782                })
19783            };
19784            hold.wait();
19785            assert_eq!(
19786                swap.registry
19787                    .deregister_connection(CANDIDATE)
19788                    .unwrap()
19789                    .len(),
19790                1
19791            );
19792            hold.wait();
19793            assert!(hello.join().unwrap().is_empty());
19794            assert_eq!(
19795                swap.candidate_rx.try_recv().unwrap().frame.header.ty,
19796                FrameType::HelloAck
19797            );
19798            assert!(swap.registry.get_candidate("aft").unwrap().is_none());
19799            assert_eq!(
19800                handler
19801                    .supervisor
19802                    .swap_hello_admission("aft", Some("candidate-nonce")),
19803                SwapHelloAdmission::Candidate,
19804                "failed publication must roll back candidate_admitted for an ungated nonce swap"
19805            );
19806            handler.supervisor.close_swap("aft");
19807        }
19808
19809        fn keyed_total(counters: &Value, key: &str) -> u64 {
19810            counters[key]
19811                .as_object()
19812                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
19813                .unwrap_or(0)
19814        }
19815
19816        #[tokio::test]
19817        async fn replacement_logs_old_end_and_new_admission_once() {
19818            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
19819            let mut swap = swap_with_incumbent().await;
19820            swap.handler
19821                .supervisor
19822                .open_swap("aft", "candidate-nonce".to_string());
19823            hello_via_sink(
19824                &swap.handler,
19825                &swap.candidate_ctx,
19826                &mut swap.candidate_rx,
19827                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
19828            )
19829            .await;
19830            cutover(&swap);
19831            swap.handler.cleanup_connection(INCUMBENT).unwrap();
19832
19833            let captured = crate::router::test_log::captured_logs(&logs);
19834            println!("captured replacement registry logs:\n{captured}");
19835            let new_admissions: Vec<_> = captured
19836                .lines()
19837                .filter(|line| {
19838                    line.contains("connection_id=40")
19839                        && line.contains("swap candidate registered; not routable until cutover")
19840                })
19841                .collect();
19842            assert_eq!(
19843                new_admissions.len(),
19844                1,
19845                "unexpected admission logs: {captured}"
19846            );
19847            assert!(new_admissions[0].contains("module_id=aft"));
19848            assert!(new_admissions[0].contains("connection_id=40"));
19849            assert!(
19850                new_admissions[0].contains("swap candidate registered; not routable until cutover")
19851            );
19852            assert!(new_admissions[0].contains("ready=true"));
19853
19854            let old_admissions: Vec<_> = captured
19855                .lines()
19856                .filter(|line| {
19857                    line.contains("module registered module_id=aft ")
19858                        && line.contains("connection_id=30")
19859                })
19860                .collect();
19861            assert_eq!(
19862                old_admissions.len(),
19863                1,
19864                "unexpected incumbent admission: {captured}"
19865            );
19866
19867            let old_ends: Vec<_> = captured
19868                .lines()
19869                .filter(|line| {
19870                    line.contains("connection_id=30") && line.contains("module registration ended")
19871                })
19872                .collect();
19873            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
19874            assert!(old_ends[0].contains("module_id=aft"));
19875            assert!(old_ends[0].contains("connection_id=30"));
19876            assert!(old_ends[0].contains("reason=replaced"));
19877            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
19878            assert_eq!(
19879                captured
19880                    .lines()
19881                    .filter(|line| line.contains("module registration promoted"))
19882                    .count(),
19883                1,
19884                "unexpected promotion logs: {captured}"
19885            );
19886        }
19887
19888        /// An ack from the incumbent for a bind it was sent before cutover,
19889        /// arriving before the incumbent is drained. The incumbent is the live
19890        /// connection carrying every other client's routes, so the ack must
19891        /// not end it: the waiting client is told to retry, the reservation is
19892        /// given back, and the incumbent is told to drop just that binding.
19893        #[tokio::test]
19894        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
19895            let mut swap = swap_with_incumbent().await;
19896            let handler = swap.handler.clone();
19897
19898            // A co-tenant route, bound on the incumbent before the swap.
19899            let cotenant = ConnectionId::new(31);
19900            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
19901            let (cotenant_task, cotenant_bind) = relay_route_open(
19902                &handler,
19903                cotenant,
19904                &cotenant_ctx.egress,
19905                &mut swap.incumbent_rx,
19906                100,
19907                "aft",
19908                "swap-cotenant",
19909            )
19910            .await;
19911            handler
19912                .handle_control_frame(
19913                    &swap.incumbent_ctx,
19914                    route_bind_ack(cotenant_bind.header.corr),
19915                )
19916                .await
19917                .unwrap();
19918            assert!(cotenant_task.await.unwrap().is_empty());
19919            let (cotenant_channel, cotenant_epoch) =
19920                published_route(&cotenant_rx.recv().await.unwrap());
19921
19922            // A second route.open, relayed to the incumbent and not yet acked.
19923            let caller = ConnectionId::new(32);
19924            let (caller_ctx, mut caller_rx) = route_ctx(caller);
19925            let (caller_task, caller_bind) = relay_route_open(
19926                &handler,
19927                caller,
19928                &caller_ctx.egress,
19929                &mut swap.incumbent_rx,
19930                101,
19931                "aft",
19932                "swap-caller",
19933            )
19934            .await;
19935            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
19936
19937            register_candidate(&swap, None);
19938            cutover(&swap);
19939
19940            // The incumbent acks after promotion and before any drain.
19941            let ack = handler
19942                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
19943                .await;
19944            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
19945            if module_loop_error.is_some() {
19946                // What the connection loop does with an untranslated router
19947                // error: end the connection, releasing every route on it.
19948                handler.cleanup_connection(INCUMBENT).unwrap();
19949            }
19950
19951            // 1. The incumbent's other routes survive.
19952            assert!(
19953                cotenant_rx.try_recv().is_err(),
19954                "the co-tenant route on the incumbent was torn down by one late ack: \
19955                 {module_loop_error:?}"
19956            );
19957            assert!(matches!(
19958                swap.forwarding
19959                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
19960                    .unwrap(),
19961                DataRoute::Client(DataRouteState::Bound(_))
19962            ));
19963            assert_eq!(module_loop_error, None);
19964            assert!(swap
19965                .registry
19966                .get_module_by_connection(INCUMBENT)
19967                .unwrap()
19968                .is_some());
19969
19970            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
19971            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
19972                .await
19973                .expect("the incumbent is told to drop the abandoned binding")
19974                .unwrap()
19975                .frame;
19976            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19977            assert_eq!(goodbye.header.channel, abandoned_channel);
19978            assert_eq!(goodbye.header.epoch, abandoned_epoch);
19979            assert!(swap.incumbent_rx.try_recv().is_err());
19980
19981            // 3. The waiting client gets a retryable refusal and no route.
19982            let response = caller_task.await.unwrap();
19983            assert_eq!(response.len(), 1);
19984            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
19985            assert!(caller_rx.try_recv().is_err());
19986
19987            // 4. The reservation pair is given back, and the pending bind
19988            //    settled exactly once: one accepted open (the co-tenant) and one
19989            //    refused open (the caller), nothing counted twice.
19990            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
19991            let counters = handler.counters().snapshot();
19992            assert_eq!(
19993                keyed_total(&counters, "route_open_accepted_by_principal"),
19994                1
19995            );
19996            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
19997            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
19998        }
19999
20000        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
20001        /// id would resolve to the promoted candidate and every new route.open
20002        /// would be refused as reloading, leaving neither process routable.
20003        #[tokio::test]
20004        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
20005            let mut swap = swap_with_incumbent().await;
20006            register_candidate(&swap, None);
20007            let incumbent = cutover(&swap);
20008            swap.forwarding
20009                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
20010                .unwrap()
20011                .expect("the incumbent is still registered");
20012
20013            let client = ConnectionId::new(33);
20014            let (client_ctx, mut client_rx) = route_ctx(client);
20015            let route_handler = swap.handler.clone();
20016            let open_ctx = RouteCtx {
20017                connection_id: client,
20018                egress: client_ctx.egress.clone(),
20019            };
20020            let mut route_task = tokio::spawn(async move {
20021                route_handler
20022                    .handle_control_frame(
20023                        &open_ctx,
20024                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
20025                    )
20026                    .await
20027                    .unwrap()
20028            });
20029            let bind = tokio::select! {
20030                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
20031                response = &mut route_task => {
20032                    let response = response.unwrap();
20033                    panic!(
20034                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
20035                        parse_error(&response[0])["code"]
20036                    );
20037                }
20038            };
20039            swap.handler
20040                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
20041                .await
20042                .unwrap();
20043            assert!(route_task.await.unwrap().is_empty());
20044            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
20045            match swap
20046                .forwarding
20047                .lookup_data_route(client, channel, epoch)
20048                .unwrap()
20049            {
20050                DataRoute::Client(DataRouteState::Bound(route)) => {
20051                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
20052                }
20053                other => panic!("expected a bound route on the candidate, got {other:?}"),
20054            }
20055            assert!(swap.incumbent_rx.try_recv().is_err());
20056        }
20057
20058        /// A candidate declares itself ready with `catalog.update` on its own
20059        /// connection. If the connection-keyed registry lookups searched only the
20060        /// active slot, this would answer `not_registered` and the candidate
20061        /// would never become ready.
20062        #[tokio::test]
20063        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
20064            let swap = swap_with_incumbent().await;
20065            register_candidate(&swap, Some(false));
20066            let update = Frame::build(
20067                FrameType::Request,
20068                control_flags(),
20069                0,
20070                0,
20071                55,
20072                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
20073                    provides: manifest("aft", PROTOCOL_VERSION).provides,
20074                    capabilities: None,
20075                    ready: Some(true),
20076                })
20077                .unwrap(),
20078            )
20079            .unwrap();
20080
20081            let replies = swap
20082                .handler
20083                .handle_control_frame(&swap.candidate_ctx, update)
20084                .await
20085                .unwrap();
20086
20087            assert_eq!(replies.len(), 1);
20088            assert_eq!(
20089                replies[0].header.ty,
20090                FrameType::Response,
20091                "candidate catalog.update was refused: {:?}",
20092                serde_json::from_slice::<Value>(&replies[0].body).ok()
20093            );
20094            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
20095            assert_eq!(
20096                swap.registry
20097                    .get_module("aft")
20098                    .unwrap()
20099                    .unwrap()
20100                    .connection_id,
20101                INCUMBENT
20102            );
20103        }
20104    }
20105
20106    /// The HELLO gate while the supervisor has a swap open: only the nonce it
20107    /// minted for the candidate admits a second process, into the candidate
20108    /// slot, and that check runs ahead of the reserved-module gate.
20109    mod swap_admission {
20110        use super::*;
20111
20112        const INCUMBENT_NONCE: &str = "incumbent-nonce";
20113        const CANDIDATE_NONCE: &str = "candidate-nonce";
20114
20115        fn handler_with_incumbent(
20116            module_id: &str,
20117            reserved: bool,
20118        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
20119            let registry = Arc::new(Registry::default());
20120            let supervisor = SupervisorHandle::new();
20121            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
20122            if reserved {
20123                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
20124            }
20125            let handler =
20126                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
20127            let incumbent = handler
20128                .handle_control(
20129                    ConnectionId::new(1),
20130                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
20131                )
20132                .unwrap();
20133            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
20134            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
20135            (registry, supervisor, handler)
20136        }
20137
20138        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
20139        /// admits every nonce, so while a swap is open the swap gate is the only
20140        /// thing between a key-holder and the candidate slot. A nonce the
20141        /// supervisor did not mint, or none at all, is refused, and neither the
20142        /// incumbent's registration nor the candidate slot moves.
20143        #[test]
20144        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
20145            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
20146
20147            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
20148                let replies = handler
20149                    .handle_control(
20150                        ConnectionId::new(connection),
20151                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
20152                    )
20153                    .unwrap();
20154                assert_eq!(replies[0].header.ty, FrameType::Error);
20155                assert_eq!(
20156                    parse_error(&replies[0])["code"],
20157                    "swap_token_invalid",
20158                    "nonce {nonce:?}"
20159                );
20160            }
20161            assert!(registry.get_candidate("aft").unwrap().is_none());
20162            assert_eq!(
20163                registry.get_module("aft").unwrap().unwrap().connection_id,
20164                ConnectionId::new(1)
20165            );
20166
20167            // Control: the minted token is admitted, into the candidate slot,
20168            // and only once.
20169            let admitted = handler
20170                .handle_control(
20171                    ConnectionId::new(4),
20172                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
20173                )
20174                .unwrap();
20175            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
20176            assert_eq!(
20177                registry
20178                    .get_candidate("aft")
20179                    .unwrap()
20180                    .unwrap()
20181                    .connection_id,
20182                ConnectionId::new(4)
20183            );
20184            assert_eq!(
20185                registry.get_module("aft").unwrap().unwrap().connection_id,
20186                ConnectionId::new(1),
20187                "the candidate must not take the active slot"
20188            );
20189            let replayed = handler
20190                .handle_control(
20191                    ConnectionId::new(5),
20192                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
20193                )
20194                .unwrap();
20195            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
20196
20197            // The case only this gate covers: the incumbent has died mid-swap,
20198            // so its duplicate refusal is gone too, and without the gate a
20199            // key-holder would take the id's ACTIVE slot.
20200            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
20201            let squatter = handler
20202                .handle_control(
20203                    ConnectionId::new(6),
20204                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
20205                )
20206                .unwrap();
20207            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
20208            assert!(
20209                registry.get_module("aft").unwrap().is_none(),
20210                "a squatter took the active slot of an id being swapped"
20211            );
20212        }
20213
20214        /// Design mutation arm (iii). A reserved module's candidate presents a
20215        /// nonce the reserved gate has never seen (that gate holds the
20216        /// incumbent's), so the swap gate must run first or the candidate is
20217        /// refused `reserved_module` and a reserved module can never be swapped.
20218        #[test]
20219        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
20220            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
20221
20222            let replies = handler
20223                .handle_control(
20224                    ConnectionId::new(2),
20225                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20226                )
20227                .unwrap();
20228
20229            assert_eq!(
20230                replies[0].header.ty,
20231                FrameType::HelloAck,
20232                "reserved candidate refused: {:?}",
20233                serde_json::from_slice::<Value>(&replies[0].body).ok()
20234            );
20235            assert_eq!(
20236                registry
20237                    .get_candidate("vault")
20238                    .unwrap()
20239                    .unwrap()
20240                    .connection_id,
20241                ConnectionId::new(2)
20242            );
20243        }
20244
20245        /// With no swap open the gate is inert: the incumbent's reserved gate
20246        /// and duplicate refusal behave exactly as before.
20247        #[test]
20248        fn without_an_open_swap_the_ordinary_gates_decide() {
20249            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
20250            supervisor.close_swap("vault");
20251
20252            let candidate = handler
20253                .handle_control(
20254                    ConnectionId::new(2),
20255                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
20256                )
20257                .unwrap();
20258            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
20259            let duplicate = handler
20260                .handle_control(
20261                    ConnectionId::new(3),
20262                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
20263                )
20264                .unwrap();
20265            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
20266            assert!(registry.get_candidate("vault").unwrap().is_none());
20267        }
20268    }
20269
20270    /// `scope.sync` and `scope.describe` through the real control handler: who
20271    /// may sync is decided by the registration and launch nonce of the module
20272    /// connection, never by the request body.
20273    mod scopes {
20274        mod close_reasons {
20275            include!("route_close_reason_tests.rs");
20276        }
20277        mod idle_tests {
20278            use super::*;
20279            include!("scope_idle_tests.rs");
20280        }
20281        mod route_open_timings {
20282            use super::*;
20283            include!("route_open_timing_tests.rs");
20284        }
20285        mod route_open_refusal_log {
20286            use super::*;
20287            include!("route_open_refusal_log_tests.rs");
20288        }
20289        mod activity {
20290            include!("activity_scope_tests.rs");
20291        }
20292        mod delta {
20293            include!("scope_handler_delta_tests.rs");
20294        }
20295        mod call_modes {
20296            include!("scope_call_mode_tests.rs");
20297        }
20298        mod federated {
20299            include!("scope_federated_tests.rs");
20300        }
20301        mod authorize {
20302            include!("route_authorize_tests.rs");
20303        }
20304        #[tokio::test]
20305        async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
20306            let rig = rig().await;
20307            let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
20308                "s".into(),
20309                1,
20310                "tool".into(),
20311            ))
20312            .unwrap();
20313            assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
20314            let (direct, _rx) = wide_ctx(999);
20315            for (ctx, expected) in [
20316                (&rig.owner, FrameType::Response),
20317                (&direct, FrameType::Error),
20318            ] {
20319                let frame =
20320                    Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
20321                        .unwrap();
20322                let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
20323                assert_eq!(replies[0].header.ty, expected);
20324                if expected == FrameType::Response {
20325                    let reply: subc_control::ScopeServedByReply =
20326                        serde_json::from_slice(&replies[0].body).unwrap();
20327                    assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
20328                } else {
20329                    assert_eq!(parse_error(&replies[0])["code"], "not_registered");
20330                }
20331            }
20332            let reply = call(
20333                &rig.handler,
20334                &rig.owner,
20335                &ModuleControlRequestFromModule::ScopeDescribe {
20336                    owner: Principal::Reserved {
20337                        module_id: OWNER.into(),
20338                    },
20339                    scope_ref: "s".into(),
20340                },
20341            )
20342            .await;
20343            assert!(matches!(
20344                serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
20345                ModuleControlResponseToModule::ScopeDescribe { .. }
20346            ));
20347        }
20348
20349        use subc_protocol::scope::{
20350            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
20351            ScopeStatus,
20352        };
20353
20354        use super::*;
20355
20356        const OWNER: &str = "prefrontal-core";
20357
20358        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
20359            ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
20360        }
20361
20362        async fn call(
20363            handler: &ControlHandler,
20364            ctx: &RouteCtx,
20365            request: &ModuleControlRequestFromModule,
20366        ) -> Frame {
20367            let body = serde_json::to_vec(request).unwrap();
20368            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
20369            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
20370            assert_eq!(replies.len(), 1, "{replies:?}");
20371            replies.pop().unwrap()
20372        }
20373
20374        async fn sync(
20375            handler: &ControlHandler,
20376            ctx: &RouteCtx,
20377            generation: u64,
20378            scopes: Vec<ScopeRecord>,
20379        ) -> Result<ModuleControlResponseToModule, String> {
20380            let reply = call(
20381                handler,
20382                ctx,
20383                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
20384            )
20385            .await;
20386            match reply.header.ty {
20387                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
20388                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
20389            }
20390        }
20391
20392        async fn describe(
20393            handler: &ControlHandler,
20394            ctx: &RouteCtx,
20395            owner: &str,
20396            scope_ref: &str,
20397        ) -> ModuleControlResponseToModule {
20398            let reply = call(
20399                handler,
20400                ctx,
20401                &ModuleControlRequestFromModule::ScopeDescribe {
20402                    owner: Principal::Reserved {
20403                        module_id: owner.to_string(),
20404                    },
20405                    scope_ref: scope_ref.to_string(),
20406                },
20407            )
20408            .await;
20409            assert_eq!(
20410                reply.header.ty,
20411                FrameType::Response,
20412                "{:?}",
20413                parse_error(&reply)
20414            );
20415            serde_json::from_slice(&reply.body).unwrap()
20416        }
20417
20418        #[tokio::test]
20419        async fn scope_describe_never_echoes_bind_proof_or_hash() {
20420            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
20421            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
20422            let mut rig = rig().await;
20423            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
20424                .await;
20425            let _bound = rig
20426                .bound(
20427                    Some(AFT),
20428                    PLEXUS,
20429                    Some(
20430                        rig.selector("s", Some(1))
20431                            .with_bind_proof(Some(PROOF.into())),
20432                    ),
20433                )
20434                .await;
20435            let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
20436            let output = serde_json::to_string(&reply).unwrap();
20437            assert!(output.contains("\"status\":\"live\""));
20438            assert!(!output.contains(PROOF));
20439            assert!(!output.contains(HASH));
20440        }
20441
20442        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
20443        async fn module(
20444            handler: &ControlHandler,
20445            connection: u64,
20446            module_id: &str,
20447            nonce: Option<&str>,
20448        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20449            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
20450            hello_via_sink(
20451                handler,
20452                &ctx,
20453                &mut rx,
20454                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
20455            )
20456            .await;
20457            (ctx, rx)
20458        }
20459
20460        /// `direct` and every other client connection has no registration, so
20461        /// it can neither sync nor own a scope.
20462        #[tokio::test]
20463        async fn a_client_connection_cannot_sync_or_describe() {
20464            let handler = ControlHandler::new(Arc::new(Registry::default()));
20465            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
20466            for request in [
20467                ModuleControlRequestFromModule::ScopeSync {
20468                    generation: 1,
20469                    scopes: vec![head("s", 1)],
20470                },
20471                ModuleControlRequestFromModule::ScopeDescribe {
20472                    owner: Principal::Direct,
20473                    scope_ref: "s".to_string(),
20474                },
20475            ] {
20476                let reply = call(&handler, &ctx, &request).await;
20477                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
20478            }
20479            assert!(
20480                !handler
20481                    .scopes
20482                    .read()
20483                    .unwrap()
20484                    .describe(
20485                        &Principal::Reserved {
20486                            module_id: OWNER.to_string()
20487                        },
20488                        "s"
20489                    )
20490                    .owner_synced
20491            );
20492        }
20493
20494        /// A module the supervisor did not spawn registers without a launch
20495        /// nonce, so it is never an owner's current launch.
20496        #[tokio::test]
20497        async fn a_module_without_a_supervised_launch_cannot_sync() {
20498            let handler = ControlHandler::new(Arc::new(Registry::default()));
20499            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
20500            assert_eq!(
20501                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
20502                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20503            );
20504        }
20505
20506        #[tokio::test]
20507        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
20508            let supervisor = SupervisorHandle::new();
20509            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20510            let handler = ControlHandler::new(Arc::new(Registry::default()))
20511                .with_supervisor(supervisor.clone());
20512            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20513            sync(&handler, &incumbent, 1, vec![head("s", 1)])
20514                .await
20515                .expect("the current launch syncs");
20516
20517            // A swap candidate registers with the swap token and is refused
20518            // while the incumbent keeps syncing.
20519            supervisor.open_swap(OWNER, "n2".to_string());
20520            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
20521            assert_eq!(
20522                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20523                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20524            );
20525            sync(&handler, &incumbent, 2, vec![head("s", 1)])
20526                .await
20527                .expect("the serving owner syncs during the swap");
20528
20529            // The swap fails and is rolled back. The candidate never held sync
20530            // authority, and still cannot sync.
20531            supervisor.close_swap(OWNER);
20532            assert_eq!(
20533                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
20534                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20535            );
20536            sync(&handler, &incumbent, 3, vec![head("s", 1)])
20537                .await
20538                .expect("the serving owner syncs after the rollback");
20539            handler.cleanup_connection(candidate.connection_id).unwrap();
20540
20541            // A swap that cuts over. Promotion records the candidate's nonce as
20542            // the module's spawn nonce, which is what `set_spawn_nonce` does
20543            // here; the promoted connection then takes authority at any
20544            // generation and the superseded incumbent is refused.
20545            supervisor.open_swap(OWNER, "n3".to_string());
20546            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
20547            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
20548            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
20549                .await
20550                .expect("the promoted launch takes authority");
20551            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
20552                panic!("unexpected reply {reply:?}");
20553            };
20554            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
20555            assert_eq!(
20556                sync(&handler, &incumbent, 4, Vec::new()).await,
20557                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
20558            );
20559        }
20560
20561        /// Authority dies with its connection: the cleanup path releases it,
20562        /// so the owner's next connection takes it at any generation.
20563        #[tokio::test]
20564        async fn closing_the_authority_connection_frees_sync_authority() {
20565            let supervisor = SupervisorHandle::new();
20566            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20567            let handler = ControlHandler::new(Arc::new(Registry::default()))
20568                .with_supervisor(supervisor.clone());
20569            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
20570            sync(&handler, &first, 10, vec![head("s", 1)])
20571                .await
20572                .unwrap();
20573            handler.cleanup_connection(first.connection_id).unwrap();
20574
20575            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
20576            sync(&handler, &second, 1, vec![head("s", 1)])
20577                .await
20578                .expect("the next connection takes the released authority");
20579        }
20580
20581        #[tokio::test]
20582        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
20583            let supervisor = SupervisorHandle::new();
20584            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
20585            let handler =
20586                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
20587            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
20588            sync(&handler, &first, 10, vec![head("s", 1)])
20589                .await
20590                .unwrap();
20591            handler
20592                .handle_control_frame(
20593                    &first,
20594                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
20595                )
20596                .await
20597                .unwrap();
20598            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
20599            sync(&handler, &second, 1, vec![head("s", 1)])
20600                .await
20601                .expect("GOODBYE releases authority even if the old socket remains open");
20602        }
20603
20604        #[tokio::test]
20605        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
20606            let registry = Arc::new(Registry::default());
20607            let supervisor_handle = SupervisorHandle::new();
20608            let supervisor =
20609                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
20610                    .with_handle(supervisor_handle.clone())
20611                    .with_daemon_incarnation("incarnation-7".to_string());
20612            // Configured with enabled: false, so the supervisor lists the
20613            // module without spawning a process for it.
20614            supervisor
20615                .supervise_configured(
20616                    ModuleSpec {
20617                        script: None,
20618                        carry_containment: false,
20619                        module_id: OWNER.to_string(),
20620                        program: PathBuf::from("/nonexistent/prefrontal-core"),
20621                        args: Vec::new(),
20622                        env: Vec::new(),
20623                        reserved: false,
20624                        reserved_prefixes: Vec::new(),
20625                        protocol: ModuleProtocol::Subc,
20626                        overlap: Default::default(),
20627                    },
20628                    false,
20629                )
20630                .unwrap();
20631            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
20632            let handler =
20633                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
20634            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
20635
20636            // Configured but not yet synced: a reader waits for the owner.
20637            let ModuleControlResponseToModule::ScopeDescribe {
20638                status,
20639                daemon_incarnation,
20640                owner_synced,
20641                owner_configured,
20642                scope,
20643                ..
20644            } = describe(&handler, &reader, OWNER, "s").await
20645            else {
20646                panic!("not a describe reply");
20647            };
20648            assert_eq!(status, ScopeStatus::NotLive);
20649            assert_eq!(daemon_incarnation, "incarnation-7");
20650            assert!(!owner_synced);
20651            assert!(owner_configured);
20652            assert!(scope.is_none());
20653
20654            // Not a supervised module: the owner will never sync, and a reader
20655            // refuses rather than waits.
20656            let ModuleControlResponseToModule::ScopeDescribe {
20657                status,
20658                owner_configured,
20659                ..
20660            } = describe(&handler, &reader, "ghost", "s").await
20661            else {
20662                panic!("not a describe reply");
20663            };
20664            assert_eq!(status, ScopeStatus::NotLive);
20665            assert!(!owner_configured);
20666
20667            // Live, with the stamp fields and the computed owner_authorized.
20668            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
20669            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
20670            let ModuleControlResponseToModule::ScopeDescribe {
20671                status,
20672                scope_epoch,
20673                owner_synced,
20674                scope,
20675                ..
20676            } = describe(&handler, &reader, OWNER, "s").await
20677            else {
20678                panic!("not a describe reply");
20679            };
20680            assert_eq!(status, ScopeStatus::Live);
20681            assert_eq!(scope_epoch, Some(4));
20682            assert!(owner_synced);
20683            let stamp = scope.expect("a live scope carries its stamp");
20684            assert!(
20685                stamp.owner_authorized,
20686                "prefrontal-core is the default authority"
20687            );
20688            assert_eq!(stamp.kind, ScopeKind::Head);
20689        }
20690
20691        #[tokio::test]
20692        async fn scope_authority_owners_decides_owner_authorized() {
20693            let supervisor = SupervisorHandle::new();
20694            supervisor.set_spawn_nonce("broca", "b1".to_string());
20695            let handler = ControlHandler::new(Arc::new(Registry::default()))
20696                .with_supervisor(supervisor)
20697                .with_scope_authority_owners(vec!["broca".to_string()]);
20698            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
20699            let mut gated = head("s", 1);
20700            gated.attributes.agent_id = Some("agent".to_string());
20701            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
20702            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
20703                describe(&handler, &broca, "broca", "s").await
20704            else {
20705                panic!("not a describe reply");
20706            };
20707            assert!(scope.unwrap().owner_authorized);
20708        }
20709
20710        /// With route admission, the stamp, the commit re-check and drains in
20711        /// place, the feature is advertised: the module ops in HELLO_ACK, and
20712        /// `scopes/v1` in HELLO_ACK and `server.describe`.
20713        #[tokio::test]
20714        async fn scope_ops_and_the_scopes_capability_are_advertised() {
20715            let handler = ControlHandler::new(Arc::new(Registry::default()));
20716            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
20717            let ack = hello_via_sink(
20718                &handler,
20719                &ctx,
20720                &mut rx,
20721                hello_frame("m", PROTOCOL_VERSION, 1),
20722            )
20723            .await;
20724            let ack = parse_ack(&ack);
20725            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
20726                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
20727            }
20728            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
20729
20730            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
20731            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
20732            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
20733            let reply = handler
20734                .handle_control_frame(&client, frame)
20735                .await
20736                .unwrap()
20737                .pop()
20738                .unwrap();
20739            let ClientControlResponse::ServerDescribe { capabilities, .. } =
20740                serde_json::from_slice(&reply.body).unwrap()
20741            else {
20742                panic!("not a server.describe reply");
20743            };
20744            assert!(
20745                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
20746                "{capabilities:?}"
20747            );
20748        }
20749
20750        // ---- route admission, stamps, commit re-check and drains ----------
20751
20752        const PLEXUS: &str = "plexus";
20753        const OTHER: &str = "other";
20754        const AFT: &str = "aft";
20755        const BROCA: &str = "broca";
20756        const MAGIC: &str = "magic-context";
20757
20758        fn nonce(module_id: &str) -> String {
20759            format!("nonce-{module_id}")
20760        }
20761
20762        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
20763            let (tx, rx) = mpsc::channel(64);
20764            (
20765                RouteCtx {
20766                    connection_id: ConnectionId::new(connection),
20767                    egress: FrameSink::new(tx),
20768                },
20769                rx,
20770            )
20771        }
20772
20773        /// A daemon with a configured owner (prefrontal-core) registered on its
20774        /// own module connection, two routable targets (plexus, other), and
20775        /// launch nonces minted for the modules that open routes as carriers.
20776        struct Rig {
20777            handler: ControlHandler,
20778            forwarding: Arc<ForwardingTable>,
20779            owner: RouteCtx,
20780            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
20781            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
20782            generation: u64,
20783            next_connection: u64,
20784            _supervisor: Supervisor,
20785        }
20786
20787        async fn rig() -> Rig {
20788            rig_with_flow_support(true).await
20789        }
20790
20791        async fn rig_with_flow_support(flow_support: bool) -> Rig {
20792            rig_with_scope_capabilities(flow_support, false).await
20793        }
20794
20795        async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
20796            let mut provides = Vec::new();
20797            if flow_support {
20798                provides.push("flow-scopes/v1");
20799            }
20800            if run_support {
20801                provides.push("agent-run-scopes/v1");
20802            }
20803            rig_with_target_capabilities(&provides).await
20804        }
20805
20806        fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
20807            let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
20808            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
20809            // Decoder versions alone cannot admit scoped calls: only a capability
20810            // declaration promises the required run or flow behaviour.
20811            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
20812            body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
20813            Frame::build(
20814                FrameType::Hello,
20815                control_flags(),
20816                0,
20817                0,
20818                connection,
20819                serde_json::to_vec(&body).unwrap(),
20820            )
20821            .unwrap()
20822        }
20823
20824        async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
20825            let registry = Arc::new(Registry::default());
20826            let forwarding = Arc::new(ForwardingTable::default());
20827            let supervisor_handle = SupervisorHandle::new();
20828            let supervisor =
20829                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
20830                    .with_handle(supervisor_handle.clone());
20831            supervisor
20832                .supervise_configured(
20833                    ModuleSpec {
20834                        script: None,
20835                        carry_containment: false,
20836                        module_id: OWNER.to_string(),
20837                        program: PathBuf::from("/nonexistent/prefrontal-core"),
20838                        args: Vec::new(),
20839                        env: Vec::new(),
20840                        reserved: false,
20841                        reserved_prefixes: Vec::new(),
20842                        protocol: ModuleProtocol::Subc,
20843                        overlap: Default::default(),
20844                    },
20845                    false,
20846                )
20847                .unwrap();
20848            for module_id in [OWNER, AFT, BROCA, MAGIC] {
20849                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
20850            }
20851            let handler =
20852                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
20853                    .with_supervisor(supervisor_handle);
20854            let (owner, mut owner_rx) = wide_ctx(1);
20855            hello_via_sink(
20856                &handler,
20857                &owner,
20858                &mut owner_rx,
20859                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
20860            )
20861            .await;
20862            let mut modules = BTreeMap::new();
20863            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
20864                let (ctx, mut rx) = wide_ctx(connection);
20865                let hello = hello_with_capabilities(module_id, connection, provides);
20866                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
20867                modules.insert(module_id.to_string(), (ctx, rx));
20868            }
20869            Rig {
20870                handler,
20871                forwarding,
20872                owner,
20873                _owner_rx: owner_rx,
20874                modules,
20875                generation: 0,
20876                next_connection: 100,
20877                _supervisor: supervisor,
20878            }
20879        }
20880
20881        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
20882            ScopeCarrier::new(Principal::Reserved {
20883                module_id: module_id.to_string(),
20884            })
20885            .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
20886        }
20887
20888        /// The scope most tests open under: aft carries to any module, broca
20889        /// only to plexus and other, and the owner delegates as agent-1.
20890        fn session(scope_epoch: u64) -> ScopeRecord {
20891            let mut record = head("s", scope_epoch);
20892            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
20893            record.attributes.agent_id = Some("agent-1".to_string());
20894            record.attributes.delegates = true;
20895            record
20896        }
20897
20898        impl Rig {
20899            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
20900                self.generation += 1;
20901                sync(&self.handler, &self.owner, self.generation, scopes)
20902                    .await
20903                    .expect("the owner's sync is accepted");
20904            }
20905
20906            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
20907                ScopeSelector::new(
20908                    Principal::Reserved {
20909                        module_id: OWNER.to_string(),
20910                    },
20911                    scope_ref.to_string(),
20912                    scope_epoch,
20913                )
20914            }
20915
20916            fn open_frame(
20917                &mut self,
20918                opener: Option<&str>,
20919                target: &str,
20920                scope: Option<ScopeSelector>,
20921            ) -> (
20922                RouteCtx,
20923                mpsc::Receiver<crate::router::OutboundFrame>,
20924                Frame,
20925            ) {
20926                self.next_connection += 1;
20927                let (ctx, rx) = wide_ctx(self.next_connection);
20928                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
20929                    busy_refusals: None,
20930                    target: RouteTarget::ToolProvider {
20931                        module_id: target.to_string(),
20932                    },
20933                    identity: BindIdentity::new("unit".to_string(), "session".to_string()),
20934                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
20935                        module_id: module_id.to_string(),
20936                        launch_nonce: nonce(module_id),
20937                    }),
20938                    consumer_capabilities: None,
20939                    role_versions: None,
20940                    admission_facts: None,
20941                    scope,
20942                })
20943                .unwrap();
20944                let frame = Frame::build(
20945                    FrameType::Request,
20946                    control_flags(),
20947                    0,
20948                    0,
20949                    self.next_connection,
20950                    body,
20951                )
20952                .unwrap();
20953                (ctx, rx, frame)
20954            }
20955
20956            /// Open and expect a refusal before anything is relayed.
20957            async fn refused(
20958                &mut self,
20959                opener: Option<&str>,
20960                target: &str,
20961                scope: Option<ScopeSelector>,
20962            ) -> String {
20963                self.refusal_body(opener, target, scope).await["code"]
20964                    .as_str()
20965                    .unwrap()
20966                    .to_string()
20967            }
20968
20969            async fn refusal_body(
20970                &mut self,
20971                opener: Option<&str>,
20972                target: &str,
20973                scope: Option<ScopeSelector>,
20974            ) -> Value {
20975                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
20976                let replies = tokio::time::timeout(
20977                    Duration::from_secs(2),
20978                    self.handler.handle_control_frame(&ctx, frame),
20979                )
20980                .await
20981                .expect("the open must be refused before waiting for a bind ack")
20982                .unwrap();
20983                assert_eq!(replies.len(), 1, "{replies:?}");
20984                assert_eq!(replies[0].header.ty, FrameType::Error);
20985                let (_, module_rx) = self.modules.get_mut(target).unwrap();
20986                assert!(
20987                    module_rx.try_recv().is_err(),
20988                    "a refused open relays nothing"
20989                );
20990                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
20991                parse_error(&replies[0])
20992            }
20993
20994            /// Start an open and return its task and the bind the target got.
20995            async fn relayed(
20996                &mut self,
20997                opener: Option<&str>,
20998                target: &str,
20999                scope: Option<ScopeSelector>,
21000            ) -> Relayed {
21001                self.relayed_with_legacy_root(opener, target, scope, None)
21002                    .await
21003            }
21004
21005            async fn relayed_with_legacy_root(
21006                &mut self,
21007                opener: Option<&str>,
21008                target: &str,
21009                scope: Option<ScopeSelector>,
21010                legacy_root: Option<&std::path::Path>,
21011            ) -> Relayed {
21012                let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
21013                if let Some(root) = legacy_root {
21014                    let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
21015                    body["identity"]["project_root"] = serde_json::json!(root);
21016                    frame.body = serde_json::to_vec(&body).unwrap();
21017                }
21018                let handler = self.handler.clone();
21019                let task_ctx = ctx.clone();
21020                let task = tokio::spawn(async move {
21021                    handler
21022                        .handle_control_frame(&task_ctx, frame)
21023                        .await
21024                        .unwrap()
21025                });
21026                let (_, module_rx) = self.modules.get_mut(target).unwrap();
21027                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
21028                    .await
21029                    .expect("the target receives the relayed route.bind")
21030                    .unwrap()
21031                    .frame;
21032                Relayed {
21033                    target: target.to_string(),
21034                    client: ctx,
21035                    client_rx: rx,
21036                    task,
21037                    bind,
21038                }
21039            }
21040
21041            async fn ack(&self, relayed: &Relayed) {
21042                let (module, _) = &self.modules[&relayed.target];
21043                self.handler
21044                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
21045                    .await
21046                    .unwrap();
21047            }
21048
21049            /// Open, ack and return the bound route.
21050            async fn bound(
21051                &mut self,
21052                opener: Option<&str>,
21053                target: &str,
21054                scope: Option<ScopeSelector>,
21055            ) -> Bound {
21056                let relayed = self.relayed(opener, target, scope).await;
21057                self.commit_relayed(relayed).await
21058            }
21059
21060            async fn bound_with_legacy_root(
21061                &mut self,
21062                opener: Option<&str>,
21063                target: &str,
21064                scope: Option<ScopeSelector>,
21065                root: &std::path::Path,
21066            ) -> Bound {
21067                let relayed = self
21068                    .relayed_with_legacy_root(opener, target, scope, Some(root))
21069                    .await;
21070                self.commit_relayed(relayed).await
21071            }
21072
21073            async fn commit_relayed(&self, relayed: Relayed) -> Bound {
21074                self.ack(&relayed).await;
21075                let Relayed {
21076                    target,
21077                    client,
21078                    mut client_rx,
21079                    task,
21080                    bind,
21081                } = relayed;
21082                assert!(
21083                    task.await.unwrap().is_empty(),
21084                    "the open is answered by commit"
21085                );
21086                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
21087                Bound {
21088                    target,
21089                    client,
21090                    client_rx,
21091                    channel,
21092                    epoch,
21093                    bind,
21094                }
21095            }
21096
21097            fn live(&self, route: &Bound) -> bool {
21098                matches!(
21099                    self.forwarding
21100                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21101                        .unwrap(),
21102                    DataRoute::Client(DataRouteState::Bound(_))
21103                )
21104            }
21105        }
21106
21107        struct Relayed {
21108            target: String,
21109            client: RouteCtx,
21110            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21111            task: tokio::task::JoinHandle<Vec<Frame>>,
21112            bind: Frame,
21113        }
21114
21115        struct Bound {
21116            target: String,
21117            client: RouteCtx,
21118            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
21119            channel: u16,
21120            epoch: u32,
21121            bind: Frame,
21122        }
21123
21124        impl Bound {
21125            /// The reason of the `route.closed` this client was sent, after
21126            /// checking it also got a GOODBYE on exactly this route.
21127            fn closed_reason(&mut self) -> RouteCloseReason {
21128                let mut reason = None;
21129                let mut goodbye = false;
21130                while let Ok(outbound) = self.client_rx.try_recv() {
21131                    let frame = outbound.frame;
21132                    match frame.header.ty {
21133                        FrameType::Goodbye => {
21134                            assert_eq!(
21135                                (frame.header.channel, frame.header.epoch),
21136                                (self.channel, self.epoch)
21137                            );
21138                            goodbye = true;
21139                        }
21140                        FrameType::Push => {
21141                            let ClientControlPush::RouteClosed {
21142                                reason: r,
21143                                module_id,
21144                                ..
21145                            } = serde_json::from_slice(&frame.body).unwrap()
21146                            else {
21147                                panic!("unexpected push");
21148                            };
21149                            assert_eq!(module_id, self.target);
21150                            reason = Some(r);
21151                        }
21152                        other => panic!("unexpected frame {other:?}"),
21153                    }
21154                }
21155                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
21156                reason.expect("the client is told why the route closed")
21157            }
21158
21159            fn untouched(&mut self) -> bool {
21160                self.client_rx.try_recv().is_err()
21161            }
21162
21163            fn stamp(&self) -> Option<ScopeStamp> {
21164                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
21165                    ModuleControlRequest::RouteBind { scope, .. } => scope,
21166                    other => panic!("expected a route.bind, got {other:?}"),
21167                }
21168            }
21169        }
21170
21171        #[tokio::test]
21172        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
21173        ) {
21174            let mut rig = rig().await;
21175            let mut record = session(1);
21176            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
21177            record.child_owners = vec![Principal::Reserved {
21178                module_id: MAGIC.to_string(),
21179            }];
21180            rig.sync(vec![record]).await;
21181            let scope = || Some(rig_selector("s", Some(1)));
21182
21183            // Admitted: the owner, a bare carrier to any module, a targeted
21184            // carrier to its listed module.
21185            rig.bound(Some(OWNER), PLEXUS, scope()).await;
21186            rig.bound(Some(AFT), OTHER, scope()).await;
21187            rig.bound(Some(BROCA), PLEXUS, scope()).await;
21188
21189            // Refused scope_not_carrier: a targeted carrier to an unlisted
21190            // module, a module that is not listed at all (a child owner is not
21191            // a carrier), and a direct key-holder.
21192            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
21193                assert_eq!(
21194                    rig.refused(opener, target, scope()).await,
21195                    error_codes::SCOPE_NOT_CARRIER,
21196                    "{opener:?} -> {target}"
21197                );
21198            }
21199        }
21200
21201        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
21202            ScopeSelector::new(
21203                Principal::Reserved {
21204                    module_id: OWNER.to_string(),
21205                },
21206                scope_ref.to_string(),
21207                scope_epoch,
21208            )
21209        }
21210
21211        #[tokio::test]
21212        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
21213            let mut rig = rig().await;
21214            rig.sync(vec![session(1)]).await;
21215            for opener in [OWNER, AFT] {
21216                assert_eq!(
21217                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
21218                        .await,
21219                    error_codes::SCOPE_EPOCH_REQUIRED,
21220                    "{opener}"
21221                );
21222            }
21223        }
21224
21225        #[tokio::test]
21226        async fn admission_separates_not_synced_not_live_and_ended() {
21227            let mut rig = rig().await;
21228            // Before the configured owner's first sync: retryable.
21229            let code = rig
21230                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21231                .await;
21232            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
21233            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
21234
21235            // An owner that is not configured will never sync: terminal.
21236            let ghost = ScopeSelector::new(
21237                Principal::Reserved {
21238                    module_id: "ghost".to_string(),
21239                },
21240                "s".to_string(),
21241                Some(1),
21242            );
21243            assert_eq!(
21244                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
21245                error_codes::SCOPE_NOT_LIVE
21246            );
21247
21248            rig.sync(vec![session(2)]).await;
21249            assert_eq!(
21250                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
21251                    .await,
21252                error_codes::SCOPE_NOT_LIVE
21253            );
21254            for epoch in [1, 3] {
21255                assert_eq!(
21256                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
21257                        .await,
21258                    error_codes::SCOPE_ENDED,
21259                    "epoch {epoch}"
21260                );
21261            }
21262            // Control: the live epoch is admitted.
21263            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
21264                .await;
21265        }
21266
21267        #[tokio::test]
21268        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
21269            let mut rig = rig().await;
21270            rig.sync(vec![session(1)]).await;
21271            let route = rig
21272                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21273                .await;
21274            let stamp = route.stamp().expect("a scoped bind carries the stamp");
21275            assert_eq!(stamp.scope_ref, "s");
21276            assert_eq!(stamp.scope_epoch, 1);
21277            assert_eq!(stamp.kind, ScopeKind::Head);
21278            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
21279            assert!(stamp.attributes.delegates);
21280            assert!(stamp.owner_authorized);
21281            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21282            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
21283
21284            // broca owns a scope of its own on its own module connection; it is
21285            // not in scope_authority_owners, so its stamp is not authorized.
21286            let (broca, mut broca_rx) = wide_ctx(50);
21287            hello_via_sink(
21288                &rig.handler,
21289                &broca,
21290                &mut broca_rx,
21291                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
21292            )
21293            .await;
21294            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
21295                .await
21296                .unwrap();
21297            let own = ScopeSelector::new(
21298                Principal::Reserved {
21299                    module_id: BROCA.to_string(),
21300                },
21301                "b".to_string(),
21302                Some(1),
21303            );
21304            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
21305            assert!(!route.stamp().unwrap().owner_authorized);
21306        }
21307
21308        #[tokio::test]
21309        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
21310            let mut rig = rig().await;
21311            let mut record = head("s", 1);
21312            record.carriers = vec![carrier(AFT, None)];
21313            let flow_id = "Flow:run-7/step_2!~";
21314            record.attributes.flow_id = Some(flow_id.to_string());
21315            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
21316                .await
21317                .unwrap();
21318            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
21319                panic!("not a sync reply");
21320            };
21321            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
21322            let route = rig
21323                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21324                .await;
21325            assert!(rig.live(&route), "the stamped bind committed");
21326            let stamp = route.stamp().expect("a flow scope carries a stamp");
21327            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
21328            assert_eq!(stamp.attributes.agent_id, None);
21329            assert!(!stamp.attributes.delegates);
21330            assert!(stamp.owner_authorized);
21331            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21332            assert_eq!(unscoped.stamp(), None);
21333        }
21334
21335        #[tokio::test]
21336        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
21337            let mut rig = rig_with_flow_support(false).await;
21338            let mut record = session(1);
21339            record.attributes.flow_id = Some("flow:7".to_string());
21340            rig.sync(vec![record]).await;
21341            for opener in [OWNER, AFT] {
21342                let body = rig
21343                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21344                    .await;
21345                assert_eq!(body["code"], "target_flow_unsupported");
21346                let message = body["message"].as_str().unwrap();
21347                for required in [PLEXUS, "flow-scopes/v1"] {
21348                    assert!(message.contains(required), "{message}");
21349                }
21350            }
21351        }
21352
21353        #[tokio::test]
21354        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
21355            let mut rig = rig_with_flow_support(true).await;
21356            let mut record = session(1);
21357            record.attributes.flow_id = Some("flow:7".to_string());
21358            rig.sync(vec![record]).await;
21359            for opener in [OWNER, AFT] {
21360                let route = rig
21361                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
21362                    .await;
21363                assert!(rig.live(&route));
21364                assert_eq!(
21365                    route.stamp().unwrap().attributes.flow_id.as_deref(),
21366                    Some("flow:7")
21367                );
21368            }
21369        }
21370
21371        #[tokio::test]
21372        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
21373            use std::future::Future;
21374
21375            let mut rig = rig_with_flow_support(true).await;
21376            let mut record = session(1);
21377            record.attributes.flow_id = Some("flow:7".to_string());
21378            rig.sync(vec![record]).await;
21379            let (client, mut client_rx, frame) =
21380                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
21381            // Hold the route-open response permit so admission sees the first
21382            // target but relay reservation cannot capture an endpoint yet.
21383            for _ in 0..64 {
21384                client.egress.try_send(route_bind_ack(1)).unwrap();
21385            }
21386            let handler = rig.handler.clone();
21387            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21388            std::future::poll_fn(|cx| {
21389                assert!(open.as_mut().poll(cx).is_pending());
21390                std::task::Poll::Ready(())
21391            })
21392            .await;
21393            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21394
21395            let old_connection = rig.modules[PLEXUS].0.connection_id;
21396            rig.handler.cleanup_connection(old_connection).unwrap();
21397            let (replacement, mut replacement_rx) = wide_ctx(200);
21398            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
21399            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
21400            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
21401            let hello = Frame::build(
21402                FrameType::Hello,
21403                control_flags(),
21404                0,
21405                0,
21406                200,
21407                serde_json::to_vec(&body).unwrap(),
21408            )
21409            .unwrap();
21410            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
21411
21412            client_rx.try_recv().unwrap();
21413            let replies = tokio::time::timeout(Duration::from_secs(2), open)
21414                .await
21415                .expect("the replacement is refused without waiting for a bind ack")
21416                .unwrap();
21417            assert_eq!(replies.len(), 1);
21418            let body = parse_error(&replies[0]);
21419            assert_eq!(body["code"], "target_flow_unsupported");
21420            for required in [PLEXUS, "flow-scopes/v1"] {
21421                assert!(body["message"].as_str().unwrap().contains(required));
21422            }
21423            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
21424            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21425            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21426            assert!(rig
21427                .handler
21428                .registry
21429                .get_module_by_connection(replacement.connection_id)
21430                .unwrap()
21431                .is_some());
21432        }
21433
21434        #[tokio::test]
21435        async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
21436            use std::future::Future;
21437            const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
21438            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21439            const NEXT_HASH: &str =
21440                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21441            let mut rig = rig().await;
21442            let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
21443            record.carriers.push(ScopeCarrier::new(Principal::Direct));
21444            rig.sync(vec![record.clone()]).await;
21445            let (client, mut rx, frame) = rig.open_frame(
21446                None,
21447                PLEXUS,
21448                Some(
21449                    rig.selector("s", Some(1))
21450                        .with_bind_proof(Some(PROOF.into())),
21451                ),
21452            );
21453            // Hold egress reservation while the scope owner changes its bind-proof
21454            // hash, so a proof valid at admission must be checked again at relay.
21455            for _ in 0..64 {
21456                client.egress.try_send(route_bind_ack(1)).unwrap();
21457            }
21458            let handler = rig.handler.clone();
21459            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21460            std::future::poll_fn(|cx| {
21461                assert!(open.as_mut().poll(cx).is_pending());
21462                std::task::Poll::Ready(())
21463            })
21464            .await;
21465            record.bind_proof_sha256 = Some(NEXT_HASH.into());
21466            rig.sync(vec![record]).await;
21467            rx.try_recv().unwrap();
21468            let replies = tokio::time::timeout(Duration::from_secs(2), open)
21469                .await
21470                .unwrap()
21471                .unwrap();
21472            assert_eq!(
21473                parse_error(&replies[0])["code"],
21474                error_codes::SCOPE_BIND_PROOF_MISMATCH
21475            );
21476            assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
21477            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
21478            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21479        }
21480
21481        #[tokio::test]
21482        async fn scope_bind_proof_reserved_openers_ignore_proof() {
21483            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21484            let mut rig = rig().await;
21485            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21486                .await;
21487            for opener in [OWNER, AFT] {
21488                for proof in [None, Some("malformed+proof=")] {
21489                    let route = rig
21490                        .bound(
21491                            Some(opener),
21492                            PLEXUS,
21493                            Some(
21494                                rig.selector("s", Some(1))
21495                                    .with_bind_proof(proof.map(str::to_string)),
21496                            ),
21497                        )
21498                        .await;
21499                    assert!(rig.live(&route));
21500                }
21501            }
21502        }
21503
21504        #[tokio::test]
21505        async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
21506            use std::future::Future;
21507            const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
21508            const NEXT_HASH: &str =
21509                "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
21510            let mut rig = rig().await;
21511            rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
21512                .await;
21513            let (client, mut rx, frame) =
21514                rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
21515            for _ in 0..64 {
21516                client.egress.try_send(route_bind_ack(1)).unwrap();
21517            }
21518            let handler = rig.handler.clone();
21519            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
21520            std::future::poll_fn(|cx| {
21521                assert!(open.as_mut().poll(cx).is_pending());
21522                std::task::Poll::Ready(())
21523            })
21524            .await;
21525            rig.sync(vec![
21526                session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
21527            ])
21528            .await;
21529            rx.try_recv().unwrap();
21530            let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21531            let bind = std::future::poll_fn(|cx| {
21532                assert!(open.as_mut().poll(cx).is_pending());
21533                module_rx.poll_recv(cx)
21534            })
21535            .await
21536            .unwrap()
21537            .frame;
21538            handler
21539                .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
21540                .await
21541                .unwrap();
21542            let replies = tokio::time::timeout(Duration::from_secs(2), open)
21543                .await
21544                .unwrap()
21545                .unwrap();
21546            assert_eq!(replies.len(), 1);
21547            // Proof rechecking allowed the relay. Independent tag settlement
21548            // still asks the opener to retry after any same-epoch content change.
21549            assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
21550            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21551            while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
21552            let route = rig
21553                .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
21554                .await;
21555            assert!(rig.live(&route));
21556        }
21557
21558        #[tokio::test]
21559        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
21560        {
21561            let mut rig = rig_with_flow_support(false).await;
21562            rig.sync(vec![session(1)]).await;
21563            let route = rig
21564                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21565                .await;
21566            assert!(rig.live(&route));
21567            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
21568            let mut record = session(1);
21569            record.attributes.flow_id = Some("flow:7".to_string());
21570            rig.sync(vec![record]).await;
21571            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
21572            assert!(rig.live(&unscoped));
21573            assert_eq!(unscoped.stamp(), None);
21574        }
21575
21576        #[tokio::test]
21577        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
21578            let mut rig = rig().await;
21579            let mut record = session(1);
21580            record.attributes.flow_id = Some("flow:7".to_string());
21581            rig.sync(vec![record.clone()]).await;
21582            let before = rig.forwarding.published_scope_tag(OWNER, "s");
21583            let mut owner_route = rig
21584                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21585                .await;
21586            let mut carrier_route = rig
21587                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
21588                .await;
21589            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
21590            rig.sync(vec![record.clone()]).await;
21591            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21592            assert!(rig.live(&owner_route) && owner_route.untouched());
21593            assert!(rig.live(&carrier_route) && carrier_route.untouched());
21594
21595            record.attributes.flow_id = Some("flow:8".to_string());
21596            rig.sync(vec![record]).await;
21597            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
21598            let before = before.unwrap();
21599            assert_eq!(after.scope_epoch, before.scope_epoch);
21600            assert!(after.version > before.version);
21601            for route in [&mut owner_route, &mut carrier_route] {
21602                assert!(!rig.live(route));
21603                assert_eq!(
21604                    route.closed_reason(),
21605                    RouteCloseReason::ScopeDelegationChanged
21606                );
21607            }
21608            assert!(rig.live(&unscoped) && unscoped.untouched());
21609            // Each provider also receives a GOODBYE for its drained route;
21610            // consume it before expecting the next route.bind on that sink.
21611            for target in [PLEXUS, OTHER] {
21612                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
21613                let goodbye = module_rx
21614                    .try_recv()
21615                    .expect("the provider sees the drain")
21616                    .frame;
21617                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21618                assert!(module_rx.try_recv().is_err());
21619            }
21620            let rebound = rig
21621                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21622                .await;
21623            assert_eq!(
21624                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
21625                Some("flow:8")
21626            );
21627        }
21628
21629        /// The owner's sync lands between admission and the module's ack. The
21630        /// open is refused by name, the module's other routes stay up, and the
21631        /// reserved pair is released. Changed content is retryable; an ended
21632        /// scope is not.
21633        #[tokio::test]
21634        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
21635            let mut rig = rig().await;
21636            rig.sync(vec![session(1)]).await;
21637            let mut cotenant = rig
21638                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
21639                .await;
21640
21641            let mut changed = session(1);
21642            changed.child_owners.push(Principal::Reserved {
21643                module_id: MAGIC.to_string(),
21644            });
21645            let mut ended = None;
21646            for (code, next) in [
21647                (error_codes::SCOPE_CHANGED, vec![changed]),
21648                (error_codes::SCOPE_ENDED, Vec::new()),
21649            ] {
21650                let relayed = rig
21651                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21652                    .await;
21653                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
21654                ended = Some(next.is_empty());
21655                rig.sync(next).await;
21656                rig.ack(&relayed).await;
21657                let replies = relayed.task.await.unwrap();
21658                assert_eq!(replies.len(), 1, "{replies:?}");
21659                assert_eq!(parse_error(&replies[0])["code"], code);
21660                assert_eq!(
21661                    subc_protocol::error_codes::is_retryable_route_open(code),
21662                    code == error_codes::SCOPE_CHANGED
21663                );
21664                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
21665                // The module is told to drop just the binding it created.
21666                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21667                // Collected, because ending the scope also closes the co-tenant
21668                // route, whose GOODBYE comes first.
21669                let mut goodbyes = Vec::new();
21670                while let Ok(outbound) = plexus_rx.try_recv() {
21671                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
21672                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
21673                }
21674                assert!(
21675                    goodbyes.contains(&(bind_channel, bind_epoch)),
21676                    "{goodbyes:?}"
21677                );
21678                assert!(rig
21679                    .handler
21680                    .registry
21681                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
21682                    .unwrap()
21683                    .is_some());
21684            }
21685            assert_eq!(ended, Some(true));
21686            // The co-tenant stayed up through the change, and closed only when
21687            // the scope ended, by the drain rule rather than by the commit.
21688            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
21689        }
21690
21691        /// Each row of the drain table on one set of routes: the owner's, a
21692        /// bare carrier's, and a targeted carrier's to each of its targets.
21693        #[tokio::test]
21694        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
21695            struct Case {
21696                name: &'static str,
21697                change: fn(&mut ScopeRecord),
21698                /// Closed routes by index: owner->plexus, aft->plexus,
21699                /// broca->plexus, broca->other.
21700                closed: [Option<RouteCloseReason>; 4],
21701            }
21702            use RouteCloseReason::*;
21703            let cases = [
21704                Case {
21705                    name: "a carrier entry removed",
21706                    change: |r| {
21707                        r.carriers.retain(|c| {
21708                            c.principal
21709                                != Principal::Reserved {
21710                                    module_id: AFT.to_string(),
21711                                }
21712                        })
21713                    },
21714                    closed: [None, Some(ScopeCarrierRemoved), None, None],
21715                },
21716                Case {
21717                    name: "a target removed from a carrier",
21718                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
21719                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
21720                },
21721                Case {
21722                    name: "a bare carrier narrowed to targets",
21723                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
21724                    closed: [None, Some(ScopeCarrierRemoved), None, None],
21725                },
21726                Case {
21727                    name: "delegates turned off",
21728                    change: |r| r.attributes.delegates = false,
21729                    closed: [Some(ScopeDelegationChanged); 4],
21730                },
21731                Case {
21732                    name: "agent_id changed",
21733                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
21734                    closed: [Some(ScopeDelegationChanged); 4],
21735                },
21736                Case {
21737                    name: "a carrier added, child owners changed, the record re-sent",
21738                    change: |r| {
21739                        r.carriers.push(carrier(MAGIC, None));
21740                        r.child_owners.push(Principal::Reserved {
21741                            module_id: MAGIC.to_string(),
21742                        });
21743                    },
21744                    closed: [None; 4],
21745                },
21746                Case {
21747                    name: "a target added",
21748                    change: |r| {
21749                        r.carriers[1]
21750                            .targets
21751                            .as_mut()
21752                            .unwrap()
21753                            .push("third".to_string())
21754                    },
21755                    closed: [None; 4],
21756                },
21757                Case {
21758                    name: "delegates turned on",
21759                    change: |r| r.attributes.delegates = true,
21760                    closed: [None; 4],
21761                },
21762            ];
21763            for case in cases {
21764                let mut rig = rig().await;
21765                rig.sync(vec![session(1)]).await;
21766                let scope = || Some(rig_selector("s", Some(1)));
21767                let mut routes = [
21768                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
21769                    rig.bound(Some(AFT), PLEXUS, scope()).await,
21770                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
21771                    rig.bound(Some(BROCA), OTHER, scope()).await,
21772                ];
21773                let mut record = session(1);
21774                (case.change)(&mut record);
21775                rig.sync(vec![record]).await;
21776                for (index, expected) in case.closed.iter().enumerate() {
21777                    let route = &mut routes[index];
21778                    match expected {
21779                        Some(reason) => {
21780                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
21781                            assert_eq!(
21782                                route.closed_reason(),
21783                                *reason,
21784                                "{}: route {index}",
21785                                case.name
21786                            );
21787                        }
21788                        None => {
21789                            assert!(rig.live(route), "{}: route {index} closed", case.name);
21790                            assert!(
21791                                route.untouched(),
21792                                "{}: route {index} was told something",
21793                                case.name
21794                            );
21795                        }
21796                    }
21797                }
21798            }
21799        }
21800
21801        #[tokio::test]
21802        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
21803            // Removed, and replaced by a higher epoch.
21804            for next in [Vec::new(), vec![session(2)]] {
21805                let mut rig = rig().await;
21806                rig.sync(vec![session(1)]).await;
21807                let mut route = rig
21808                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21809                    .await;
21810                rig.sync(next).await;
21811                assert!(!rig.live(&route));
21812                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
21813            }
21814
21815            // A child whose parent ends: its routes close as parent-ended, the
21816            // child stays live, and routes under the parent close as ended.
21817            let mut rig = rig().await;
21818            let mut child = session(1);
21819            child.scope_ref = "child".to_string();
21820            child.kind = ScopeKind::Worker;
21821            child.parent = Some(ScopeParent::new(
21822                Principal::Reserved {
21823                    module_id: OWNER.to_string(),
21824                },
21825                "s".to_string(),
21826                1,
21827            ));
21828            rig.sync(vec![session(1), child.clone()]).await;
21829            let mut child_route = rig
21830                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
21831                .await;
21832            assert_eq!(
21833                child_route.stamp().unwrap().parent_state,
21834                Some(ParentState::Linked)
21835            );
21836            rig.sync(vec![child]).await;
21837            assert!(!rig.live(&child_route));
21838            assert_eq!(
21839                child_route.closed_reason(),
21840                RouteCloseReason::ScopeParentEnded
21841            );
21842        }
21843
21844        #[tokio::test]
21845        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
21846        ) {
21847            let mut rig = rig().await;
21848            rig.sync(vec![session(1)]).await;
21849            let mut route = rig
21850                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21851                .await;
21852            let before = rig.forwarding.published_scope_tag(OWNER, "s");
21853            rig.sync(vec![session(1)]).await;
21854            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
21855            assert!(rig.live(&route) && route.untouched());
21856
21857            // A call in flight on the route when another carrier is added. A
21858            // forwarded REQUEST holds one credit on the route's flow until the
21859            // module answers; the router takes it exactly like this.
21860            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
21861                .forwarding
21862                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
21863                .unwrap()
21864            else {
21865                panic!("the route is bound");
21866            };
21867            binding.flow.acquire_tagged(9, false).await.unwrap();
21868            let mut widened = session(1);
21869            widened.carriers.push(carrier(MAGIC, None));
21870            rig.sync(vec![widened]).await;
21871            assert!(rig.live(&route) && route.untouched());
21872            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21873            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
21874            // The call's credit is still held on an open flow, so its answer
21875            // will be delivered: closing the route would have closed the flow.
21876            assert_eq!(binding.flow.in_flight(), 1);
21877            binding
21878                .flow
21879                .acquire_tagged(10, false)
21880                .await
21881                .expect("the flow is still open");
21882        }
21883
21884        /// A swap's superseded endpoint keeps its routes until drained; ending
21885        /// the scope closes them there too.
21886        #[tokio::test]
21887        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
21888            let mut rig = rig().await;
21889            rig.sync(vec![session(1)]).await;
21890            let mut on_incumbent = rig
21891                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
21892                .await;
21893
21894            // Swap plexus: register a candidate and cut over, leaving the
21895            // incumbent superseded with the route still on it.
21896            let (candidate, _candidate_rx) = wide_ctx(9);
21897            let registration = rig
21898                .handler
21899                .registry
21900                .register_candidate_with_control_ops(
21901                    manifest(PLEXUS, PROTOCOL_VERSION),
21902                    PROTOCOL_VERSION,
21903                    candidate.connection_id,
21904                    module_baseline_control_ops(),
21905                )
21906                .unwrap();
21907            rig.forwarding
21908                .register_candidate_module_connection(
21909                    candidate.connection_id,
21910                    PLEXUS.to_string(),
21911                    PROTOCOL_VERSION,
21912                    manifest_concurrency(&registration.manifest),
21913                    candidate.egress.clone(),
21914                )
21915                .unwrap();
21916            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
21917            rig.handler
21918                .registry
21919                .promote_candidate(PLEXUS)
21920                .unwrap()
21921                .unwrap();
21922            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
21923
21924            rig.sync(Vec::new()).await;
21925            assert!(!rig.live(&on_incumbent));
21926            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
21927            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
21928            let goodbye = incumbent_rx
21929                .try_recv()
21930                .expect("the superseded endpoint is told")
21931                .frame;
21932            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
21933        }
21934    }
21935}
21936
21937#[cfg(test)]
21938mod concurrency_default_exposure_tests {
21939    use super::*;
21940
21941    fn hello_body(role_json: &str) -> Vec<u8> {
21942        format!(
21943            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
21944        )
21945        .into_bytes()
21946    }
21947
21948    fn manifest_from(body: &[u8]) -> ModuleManifest {
21949        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
21950        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
21951            .expect("manifest parses")
21952    }
21953
21954    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
21955
21956    #[test]
21957    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
21958        let body = hello_body(&format!(
21959            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
21960        ));
21961        let manifest = manifest_from(&body);
21962        // Precondition: serde really resolved it to the default, so the typed
21963        // manifest alone cannot answer the question this probe exists for.
21964        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21965        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
21966    }
21967
21968    #[test]
21969    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
21970        let body = hello_body(&format!(
21971            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
21972        ));
21973        let manifest = manifest_from(&body);
21974        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
21975        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21976    }
21977
21978    #[test]
21979    fn non_management_roles_are_never_reported() {
21980        let body = hello_body(
21981            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
21982        );
21983        let manifest = manifest_from(&body);
21984        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
21985    }
21986}